weaselab/conflict-set
2
0
Fork 0
Code Issues 4 Packages Releases 13 Activity

Add artificial root parent and stop plumbing impl everywhere

Browse Source
This commit is contained in:
Andrew Noyes
2024-11-21 16:32:13 -08:00
parent 972f16ed8f
commit 0cea5565b5
1 changed files with 142 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

288
ConflictSet.cpp
View File

@@ -1114,8 +1114,9 @@ InternalVersionT exchangeMaxVersion(Node *n, InternalVersionT newMax) {
} }
} }
// Precondition `n` is not the root
void setMaxVersion(Node *n, InternalVersionT newMax) { void setMaxVersion(Node *n, InternalVersionT newMax) {
// This should not be the rootParent
assert(n->parent != nullptr);
assert(newMax >= InternalVersionT::zero); assert(newMax >= InternalVersionT::zero);
int index = n->parentsIndex; int index = n->parentsIndex;
n = n->parent; n = n->parent;
@@ -1158,8 +1159,7 @@ void setMaxVersion(Node *n, InternalVersionT newMax) {
} }
} }
// If impl is nullptr, then n->parent must not be nullptr TaggedNodePointer &getInTree(Node *n);
TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *impl);
TaggedNodePointer getChild(Node0 *, uint8_t) { return nullptr; } TaggedNodePointer getChild(Node0 *, uint8_t) { return nullptr; }
TaggedNodePointer getChild(Node3 *self, uint8_t index) { TaggedNodePointer getChild(Node3 *self, uint8_t index) {
@@ -1375,13 +1375,11 @@ bool checkRangeVersionOfFirstGeq(Node *node, InternalVersionT readVersion) {
} }
// self must not be the root // self must not be the root
void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext, void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext);
ConflictSet::Impl *impl);
void consumePartialKeyFull(TaggedNodePointer &self, TrivialSpan &key, void consumePartialKeyFull(TaggedNodePointer &self, TrivialSpan &key,
InternalVersionT writeVersion, InternalVersionT writeVersion,
WriteContext *writeContext, WriteContext *writeContext) {
ConflictSet::Impl *impl) {
// Handle an existing partial key // Handle an existing partial key
int commonLen = std::min<int>(self->partialKeyLen, key.size()); int commonLen = std::min<int>(self->partialKeyLen, key.size());
int partialKeyIndex = int partialKeyIndex =
@@ -1419,7 +1417,7 @@ void consumePartialKeyFull(TaggedNodePointer &self, TrivialSpan &key,
partialKeyIndex + 1); partialKeyIndex + 1);
// Maintain memory capacity invariant // Maintain memory capacity invariant
maybeDecreaseCapacity(old, writeContext, impl); maybeDecreaseCapacity(old, writeContext);
} }
key = key.subspan(partialKeyIndex, key.size() - partialKeyIndex); key = key.subspan(partialKeyIndex, key.size() - partialKeyIndex);
} }
@@ -1428,10 +1426,9 @@ void consumePartialKeyFull(TaggedNodePointer &self, TrivialSpan &key,
// `key` such that `self` is along the search path of `key` // `key` such that `self` is along the search path of `key`
inline __attribute__((always_inline)) void inline __attribute__((always_inline)) void
consumePartialKey(TaggedNodePointer &self, TrivialSpan &key, consumePartialKey(TaggedNodePointer &self, TrivialSpan &key,
InternalVersionT writeVersion, WriteContext *writeContext, InternalVersionT writeVersion, WriteContext *writeContext) {
ConflictSet::Impl *impl) {
if (self->partialKeyLen > 0) { if (self->partialKeyLen > 0) {
consumePartialKeyFull(self, key, writeVersion, writeContext, impl); consumePartialKeyFull(self, key, writeVersion, writeContext);
} }
} }
@@ -1441,8 +1438,7 @@ consumePartialKey(TaggedNodePointer &self, TrivialSpan &key,
// `maxVersion` for result. // `maxVersion` for result.
TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key, TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key,
InternalVersionT newMaxVersion, InternalVersionT newMaxVersion,
WriteContext *writeContext, WriteContext *writeContext) {
ConflictSet::Impl *impl) {
int index = key.front(); int index = key.front();
key = key.subspan(1, key.size() - 1); key = key.subspan(1, key.size() - 1);
@@ -1455,8 +1451,7 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key,
auto *self3 = static_cast<Node3 *>(self); auto *self3 = static_cast<Node3 *>(self);
int i = getNodeIndex(self3, index); int i = getNodeIndex(self3, index);
if (i >= 0) { if (i >= 0) {
consumePartialKey(self3->children[i], key, newMaxVersion, writeContext, consumePartialKey(self3->children[i], key, newMaxVersion, writeContext);
impl);
self3->childMaxVersion[i] = newMaxVersion; self3->childMaxVersion[i] = newMaxVersion;
return self3->children[i]; return self3->children[i];
} }
@@ -1465,8 +1460,7 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key,
auto *self16 = static_cast<Node16 *>(self); auto *self16 = static_cast<Node16 *>(self);
int i = getNodeIndex(self16, index); int i = getNodeIndex(self16, index);
if (i >= 0) { if (i >= 0) {
consumePartialKey(self16->children[i], key, newMaxVersion, writeContext, consumePartialKey(self16->children[i], key, newMaxVersion, writeContext);
impl);
self16->childMaxVersion[i] = newMaxVersion; self16->childMaxVersion[i] = newMaxVersion;
return self16->children[i]; return self16->children[i];
} }
@@ -1476,7 +1470,7 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key,
int secondIndex = self48->index[index]; int secondIndex = self48->index[index];
if (secondIndex >= 0) { if (secondIndex >= 0) {
consumePartialKey(self48->children[secondIndex], key, newMaxVersion, consumePartialKey(self48->children[secondIndex], key, newMaxVersion,
writeContext, impl); writeContext);
self48->childMaxVersion[secondIndex] = newMaxVersion; self48->childMaxVersion[secondIndex] = newMaxVersion;
self48->maxOfMax[secondIndex >> Node48::kMaxOfMaxShift] = self48->maxOfMax[secondIndex >> Node48::kMaxOfMaxShift] =
std::max(self48->maxOfMax[secondIndex >> Node48::kMaxOfMaxShift], std::max(self48->maxOfMax[secondIndex >> Node48::kMaxOfMaxShift],
@@ -1487,7 +1481,7 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key,
case Type_Node256: { case Type_Node256: {
auto *self256 = static_cast<Node256 *>(self); auto *self256 = static_cast<Node256 *>(self);
if (auto &result = self256->children[index]; result != nullptr) { if (auto &result = self256->children[index]; result != nullptr) {
consumePartialKey(result, key, newMaxVersion, writeContext, impl); consumePartialKey(result, key, newMaxVersion, writeContext);
self256->childMaxVersion[index] = newMaxVersion; self256->childMaxVersion[index] = newMaxVersion;
self256->maxOfMax[index >> Node256::kMaxOfMaxShift] = std::max( self256->maxOfMax[index >> Node256::kMaxOfMaxShift] = std::max(
self256->maxOfMax[index >> Node256::kMaxOfMaxShift], newMaxVersion); self256->maxOfMax[index >> Node256::kMaxOfMaxShift], newMaxVersion);
@@ -1666,15 +1660,15 @@ downLeftSpine:
return node; return node;
} }
void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext, void freeAndMakeCapacity(Node *&self, int capacity,
ConflictSet::Impl *impl) { WriteContext *writeContext) {
++writeContext->accum.nodes_resized; ++writeContext->accum.nodes_resized;
switch (self->getType()) { switch (self->getType()) {
case Type_Node0: { case Type_Node0: {
auto *self0 = (Node0 *)self; auto *self0 = (Node0 *)self;
auto *newSelf = writeContext->allocate<Node0>(capacity); auto *newSelf = writeContext->allocate<Node0>(capacity);
newSelf->copyChildrenAndKeyFrom(*self0); newSelf->copyChildrenAndKeyFrom(*self0);
getInTree(self, impl) = newSelf; getInTree(self) = newSelf;
writeContext->deferRelease(self0, newSelf); writeContext->deferRelease(self0, newSelf);
self = newSelf; self = newSelf;
} break; } break;
@@ -1682,7 +1676,7 @@ void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext,
auto *self3 = (Node3 *)self; auto *self3 = (Node3 *)self;
auto *newSelf = writeContext->allocate<Node3>(capacity); auto *newSelf = writeContext->allocate<Node3>(capacity);
newSelf->copyChildrenAndKeyFrom(*self3); newSelf->copyChildrenAndKeyFrom(*self3);
getInTree(self, impl) = newSelf; getInTree(self) = newSelf;
writeContext->deferRelease(self3, newSelf); writeContext->deferRelease(self3, newSelf);
self = newSelf; self = newSelf;
} break; } break;
@@ -1690,7 +1684,7 @@ void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext,
auto *self16 = (Node16 *)self; auto *self16 = (Node16 *)self;
auto *newSelf = writeContext->allocate<Node16>(capacity); auto *newSelf = writeContext->allocate<Node16>(capacity);
newSelf->copyChildrenAndKeyFrom(*self16); newSelf->copyChildrenAndKeyFrom(*self16);
getInTree(self, impl) = newSelf; getInTree(self) = newSelf;
writeContext->deferRelease(self16, newSelf); writeContext->deferRelease(self16, newSelf);
self = newSelf; self = newSelf;
} break; } break;
@@ -1698,7 +1692,7 @@ void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext,
auto *self48 = (Node48 *)self; auto *self48 = (Node48 *)self;
auto *newSelf = writeContext->allocate<Node48>(capacity); auto *newSelf = writeContext->allocate<Node48>(capacity);
newSelf->copyChildrenAndKeyFrom(*self48); newSelf->copyChildrenAndKeyFrom(*self48);
getInTree(self, impl) = newSelf; getInTree(self) = newSelf;
writeContext->deferRelease(self48, newSelf); writeContext->deferRelease(self48, newSelf);
self = newSelf; self = newSelf;
} break; } break;
@@ -1706,7 +1700,7 @@ void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext,
auto *self256 = (Node256 *)self; auto *self256 = (Node256 *)self;
auto *newSelf = writeContext->allocate<Node256>(capacity); auto *newSelf = writeContext->allocate<Node256>(capacity);
newSelf->copyChildrenAndKeyFrom(*self256); newSelf->copyChildrenAndKeyFrom(*self256);
getInTree(self, impl) = newSelf; getInTree(self) = newSelf;
writeContext->deferRelease(self256, newSelf); writeContext->deferRelease(self256, newSelf);
self = newSelf; self = newSelf;
} break; } break;
@@ -1716,8 +1710,7 @@ void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext,
} }
// Fix larger-than-desired capacities. self must not be the root // Fix larger-than-desired capacities. self must not be the root
void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext, void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext) {
ConflictSet::Impl *impl) {
const int maxCapacity = const int maxCapacity =
(self->numChildren + int(self->entryPresent)) * (self->partialKeyLen + 1); (self->numChildren + int(self->entryPresent)) * (self->partialKeyLen + 1);
@@ -1729,7 +1722,7 @@ void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext,
if (self->getCapacity() <= maxCapacity) { if (self->getCapacity() <= maxCapacity) {
return; return;
} }
freeAndMakeCapacity(self, self->partialKeyLen, writeContext, impl); freeAndMakeCapacity(self, self->partialKeyLen, writeContext);
} }
#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__) #if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
@@ -1799,13 +1792,13 @@ void rezero(Node *n, InternalVersionT z) {
#endif #endif
void mergeWithChild(TaggedNodePointer &self, WriteContext *writeContext, void mergeWithChild(TaggedNodePointer &self, WriteContext *writeContext,
Node3 *self3, ConflictSet::Impl *impl) { Node3 *self3) {
assert(!self3->entryPresent); assert(!self3->entryPresent);
Node *child = self3->children[0]; Node *child = self3->children[0];
const int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen; const int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen;
if (minCapacity > child->getCapacity()) { if (minCapacity > child->getCapacity()) {
freeAndMakeCapacity(child, minCapacity, writeContext, impl); freeAndMakeCapacity(child, minCapacity, writeContext);
} }
// Merge partial key with child // Merge partial key with child
@@ -1826,9 +1819,6 @@ void mergeWithChild(TaggedNodePointer &self, WriteContext *writeContext,
child->parent = self->parent; child->parent = self->parent;
child->parentsIndex = self->parentsIndex; child->parentsIndex = self->parentsIndex;
// Max versions are stored in the parent, so we need to update it now
// that we have a new parent. Safe to call since the root never has a partial
// key.
setMaxVersion(child, std::max(childMaxVersion, writeContext->zero)); setMaxVersion(child, std::max(childMaxVersion, writeContext->zero));
self = child; self = child;
@@ -1841,63 +1831,59 @@ bool needsDownsize(Node *n) {
return n->numChildren + n->entryPresent < minTable[n->getType()]; return n->numChildren + n->entryPresent < minTable[n->getType()];
} }
void downsize(Node3 *self, WriteContext *writeContext, void downsize(Node3 *self, WriteContext *writeContext) {
ConflictSet::Impl *impl) {
if (self->numChildren == 0) { if (self->numChildren == 0) {
auto *newSelf = writeContext->allocate<Node0>(self->partialKeyLen); auto *newSelf = writeContext->allocate<Node0>(self->partialKeyLen);
newSelf->copyChildrenAndKeyFrom(*self); newSelf->copyChildrenAndKeyFrom(*self);
getInTree(self, impl) = newSelf; getInTree(self) = newSelf;
writeContext->deferRelease(self, newSelf); writeContext->deferRelease(self, newSelf);
} else { } else {
assert(self->numChildren == 1 && !self->entryPresent); assert(self->numChildren == 1 && !self->entryPresent);
mergeWithChild(getInTree(self, impl), writeContext, self, impl); mergeWithChild(getInTree(self), writeContext, self);
} }
} }
void downsize(Node16 *self, WriteContext *writeContext, void downsize(Node16 *self, WriteContext *writeContext) {
ConflictSet::Impl *impl) {
assert(self->numChildren + int(self->entryPresent) < kMinChildrenNode16); assert(self->numChildren + int(self->entryPresent) < kMinChildrenNode16);
auto *newSelf = writeContext->allocate<Node3>(self->partialKeyLen); auto *newSelf = writeContext->allocate<Node3>(self->partialKeyLen);
newSelf->copyChildrenAndKeyFrom(*self); newSelf->copyChildrenAndKeyFrom(*self);
getInTree(self, impl) = newSelf; getInTree(self) = newSelf;
writeContext->deferRelease(self, newSelf); writeContext->deferRelease(self, newSelf);
} }
void downsize(Node48 *self, WriteContext *writeContext, void downsize(Node48 *self, WriteContext *writeContext) {
ConflictSet::Impl *impl) {
assert(self->numChildren + int(self->entryPresent) < kMinChildrenNode48); assert(self->numChildren + int(self->entryPresent) < kMinChildrenNode48);
auto *newSelf = writeContext->allocate<Node16>(self->partialKeyLen); auto *newSelf = writeContext->allocate<Node16>(self->partialKeyLen);
newSelf->copyChildrenAndKeyFrom(*self); newSelf->copyChildrenAndKeyFrom(*self);
getInTree(self, impl) = newSelf; getInTree(self) = newSelf;
writeContext->deferRelease(self, newSelf); writeContext->deferRelease(self, newSelf);
} }
void downsize(Node256 *self, WriteContext *writeContext, void downsize(Node256 *self, WriteContext *writeContext) {
ConflictSet::Impl *impl) {
assert(self->numChildren + int(self->entryPresent) < kMinChildrenNode256); assert(self->numChildren + int(self->entryPresent) < kMinChildrenNode256);
auto *self256 = (Node256 *)self; auto *self256 = (Node256 *)self;
auto *newSelf = writeContext->allocate<Node48>(self->partialKeyLen); auto *newSelf = writeContext->allocate<Node48>(self->partialKeyLen);
newSelf->copyChildrenAndKeyFrom(*self256); newSelf->copyChildrenAndKeyFrom(*self256);
getInTree(self, impl) = newSelf; getInTree(self) = newSelf;
writeContext->deferRelease(self256, newSelf); writeContext->deferRelease(self256, newSelf);
} }
void downsize(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl) { void downsize(Node *self, WriteContext *writeContext) {
switch (self->getType()) { switch (self->getType()) {
case Type_Node0: // GCOVR_EXCL_LINE case Type_Node0: // GCOVR_EXCL_LINE
__builtin_unreachable(); // GCOVR_EXCL_LINE __builtin_unreachable(); // GCOVR_EXCL_LINE
case Type_Node3: case Type_Node3:
downsize(static_cast<Node3 *>(self), writeContext, impl); downsize(static_cast<Node3 *>(self), writeContext);
break; break;
case Type_Node16: case Type_Node16:
downsize(static_cast<Node16 *>(self), writeContext, impl); downsize(static_cast<Node16 *>(self), writeContext);
break; break;
case Type_Node48: case Type_Node48:
downsize(static_cast<Node48 *>(self), writeContext, impl); downsize(static_cast<Node48 *>(self), writeContext);
break; break;
case Type_Node256: case Type_Node256:
downsize(static_cast<Node256 *>(self), writeContext, impl); downsize(static_cast<Node256 *>(self), writeContext);
break; break;
default: // GCOVR_EXCL_LINE default: // GCOVR_EXCL_LINE
__builtin_unreachable(); // GCOVR_EXCL_LINE __builtin_unreachable(); // GCOVR_EXCL_LINE
@@ -1908,8 +1894,7 @@ void downsize(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl) {
// path to self. May invalidate children of self->parent. Returns a pointer to // path to self. May invalidate children of self->parent. Returns a pointer to
// the node after self. Precondition: `self->entryPresent` // the node after self. Precondition: `self->entryPresent`
Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl, Node *erase(Node *self, WriteContext *writeContext, bool logical) {
bool logical) {
++writeContext->accum.entries_erased; ++writeContext->accum.entries_erased;
assert(self->parent != nullptr); assert(self->parent != nullptr);
@@ -1928,12 +1913,12 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl,
if (self->numChildren != 0) { if (self->numChildren != 0) {
if (needsDownsize(self)) { if (needsDownsize(self)) {
downsize(self, writeContext, impl); downsize(self, writeContext);
} }
while (self->releaseDeferred) { while (self->releaseDeferred) {
self = self->forwardTo; self = self->forwardTo;
} }
maybeDecreaseCapacity(self, writeContext, impl); maybeDecreaseCapacity(self, writeContext);
if (result != nullptr) { if (result != nullptr) {
while (result->releaseDeferred) { while (result->releaseDeferred) {
result = result->forwardTo; result = result->forwardTo;
@@ -1964,7 +1949,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl,
sizeof(parent3->children[0])); sizeof(parent3->children[0]));
if (needsDownsize(parent3)) { if (needsDownsize(parent3)) {
downsize(parent3, writeContext, impl); downsize(parent3, writeContext);
} }
} break; } break;
case Type_Node16: { case Type_Node16: {
@@ -1983,7 +1968,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl,
sizeof(parent16->children[0])); sizeof(parent16->children[0]));
if (needsDownsize(parent16)) { if (needsDownsize(parent16)) {
downsize(parent16, writeContext, impl); downsize(parent16, writeContext);
} }
} break; } break;
case Type_Node48: { case Type_Node48: {
@@ -2013,7 +1998,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl,
sizeof(parent48->children[0])); sizeof(parent48->children[0]));
if (needsDownsize(parent48)) { if (needsDownsize(parent48)) {
downsize(parent48, writeContext, impl); downsize(parent48, writeContext);
} }
} break; } break;
case Type_Node256: { case Type_Node256: {
@@ -2024,7 +2009,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl,
--parent->numChildren; --parent->numChildren;
if (needsDownsize(parent256)) { if (needsDownsize(parent256)) {
downsize(parent256, writeContext, impl); downsize(parent256, writeContext);
} }
} break; } break;
default: // GCOVR_EXCL_LINE default: // GCOVR_EXCL_LINE
@@ -2034,7 +2019,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl,
while (parent->releaseDeferred) { while (parent->releaseDeferred) {
parent = parent->forwardTo; parent = parent->forwardTo;
} }
maybeDecreaseCapacity(parent, writeContext, impl); maybeDecreaseCapacity(parent, writeContext);
if (result != nullptr) { if (result != nullptr) {
while (result->releaseDeferred) { while (result->releaseDeferred) {
@@ -2631,11 +2616,13 @@ TrivialSpan getSearchPath(Arena &arena, Node *n) {
for (int i = n->partialKeyLen - 1; i >= 0; --i) { for (int i = n->partialKeyLen - 1; i >= 0; --i) {
result.push_back(n->partialKey()[i]); result.push_back(n->partialKey()[i]);
} }
if (n->parent == nullptr) {
break;
}
result.push_back(n->parentsIndex); result.push_back(n->parentsIndex);
n = n->parent; n = n->parent;
if (n->parent == nullptr) {
// Implicit byte from rootParent
result.pop_back();
break;
}
} }
std::reverse(result.begin(), result.end()); std::reverse(result.begin(), result.end());
return {result.begin(), result.size()}; return {result.begin(), result.size()};
@@ -2735,12 +2722,13 @@ checkMaxBetweenExclusiveImpl<true>(Node256 *n, int begin, int end,
// of the result will have `maxVersion` set to `writeVersion` as a // of the result will have `maxVersion` set to `writeVersion` as a
// postcondition. Nodes along the search path may be invalidated. Callers must // postcondition. Nodes along the search path may be invalidated. Callers must
// ensure that the max version of the self argument is updated. // ensure that the max version of the self argument is updated.
[[nodiscard]] TaggedNodePointer * [[nodiscard]] TaggedNodePointer *insert(TaggedNodePointer *self,
insert(TaggedNodePointer *self, TrivialSpan key, InternalVersionT writeVersion, TrivialSpan key,
WriteContext *writeContext, ConflictSet::Impl *impl) { InternalVersionT writeVersion,
WriteContext *writeContext) {
for (; key.size() != 0; ++writeContext->accum.insert_iterations) { for (; key.size() != 0; ++writeContext->accum.insert_iterations) {
self = &getOrCreateChild(*self, key, writeVersion, writeContext, impl); self = &getOrCreateChild(*self, key, writeVersion, writeContext);
} }
return self; return self;
} }
@@ -2798,10 +2786,9 @@ void eraseTree(Node *root, WriteContext *writeContext) {
} }
void addPointWrite(TaggedNodePointer &root, TrivialSpan key, void addPointWrite(TaggedNodePointer &root, TrivialSpan key,
InternalVersionT writeVersion, WriteContext *writeContext, InternalVersionT writeVersion, WriteContext *writeContext) {
ConflictSet::Impl *impl) {
++writeContext->accum.point_writes; ++writeContext->accum.point_writes;
auto n = *insert(&root, key, writeVersion, writeContext, impl); auto n = *insert(&root, key, writeVersion, writeContext);
if (!n->entryPresent) { if (!n->entryPresent) {
++writeContext->accum.entries_inserted; ++writeContext->accum.entries_inserted;
auto *p = nextLogical(n); auto *p = nextLogical(n);
@@ -2930,13 +2917,12 @@ struct AddedWriteRange {
AddedWriteRange addWriteRange(Node *beginRoot, TrivialSpan begin, Node *endRoot, AddedWriteRange addWriteRange(Node *beginRoot, TrivialSpan begin, Node *endRoot,
TrivialSpan end, InternalVersionT writeVersion, TrivialSpan end, InternalVersionT writeVersion,
WriteContext *writeContext, WriteContext *writeContext) {
ConflictSet::Impl *impl) {
++writeContext->accum.range_writes; ++writeContext->accum.range_writes;
Node *beginNode = *insert(&getInTree(beginRoot, impl), begin, writeVersion, Node *beginNode =
writeContext, impl); *insert(&getInTree(beginRoot), begin, writeVersion, writeContext);
addKey(beginNode); addKey(beginNode);
if (!beginNode->entryPresent) { if (!beginNode->entryPresent) {
++writeContext->accum.entries_inserted; ++writeContext->accum.entries_inserted;
@@ -2951,8 +2937,7 @@ AddedWriteRange addWriteRange(Node *beginRoot, TrivialSpan begin, Node *endRoot,
} }
beginNode->entry.pointVersion = writeVersion; beginNode->entry.pointVersion = writeVersion;
Node *endNode = Node *endNode = *insert(&getInTree(endRoot), end, writeVersion, writeContext);
*insert(&getInTree(endRoot, impl), end, writeVersion, writeContext, impl);
addKey(endNode); addKey(endNode);
if (!endNode->entryPresent) { if (!endNode->entryPresent) {
@@ -2971,8 +2956,7 @@ AddedWriteRange addWriteRange(Node *beginRoot, TrivialSpan begin, Node *endRoot,
return {beginNode, endNode}; return {beginNode, endNode};
} }
void eraseInRange(Node *beginNode, Node *endNode, WriteContext *writeContext, void eraseInRange(Node *beginNode, Node *endNode, WriteContext *writeContext) {
ConflictSet::Impl *impl) {
auto checkNodesEq = [](Node *lhs, Node *&rhs) { auto checkNodesEq = [](Node *lhs, Node *&rhs) {
assert(!lhs->releaseDeferred); assert(!lhs->releaseDeferred);
while (rhs->releaseDeferred) [[unlikely]] { while (rhs->releaseDeferred) [[unlikely]] {
@@ -2984,7 +2968,7 @@ void eraseInRange(Node *beginNode, Node *endNode, WriteContext *writeContext,
// Erase nodes in range // Erase nodes in range
Node *iter = nextLogical(beginNode); Node *iter = nextLogical(beginNode);
for (; !checkNodesEq(iter, endNode); for (; !checkNodesEq(iter, endNode);
iter = erase(iter, writeContext, impl, /*logical*/ true)) { iter = erase(iter, writeContext, /*logical*/ true)) {
} }
// Inserting end trashed the last node's maxVersion. Fix that. Safe to call // Inserting end trashed the last node's maxVersion. Fix that. Safe to call
@@ -2993,22 +2977,21 @@ void eraseInRange(Node *beginNode, Node *endNode, WriteContext *writeContext,
} }
void addWriteRange(TaggedNodePointer &root, TrivialSpan begin, TrivialSpan end, void addWriteRange(TaggedNodePointer &root, TrivialSpan begin, TrivialSpan end,
InternalVersionT writeVersion, WriteContext *writeContext, InternalVersionT writeVersion, WriteContext *writeContext) {
ConflictSet::Impl *impl) {
int lcp = longestCommonPrefix(begin.data(), end.data(), int lcp = longestCommonPrefix(begin.data(), end.data(),
std::min(begin.size(), end.size())); std::min(begin.size(), end.size()));
if (lcp == begin.size() && end.size() == begin.size() + 1 && if (lcp == begin.size() && end.size() == begin.size() + 1 &&
end.back() == 0) { end.back() == 0) {
return addPointWrite(root, begin, writeVersion, writeContext, impl); return addPointWrite(root, begin, writeVersion, writeContext);
} }
auto useAsRoot = auto useAsRoot =
insert(&root, begin.subspan(0, lcp), writeVersion, writeContext, impl); insert(&root, begin.subspan(0, lcp), writeVersion, writeContext);
auto [beginNode, endNode] = addWriteRange( auto [beginNode, endNode] = addWriteRange(
*useAsRoot, begin.subspan(lcp, begin.size() - lcp), *useAsRoot, *useAsRoot, begin.subspan(lcp, begin.size() - lcp), *useAsRoot,
end.subspan(lcp, end.size() - lcp), writeVersion, writeContext, impl); end.subspan(lcp, end.size() - lcp), writeVersion, writeContext);
eraseInRange(beginNode, endNode, writeContext, impl); eraseInRange(beginNode, endNode, writeContext);
} }
Node *firstGeqPhysical(Node *n, const TrivialSpan key) { Node *firstGeqPhysical(Node *n, const TrivialSpan key) {
@@ -4780,10 +4763,6 @@ bool checkRangeRead(Node *n, TrivialSpan begin, TrivialSpan end,
return checkRangeRightSide(n, end, lcp, readVersion, readContext); return checkRangeRightSide(n, end, lcp, readVersion, readContext);
} }
// This makes it safe to check maxVersion within checkRangeLeftSide. If this
// were false, then we would have returned above since lcp == begin.size().
assert(!(n->parent == nullptr && begin.size() == 0));
return checkRangeStartsWith(n, begin.subspan(0, lcp), begin[lcp], end[lcp], return checkRangeStartsWith(n, begin.subspan(0, lcp), begin[lcp], end[lcp],
readVersion, readContext) && readVersion, readContext) &&
checkRangeLeftSide(n, begin, lcp + 1, readVersion, readContext) && checkRangeLeftSide(n, begin, lcp + 1, readVersion, readContext) &&
@@ -4803,14 +4782,16 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
} else { } else {
bool ok; bool ok;
if (reads[i].end.len == 0) { if (reads[i].end.len == 0) {
ok = checkPointRead( ok = checkPointRead(rootParent->children[0],
root, TrivialSpan(reads[i].begin.p, reads[i].begin.len), TrivialSpan(reads[i].begin.p, reads[i].begin.len),
InternalVersionT(reads[i].readVersion), &context.readContext); InternalVersionT(reads[i].readVersion),
&context.readContext);
} else { } else {
ok = checkRangeRead( ok = checkRangeRead(rootParent->children[0],
root, TrivialSpan(reads[i].begin.p, reads[i].begin.len), TrivialSpan(reads[i].begin.p, reads[i].begin.len),
TrivialSpan(reads[i].end.p, reads[i].end.len), TrivialSpan(reads[i].end.p, reads[i].end.len),
InternalVersionT(reads[i].readVersion), &context.readContext); InternalVersionT(reads[i].readVersion),
&context.readContext);
} }
result[i] = ok ? Commit : Conflict; result[i] = ok ? Commit : Conflict;
} }
@@ -4837,13 +4818,13 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
check::Job inProgress[kConcurrent]; check::Job inProgress[kConcurrent];
context.count = count; context.count = count;
context.oldestVersionFullPrecision = oldestVersionFullPrecision; context.oldestVersionFullPrecision = oldestVersionFullPrecision;
context.root = root; context.root = rootParent->children[0];
context.queries = reads; context.queries = reads;
context.results = result; context.results = result;
int64_t started = std::min(kConcurrent, count); int64_t started = std::min(kConcurrent, count);
context.started = started; context.started = started;
for (int i = 0; i < started; i++) { for (int i = 0; i < started; i++) {
inProgress[i].init(reads + i, result + i, root, inProgress[i].init(reads + i, result + i, rootParent->children[0],
oldestVersionFullPrecision); oldestVersionFullPrecision);
} }
for (int i = 0; i < started - 1; i++) { for (int i = 0; i < started - 1; i++) {
@@ -4921,7 +4902,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
interleaved_insert::Context context; interleaved_insert::Context context;
context.writeVersion = writeVersion; context.writeVersion = writeVersion;
context.count = count; context.count = count;
context.root = root; context.root = rootParent->children[0];
context.writes = writes; context.writes = writes;
context.results = stackResults; context.results = stackResults;
if (count > kStackResultMax) [[unlikely]] { if (count > kStackResultMax) [[unlikely]] {
@@ -4983,9 +4964,9 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
context.results[i].insertionPoint->forwardTo; context.results[i].insertionPoint->forwardTo;
} }
if (context.results[i].endInsertionPoint == nullptr) { if (context.results[i].endInsertionPoint == nullptr) {
addPointWrite(getInTree(context.results[i].insertionPoint, this), addPointWrite(getInTree(context.results[i].insertionPoint),
context.results[i].remaining, writeVersion, &writeContext, context.results[i].remaining, writeVersion,
this); &writeContext);
} else { } else {
if (firstRangeWrite == nullptr) { if (firstRangeWrite == nullptr) {
firstRangeWrite = context.results + i; firstRangeWrite = context.results + i;
@@ -5000,7 +4981,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
auto [beginNode, endNode] = addWriteRange( auto [beginNode, endNode] = addWriteRange(
context.results[i].insertionPoint, context.results[i].remaining, context.results[i].insertionPoint, context.results[i].remaining,
context.results[i].endInsertionPoint, context.results[i].endInsertionPoint,
context.results[i].endRemaining, writeVersion, &writeContext, this); context.results[i].endRemaining, writeVersion, &writeContext);
context.results[i].insertionPoint = beginNode; context.results[i].insertionPoint = beginNode;
context.results[i].endInsertionPoint = endNode; context.results[i].endInsertionPoint = endNode;
} }
@@ -5022,7 +5003,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
iter->endInsertionPoint = iter->endInsertionPoint->forwardTo; iter->endInsertionPoint = iter->endInsertionPoint->forwardTo;
} }
eraseInRange(iter->insertionPoint, iter->endInsertionPoint, eraseInRange(iter->insertionPoint, iter->endInsertionPoint,
&writeContext, this); &writeContext);
} }
} }
@@ -5053,11 +5034,11 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
auto begin = TrivialSpan(w.begin.p, w.begin.len); auto begin = TrivialSpan(w.begin.p, w.begin.len);
auto end = TrivialSpan(w.end.p, w.end.len); auto end = TrivialSpan(w.end.p, w.end.len);
if (w.end.len > 0) { if (w.end.len > 0) {
addWriteRange(root, begin, end, InternalVersionT(writeVersion), addWriteRange(rootParent->children[0], begin, end,
&writeContext, this); InternalVersionT(writeVersion), &writeContext);
} else { } else {
addPointWrite(root, begin, InternalVersionT(writeVersion), addPointWrite(rootParent->children[0], begin,
&writeContext, this); InternalVersionT(writeVersion), &writeContext);
} }
} }
} }
@@ -5078,7 +5059,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
if (oldestExtantVersion < writeVersion - kMaxCorrectVersionWindow) if (oldestExtantVersion < writeVersion - kMaxCorrectVersionWindow)
[[unlikely]] { [[unlikely]] {
if (writeVersion > newestVersionFullPrecision + kNominalVersionWindow) { if (writeVersion > newestVersionFullPrecision + kNominalVersionWindow) {
eraseTree(root, &writeContext); eraseTree(rootParent->children[0], &writeContext);
init(writeVersion - kNominalVersionWindow); init(writeVersion - kNominalVersionWindow);
} }
@@ -5106,6 +5087,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
} }
if (count > 0) { if (count > 0) {
rootParent->childMaxVersion[0] = InternalVersionT(writeVersion);
int firstNotInserted = 0; int firstNotInserted = 0;
bool batchHasOnlyPointWrites = writes[0].end.len == 0; bool batchHasOnlyPointWrites = writes[0].end.len == 0;
bool batchIsSorted = true; bool batchIsSorted = true;
@@ -5153,11 +5135,14 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
// Spends up to `fuel` gc'ing, and returns its unused fuel. Reclaims memory // Spends up to `fuel` gc'ing, and returns its unused fuel. Reclaims memory
// and updates oldestExtantVersion after spending enough fuel. // and updates oldestExtantVersion after spending enough fuel.
int64_t gcScanStep(int64_t fuel) { int64_t gcScanStep(int64_t fuel) {
Node *n = firstGeqPhysical(root, removalKey); Node *n = firstGeqPhysical(rootParent->children[0], removalKey);
// There's no way to erase removalKey without introducing a key after it // There's no way to erase removalKey without introducing a key after it
assert(n != nullptr); assert(n != nullptr);
// Don't erase the root // Don't erase the root
if (n == root) { if (n == rootParent->children[0]) {
for (auto &v : rootParent->childMaxVersion) {
v = std::max(v, oldestVersion);
}
rezero(n, oldestVersion); rezero(n, oldestVersion);
n = nextPhysical(n); n = nextPhysical(n);
} }
@@ -5176,7 +5161,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
// There's no way to insert a range such that range version of the // There's no way to insert a range such that range version of the
// right node is greater than the point version of the left node // right node is greater than the point version of the left node
assert(n->entry.rangeVersion <= oldestVersion); assert(n->entry.rangeVersion <= oldestVersion);
n = erase(n, &writeContext, this, /*logical*/ false); n = erase(n, &writeContext, /*logical*/ false);
} else { } else {
n = nextPhysical(n); n = nextPhysical(n);
} }
@@ -5227,7 +5212,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
cursor -= partialKey.size(); cursor -= partialKey.size();
memcpy(cursor, partialKey.data(), partialKey.size()); memcpy(cursor, partialKey.data(), partialKey.size());
if (n->parent == nullptr) { if (n->parent == rootParent) {
break; break;
} }
@@ -5297,17 +5282,28 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
keyUpdates = 10; keyUpdates = 10;
// Insert "" // Insert ""
root = writeContext.allocate<Node0>(0); rootParent->children[0] = writeContext.allocate<Node0>(0);
root->numChildren = 0; rootParent->children[0]->numChildren = 0;
root->parent = nullptr; rootParent->children[0]->parent = rootParent;
root->entryPresent = false; rootParent->children[0]->parentsIndex = 0;
root->partialKeyLen = 0; rootParent->children[0]->entryPresent = false;
rootParent->children[0]->partialKeyLen = 0;
rootParent->entryPresent = false;
rootParent->numChildren = 1;
rootParent->parent = nullptr;
rootParent->parentsIndex = 0;
for (auto &v : rootParent->childMaxVersion) {
v = this->oldestVersion;
}
rootParent->index[0] = 0;
rootParent->partialKeyLen = 0;
rootParent->releaseDeferred = false;
addKey(root); addKey(rootParent->children[0]);
root->entryPresent = true; rootParent->children[0]->entryPresent = true;
root->entry.pointVersion = this->oldestVersion; rootParent->children[0]->entry.pointVersion = this->oldestVersion;
root->entry.rangeVersion = this->oldestVersion; rootParent->children[0]->entry.rangeVersion = this->oldestVersion;
#if !USE_64_BIT #if !USE_64_BIT
InternalVersionT::zero = writeContext.zero = this->oldestVersion; InternalVersionT::zero = writeContext.zero = this->oldestVersion;
@@ -5318,13 +5314,15 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
explicit Impl(int64_t oldestVersion) { explicit Impl(int64_t oldestVersion) {
assert(oldestVersion >= 0); assert(oldestVersion >= 0);
rootParent = writeContext.allocate<Node3>(0);
init(oldestVersion); init(oldestVersion);
metrics = initMetrics(metricsList, metricsCount); metrics = initMetrics(metricsList, metricsCount);
} }
~Impl() { ~Impl() {
eraseTree(root, &writeContext); eraseTree(rootParent->children[0], &writeContext);
safe_free(metrics, metricsCount * sizeof(metrics[0])); safe_free(metrics, metricsCount * sizeof(metrics[0]));
safe_free(removalBuffer, removalBufferSize); safe_free(removalBuffer, removalBufferSize);
writeContext.release(rootParent);
} }
WriteContext writeContext; WriteContext writeContext;
@@ -5337,7 +5335,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
TrivialSpan removalKey; TrivialSpan removalKey;
int64_t keyUpdates; int64_t keyUpdates;
TaggedNodePointer root; Node3 *rootParent;
InternalVersionT oldestVersion; InternalVersionT oldestVersion;
int64_t oldestVersionFullPrecision; int64_t oldestVersionFullPrecision;
int64_t oldestExtantVersion; int64_t oldestExtantVersion;
@@ -5419,9 +5417,8 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
} }
}; };
TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *impl) { TaggedNodePointer &getInTree(Node *n) {
return n->parent == nullptr ? impl->root return getChildExists(n->parent, n->parentsIndex);
: getChildExists(n->parent, n->parentsIndex);
} }
// Internal entry points. Public entry points should just delegate to these // Internal entry points. Public entry points should just delegate to these
@@ -5634,11 +5631,13 @@ std::string getSearchPathPrintable(Node *n) {
for (int i = n->partialKeyLen - 1; i >= 0; --i) { for (int i = n->partialKeyLen - 1; i >= 0; --i) {
result.push_back(n->partialKey()[i]); result.push_back(n->partialKey()[i]);
} }
if (n->parent == nullptr) {
break;
}
result.push_back(n->parentsIndex); result.push_back(n->parentsIndex);
n = n->parent; n = n->parent;
if (n->parent == nullptr) {
// Implicit byte from rootParent
result.pop_back();
break;
}
} }
std::reverse(result.begin(), result.end()); std::reverse(result.begin(), result.end());
if (result.size() > 0) { if (result.size() > 0) {
@@ -5686,15 +5685,13 @@ std::string getSearchPath(Node *n) {
return std::string((const char *)result.data(), result.size()); return std::string((const char *)result.data(), result.size());
} }
[[maybe_unused]] void debugPrintDot(FILE *file, Node *node, [[maybe_unused]] void debugPrintDot(FILE *file, Node *node) {
ConflictSet::Impl *impl) {
constexpr int kSeparation = 3; constexpr int kSeparation = 3;
struct DebugDotPrinter { struct DebugDotPrinter {
explicit DebugDotPrinter(FILE *file, ConflictSet::Impl *impl) explicit DebugDotPrinter(FILE *file) : file(file) {}
: file(file), impl(impl) {}
void print(Node *n, int y = 0) { void print(Node *n, int y = 0) {
assert(n != nullptr); assert(n != nullptr);
@@ -5720,13 +5717,12 @@ std::string getSearchPath(Node *n) {
} }
int x = 0; int x = 0;
FILE *file; FILE *file;
ConflictSet::Impl *impl;
}; };
fprintf(file, "digraph ConflictSet {\n"); fprintf(file, "digraph ConflictSet {\n");
fprintf(file, " node [shape = box];\n"); fprintf(file, " node [shape = box];\n");
assert(node != nullptr); assert(node != nullptr);
DebugDotPrinter printer{file, impl}; DebugDotPrinter printer{file};
printer.print(node); printer.print(node);
fprintf(file, "}\n"); fprintf(file, "}\n");
} }
@@ -6114,7 +6110,7 @@ void printTree() {
write.begin = "art"_s; write.begin = "art"_s;
write.end = ""_s; write.end = ""_s;
cs.addWrites(&write, 1, ++writeVersion); cs.addWrites(&write, 1, ++writeVersion);
debugPrintDot(stdout, cs.root, &cs); debugPrintDot(stdout, cs.rootParent->children[0]);
} }
int main(void) { benchHorizontal16(); } int main(void) { benchHorizontal16(); }
@@ -6132,13 +6128,13 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
if (!done1) { if (!done1) {
done1 = driver1.next(); done1 = driver1.next();
if (!driver1.ok) { if (!driver1.ok) {
debugPrintDot(stdout, driver1.cs.root, &driver1.cs); debugPrintDot(stdout, driver1.cs.rootParent->children[0]);
fflush(stdout); fflush(stdout);
abort(); abort();
} }
if (!checkCorrectness(driver1.cs.root, driver1.cs.oldestVersion, if (!checkCorrectness(driver1.cs.rootParent->children[0],
&driver1.cs)) { driver1.cs.oldestVersion, &driver1.cs)) {
debugPrintDot(stdout, driver1.cs.root, &driver1.cs); debugPrintDot(stdout, driver1.cs.rootParent->children[0]);
fflush(stdout); fflush(stdout);
abort(); abort();
} }
@@ -6146,13 +6142,13 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
if (!done2) { if (!done2) {
done2 = driver2.next(); done2 = driver2.next();
if (!driver2.ok) { if (!driver2.ok) {
debugPrintDot(stdout, driver2.cs.root, &driver2.cs); debugPrintDot(stdout, driver2.cs.rootParent->children[0]);
fflush(stdout); fflush(stdout);
abort(); abort();
} }
if (!checkCorrectness(driver2.cs.root, driver2.cs.oldestVersion, if (!checkCorrectness(driver2.cs.rootParent->children[0],
&driver2.cs)) { driver2.cs.oldestVersion, &driver2.cs)) {
debugPrintDot(stdout, driver2.cs.root, &driver2.cs); debugPrintDot(stdout, driver2.cs.rootParent->children[0]);
fflush(stdout); fflush(stdout);
abort(); abort();
} }