weaselab/conflict-set
2
0
Fork 0
Code Issues 4 Packages Releases 13 Activity

Add artificial root parent and stop plumbing impl everywhere

Browse Source
This commit is contained in:
Andrew Noyes
2024-11-21 16:32:13 -08:00
parent 972f16ed8f
commit 0cea5565b5
1 changed files with 142 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

288
ConflictSet.cpp
View File

@@ -1114,8 +1114,9 @@ InternalVersionT exchangeMaxVersion(Node *n, InternalVersionT newMax) {
}
}
// Precondition `n` is not the root
void setMaxVersion(Node *n, InternalVersionT newMax) {
// This should not be the rootParent
assert(n->parent != nullptr);
assert(newMax >= InternalVersionT::zero);
int index = n->parentsIndex;
n = n->parent;
@@ -1158,8 +1159,7 @@ void setMaxVersion(Node *n, InternalVersionT newMax) {
}
}
// If impl is nullptr, then n->parent must not be nullptr
TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *impl);
TaggedNodePointer &getInTree(Node *n);
TaggedNodePointer getChild(Node0 *, uint8_t) { return nullptr; }
TaggedNodePointer getChild(Node3 *self, uint8_t index) {
@@ -1375,13 +1375,11 @@ bool checkRangeVersionOfFirstGeq(Node *node, InternalVersionT readVersion) {
}
// self must not be the root
void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext,
ConflictSet::Impl *impl);
void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext);
void consumePartialKeyFull(TaggedNodePointer &self, TrivialSpan &key,
InternalVersionT writeVersion,
WriteContext *writeContext,
ConflictSet::Impl *impl) {
WriteContext *writeContext) {
// Handle an existing partial key
int commonLen = std::min<int>(self->partialKeyLen, key.size());
int partialKeyIndex =
@@ -1419,7 +1417,7 @@ void consumePartialKeyFull(TaggedNodePointer &self, TrivialSpan &key,
partialKeyIndex + 1);
// Maintain memory capacity invariant
maybeDecreaseCapacity(old, writeContext, impl);
maybeDecreaseCapacity(old, writeContext);
}
key = key.subspan(partialKeyIndex, key.size() - partialKeyIndex);
}
@@ -1428,10 +1426,9 @@ void consumePartialKeyFull(TaggedNodePointer &self, TrivialSpan &key,
// `key` such that `self` is along the search path of `key`
inline __attribute__((always_inline)) void
consumePartialKey(TaggedNodePointer &self, TrivialSpan &key,
InternalVersionT writeVersion, WriteContext *writeContext,
ConflictSet::Impl *impl) {
InternalVersionT writeVersion, WriteContext *writeContext) {
if (self->partialKeyLen > 0) {
consumePartialKeyFull(self, key, writeVersion, writeContext, impl);
consumePartialKeyFull(self, key, writeVersion, writeContext);
}
}
@@ -1441,8 +1438,7 @@ consumePartialKey(TaggedNodePointer &self, TrivialSpan &key,
// `maxVersion` for result.
TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key,
InternalVersionT newMaxVersion,
WriteContext *writeContext,
ConflictSet::Impl *impl) {
WriteContext *writeContext) {
int index = key.front();
key = key.subspan(1, key.size() - 1);
@@ -1455,8 +1451,7 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key,
auto *self3 = static_cast<Node3 *>(self);
int i = getNodeIndex(self3, index);
if (i >= 0) {
consumePartialKey(self3->children[i], key, newMaxVersion, writeContext,
impl);
consumePartialKey(self3->children[i], key, newMaxVersion, writeContext);
self3->childMaxVersion[i] = newMaxVersion;
return self3->children[i];
}
@@ -1465,8 +1460,7 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key,
auto *self16 = static_cast<Node16 *>(self);
int i = getNodeIndex(self16, index);
if (i >= 0) {
consumePartialKey(self16->children[i], key, newMaxVersion, writeContext,
impl);
consumePartialKey(self16->children[i], key, newMaxVersion, writeContext);
self16->childMaxVersion[i] = newMaxVersion;
return self16->children[i];
}
@@ -1476,7 +1470,7 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key,
int secondIndex = self48->index[index];
if (secondIndex >= 0) {
consumePartialKey(self48->children[secondIndex], key, newMaxVersion,
writeContext, impl);
writeContext);
self48->childMaxVersion[secondIndex] = newMaxVersion;
self48->maxOfMax[secondIndex >> Node48::kMaxOfMaxShift] =
std::max(self48->maxOfMax[secondIndex >> Node48::kMaxOfMaxShift],
@@ -1487,7 +1481,7 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key,
case Type_Node256: {
auto *self256 = static_cast<Node256 *>(self);
if (auto &result = self256->children[index]; result != nullptr) {
consumePartialKey(result, key, newMaxVersion, writeContext, impl);
consumePartialKey(result, key, newMaxVersion, writeContext);
self256->childMaxVersion[index] = newMaxVersion;
self256->maxOfMax[index >> Node256::kMaxOfMaxShift] = std::max(
self256->maxOfMax[index >> Node256::kMaxOfMaxShift], newMaxVersion);
@@ -1666,15 +1660,15 @@ downLeftSpine:
return node;
}
void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext,
ConflictSet::Impl *impl) {
void freeAndMakeCapacity(Node *&self, int capacity,
WriteContext *writeContext) {
++writeContext->accum.nodes_resized;
switch (self->getType()) {
case Type_Node0: {
auto *self0 = (Node0 *)self;
auto *newSelf = writeContext->allocate<Node0>(capacity);
newSelf->copyChildrenAndKeyFrom(*self0);
getInTree(self, impl) = newSelf;
getInTree(self) = newSelf;
writeContext->deferRelease(self0, newSelf);
self = newSelf;
} break;
@@ -1682,7 +1676,7 @@ void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext,
auto *self3 = (Node3 *)self;
auto *newSelf = writeContext->allocate<Node3>(capacity);
newSelf->copyChildrenAndKeyFrom(*self3);
getInTree(self, impl) = newSelf;
getInTree(self) = newSelf;
writeContext->deferRelease(self3, newSelf);
self = newSelf;
} break;
@@ -1690,7 +1684,7 @@ void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext,
auto *self16 = (Node16 *)self;
auto *newSelf = writeContext->allocate<Node16>(capacity);
newSelf->copyChildrenAndKeyFrom(*self16);
getInTree(self, impl) = newSelf;
getInTree(self) = newSelf;
writeContext->deferRelease(self16, newSelf);
self = newSelf;
} break;
@@ -1698,7 +1692,7 @@ void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext,
auto *self48 = (Node48 *)self;
auto *newSelf = writeContext->allocate<Node48>(capacity);
newSelf->copyChildrenAndKeyFrom(*self48);
getInTree(self, impl) = newSelf;
getInTree(self) = newSelf;
writeContext->deferRelease(self48, newSelf);
self = newSelf;
} break;
@@ -1706,7 +1700,7 @@ void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext,
auto *self256 = (Node256 *)self;
auto *newSelf = writeContext->allocate<Node256>(capacity);
newSelf->copyChildrenAndKeyFrom(*self256);
getInTree(self, impl) = newSelf;
getInTree(self) = newSelf;
writeContext->deferRelease(self256, newSelf);
self = newSelf;
} break;
@@ -1716,8 +1710,7 @@ void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext,
}
// Fix larger-than-desired capacities. self must not be the root
void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext,
ConflictSet::Impl *impl) {
void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext) {
const int maxCapacity =
(self->numChildren + int(self->entryPresent)) * (self->partialKeyLen + 1);
@@ -1729,7 +1722,7 @@ void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext,
if (self->getCapacity() <= maxCapacity) {
return;
}
freeAndMakeCapacity(self, self->partialKeyLen, writeContext, impl);
freeAndMakeCapacity(self, self->partialKeyLen, writeContext);
}
#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
@@ -1799,13 +1792,13 @@ void rezero(Node *n, InternalVersionT z) {
#endif
void mergeWithChild(TaggedNodePointer &self, WriteContext *writeContext,
Node3 *self3, ConflictSet::Impl *impl) {
Node3 *self3) {
assert(!self3->entryPresent);
Node *child = self3->children[0];
const int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen;
if (minCapacity > child->getCapacity()) {
freeAndMakeCapacity(child, minCapacity, writeContext, impl);
freeAndMakeCapacity(child, minCapacity, writeContext);
}
// Merge partial key with child
@@ -1826,9 +1819,6 @@ void mergeWithChild(TaggedNodePointer &self, WriteContext *writeContext,
child->parent = self->parent;
child->parentsIndex = self->parentsIndex;
// Max versions are stored in the parent, so we need to update it now
// that we have a new parent. Safe to call since the root never has a partial
// key.
setMaxVersion(child, std::max(childMaxVersion, writeContext->zero));
self = child;
@@ -1841,63 +1831,59 @@ bool needsDownsize(Node *n) {
return n->numChildren + n->entryPresent < minTable[n->getType()];
}
void downsize(Node3 *self, WriteContext *writeContext,
ConflictSet::Impl *impl) {
void downsize(Node3 *self, WriteContext *writeContext) {
if (self->numChildren == 0) {
auto *newSelf = writeContext->allocate<Node0>(self->partialKeyLen);
newSelf->copyChildrenAndKeyFrom(*self);
getInTree(self, impl) = newSelf;
getInTree(self) = newSelf;
writeContext->deferRelease(self, newSelf);
} else {
assert(self->numChildren == 1 && !self->entryPresent);
mergeWithChild(getInTree(self, impl), writeContext, self, impl);
mergeWithChild(getInTree(self), writeContext, self);
}
}
void downsize(Node16 *self, WriteContext *writeContext,
ConflictSet::Impl *impl) {
void downsize(Node16 *self, WriteContext *writeContext) {
assert(self->numChildren + int(self->entryPresent) < kMinChildrenNode16);
auto *newSelf = writeContext->allocate<Node3>(self->partialKeyLen);
newSelf->copyChildrenAndKeyFrom(*self);
getInTree(self, impl) = newSelf;
getInTree(self) = newSelf;
writeContext->deferRelease(self, newSelf);
}
void downsize(Node48 *self, WriteContext *writeContext,
ConflictSet::Impl *impl) {
void downsize(Node48 *self, WriteContext *writeContext) {
assert(self->numChildren + int(self->entryPresent) < kMinChildrenNode48);
auto *newSelf = writeContext->allocate<Node16>(self->partialKeyLen);
newSelf->copyChildrenAndKeyFrom(*self);
getInTree(self, impl) = newSelf;
getInTree(self) = newSelf;
writeContext->deferRelease(self, newSelf);
}
void downsize(Node256 *self, WriteContext *writeContext,
ConflictSet::Impl *impl) {
void downsize(Node256 *self, WriteContext *writeContext) {
assert(self->numChildren + int(self->entryPresent) < kMinChildrenNode256);
auto *self256 = (Node256 *)self;
auto *newSelf = writeContext->allocate<Node48>(self->partialKeyLen);
newSelf->copyChildrenAndKeyFrom(*self256);
getInTree(self, impl) = newSelf;
getInTree(self) = newSelf;
writeContext->deferRelease(self256, newSelf);
}
void downsize(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl) {
void downsize(Node *self, WriteContext *writeContext) {
switch (self->getType()) {
case Type_Node0: // GCOVR_EXCL_LINE
__builtin_unreachable(); // GCOVR_EXCL_LINE
case Type_Node3:
downsize(static_cast<Node3 *>(self), writeContext, impl);
downsize(static_cast<Node3 *>(self), writeContext);
break;
case Type_Node16:
downsize(static_cast<Node16 *>(self), writeContext, impl);
downsize(static_cast<Node16 *>(self), writeContext);
break;
case Type_Node48:
downsize(static_cast<Node48 *>(self), writeContext, impl);
downsize(static_cast<Node48 *>(self), writeContext);
break;
case Type_Node256:
downsize(static_cast<Node256 *>(self), writeContext, impl);
downsize(static_cast<Node256 *>(self), writeContext);
break;
default: // GCOVR_EXCL_LINE
__builtin_unreachable(); // GCOVR_EXCL_LINE
@@ -1908,8 +1894,7 @@ void downsize(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl) {
// path to self. May invalidate children of self->parent. Returns a pointer to
// the node after self. Precondition: `self->entryPresent`
Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl,
bool logical) {
Node *erase(Node *self, WriteContext *writeContext, bool logical) {
++writeContext->accum.entries_erased;
assert(self->parent != nullptr);
@@ -1928,12 +1913,12 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl,
if (self->numChildren != 0) {
if (needsDownsize(self)) {
downsize(self, writeContext, impl);
downsize(self, writeContext);
}
while (self->releaseDeferred) {
self = self->forwardTo;
}
maybeDecreaseCapacity(self, writeContext, impl);
maybeDecreaseCapacity(self, writeContext);
if (result != nullptr) {
while (result->releaseDeferred) {
result = result->forwardTo;
@@ -1964,7 +1949,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl,
sizeof(parent3->children[0]));
if (needsDownsize(parent3)) {
downsize(parent3, writeContext, impl);
downsize(parent3, writeContext);
}
} break;
case Type_Node16: {
@@ -1983,7 +1968,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl,
sizeof(parent16->children[0]));
if (needsDownsize(parent16)) {
downsize(parent16, writeContext, impl);
downsize(parent16, writeContext);
}
} break;
case Type_Node48: {
@@ -2013,7 +1998,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl,
sizeof(parent48->children[0]));
if (needsDownsize(parent48)) {
downsize(parent48, writeContext, impl);
downsize(parent48, writeContext);
}
} break;
case Type_Node256: {
@@ -2024,7 +2009,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl,
--parent->numChildren;
if (needsDownsize(parent256)) {
downsize(parent256, writeContext, impl);
downsize(parent256, writeContext);
}
} break;
default: // GCOVR_EXCL_LINE
@@ -2034,7 +2019,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl,
while (parent->releaseDeferred) {
parent = parent->forwardTo;
}
maybeDecreaseCapacity(parent, writeContext, impl);
maybeDecreaseCapacity(parent, writeContext);
if (result != nullptr) {
while (result->releaseDeferred) {
@@ -2631,11 +2616,13 @@ TrivialSpan getSearchPath(Arena &arena, Node *n) {
for (int i = n->partialKeyLen - 1; i >= 0; --i) {
result.push_back(n->partialKey()[i]);
}
if (n->parent == nullptr) {
break;
}
result.push_back(n->parentsIndex);
n = n->parent;
if (n->parent == nullptr) {
// Implicit byte from rootParent
result.pop_back();
break;
}
}
std::reverse(result.begin(), result.end());
return {result.begin(), result.size()};
@@ -2735,12 +2722,13 @@ checkMaxBetweenExclusiveImpl<true>(Node256 *n, int begin, int end,
// of the result will have `maxVersion` set to `writeVersion` as a
// postcondition. Nodes along the search path may be invalidated. Callers must
// ensure that the max version of the self argument is updated.
[[nodiscard]] TaggedNodePointer *
insert(TaggedNodePointer *self, TrivialSpan key, InternalVersionT writeVersion,
WriteContext *writeContext, ConflictSet::Impl *impl) {
[[nodiscard]] TaggedNodePointer *insert(TaggedNodePointer *self,
TrivialSpan key,
InternalVersionT writeVersion,
WriteContext *writeContext) {
for (; key.size() != 0; ++writeContext->accum.insert_iterations) {
self = &getOrCreateChild(*self, key, writeVersion, writeContext, impl);
self = &getOrCreateChild(*self, key, writeVersion, writeContext);
}
return self;
}
@@ -2798,10 +2786,9 @@ void eraseTree(Node *root, WriteContext *writeContext) {
}
void addPointWrite(TaggedNodePointer &root, TrivialSpan key,
InternalVersionT writeVersion, WriteContext *writeContext,
ConflictSet::Impl *impl) {
InternalVersionT writeVersion, WriteContext *writeContext) {
++writeContext->accum.point_writes;
auto n = *insert(&root, key, writeVersion, writeContext, impl);
auto n = *insert(&root, key, writeVersion, writeContext);
if (!n->entryPresent) {
++writeContext->accum.entries_inserted;
auto *p = nextLogical(n);
@@ -2930,13 +2917,12 @@ struct AddedWriteRange {
AddedWriteRange addWriteRange(Node *beginRoot, TrivialSpan begin, Node *endRoot,
TrivialSpan end, InternalVersionT writeVersion,
WriteContext *writeContext,
ConflictSet::Impl *impl) {
WriteContext *writeContext) {
++writeContext->accum.range_writes;
Node *beginNode = *insert(&getInTree(beginRoot, impl), begin, writeVersion,
writeContext, impl);
Node *beginNode =
*insert(&getInTree(beginRoot), begin, writeVersion, writeContext);
addKey(beginNode);
if (!beginNode->entryPresent) {
++writeContext->accum.entries_inserted;
@@ -2951,8 +2937,7 @@ AddedWriteRange addWriteRange(Node *beginRoot, TrivialSpan begin, Node *endRoot,
}
beginNode->entry.pointVersion = writeVersion;
Node *endNode =
*insert(&getInTree(endRoot, impl), end, writeVersion, writeContext, impl);
Node *endNode = *insert(&getInTree(endRoot), end, writeVersion, writeContext);
addKey(endNode);
if (!endNode->entryPresent) {
@@ -2971,8 +2956,7 @@ AddedWriteRange addWriteRange(Node *beginRoot, TrivialSpan begin, Node *endRoot,
return {beginNode, endNode};
}
void eraseInRange(Node *beginNode, Node *endNode, WriteContext *writeContext,
ConflictSet::Impl *impl) {
void eraseInRange(Node *beginNode, Node *endNode, WriteContext *writeContext) {
auto checkNodesEq = [](Node *lhs, Node *&rhs) {
assert(!lhs->releaseDeferred);
while (rhs->releaseDeferred) [[unlikely]] {
@@ -2984,7 +2968,7 @@ void eraseInRange(Node *beginNode, Node *endNode, WriteContext *writeContext,
// Erase nodes in range
Node *iter = nextLogical(beginNode);
for (; !checkNodesEq(iter, endNode);
iter = erase(iter, writeContext, impl, /*logical*/ true)) {
iter = erase(iter, writeContext, /*logical*/ true)) {
}
// Inserting end trashed the last node's maxVersion. Fix that. Safe to call
@@ -2993,22 +2977,21 @@ void eraseInRange(Node *beginNode, Node *endNode, WriteContext *writeContext,
}
void addWriteRange(TaggedNodePointer &root, TrivialSpan begin, TrivialSpan end,
InternalVersionT writeVersion, WriteContext *writeContext,
ConflictSet::Impl *impl) {
InternalVersionT writeVersion, WriteContext *writeContext) {
int lcp = longestCommonPrefix(begin.data(), end.data(),
std::min(begin.size(), end.size()));
if (lcp == begin.size() && end.size() == begin.size() + 1 &&
end.back() == 0) {
return addPointWrite(root, begin, writeVersion, writeContext, impl);
return addPointWrite(root, begin, writeVersion, writeContext);
}
auto useAsRoot =
insert(&root, begin.subspan(0, lcp), writeVersion, writeContext, impl);
insert(&root, begin.subspan(0, lcp), writeVersion, writeContext);
auto [beginNode, endNode] = addWriteRange(
*useAsRoot, begin.subspan(lcp, begin.size() - lcp), *useAsRoot,
end.subspan(lcp, end.size() - lcp), writeVersion, writeContext, impl);
end.subspan(lcp, end.size() - lcp), writeVersion, writeContext);
eraseInRange(beginNode, endNode, writeContext, impl);
eraseInRange(beginNode, endNode, writeContext);
}
Node *firstGeqPhysical(Node *n, const TrivialSpan key) {
@@ -4780,10 +4763,6 @@ bool checkRangeRead(Node *n, TrivialSpan begin, TrivialSpan end,
return checkRangeRightSide(n, end, lcp, readVersion, readContext);
}
// This makes it safe to check maxVersion within checkRangeLeftSide. If this
// were false, then we would have returned above since lcp == begin.size().
assert(!(n->parent == nullptr && begin.size() == 0));
return checkRangeStartsWith(n, begin.subspan(0, lcp), begin[lcp], end[lcp],
readVersion, readContext) &&
checkRangeLeftSide(n, begin, lcp + 1, readVersion, readContext) &&
@@ -4803,14 +4782,16 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
} else {
bool ok;
if (reads[i].end.len == 0) {
ok = checkPointRead(
root, TrivialSpan(reads[i].begin.p, reads[i].begin.len),
InternalVersionT(reads[i].readVersion), &context.readContext);
ok = checkPointRead(rootParent->children[0],
TrivialSpan(reads[i].begin.p, reads[i].begin.len),
InternalVersionT(reads[i].readVersion),
&context.readContext);
} else {
ok = checkRangeRead(
root, TrivialSpan(reads[i].begin.p, reads[i].begin.len),
TrivialSpan(reads[i].end.p, reads[i].end.len),
InternalVersionT(reads[i].readVersion), &context.readContext);
ok = checkRangeRead(rootParent->children[0],
TrivialSpan(reads[i].begin.p, reads[i].begin.len),
TrivialSpan(reads[i].end.p, reads[i].end.len),
InternalVersionT(reads[i].readVersion),
&context.readContext);
}
result[i] = ok ? Commit : Conflict;
}
@@ -4837,13 +4818,13 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
check::Job inProgress[kConcurrent];
context.count = count;
context.oldestVersionFullPrecision = oldestVersionFullPrecision;
context.root = root;
context.root = rootParent->children[0];
context.queries = reads;
context.results = result;
int64_t started = std::min(kConcurrent, count);
context.started = started;
for (int i = 0; i < started; i++) {
inProgress[i].init(reads + i, result + i, root,
inProgress[i].init(reads + i, result + i, rootParent->children[0],
oldestVersionFullPrecision);
}
for (int i = 0; i < started - 1; i++) {
@@ -4921,7 +4902,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
interleaved_insert::Context context;
context.writeVersion = writeVersion;
context.count = count;
context.root = root;
context.root = rootParent->children[0];
context.writes = writes;
context.results = stackResults;
if (count > kStackResultMax) [[unlikely]] {
@@ -4983,9 +4964,9 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
context.results[i].insertionPoint->forwardTo;
}
if (context.results[i].endInsertionPoint == nullptr) {
addPointWrite(getInTree(context.results[i].insertionPoint, this),
context.results[i].remaining, writeVersion, &writeContext,
this);
addPointWrite(getInTree(context.results[i].insertionPoint),
context.results[i].remaining, writeVersion,
&writeContext);
} else {
if (firstRangeWrite == nullptr) {
firstRangeWrite = context.results + i;
@@ -5000,7 +4981,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
auto [beginNode, endNode] = addWriteRange(
context.results[i].insertionPoint, context.results[i].remaining,
context.results[i].endInsertionPoint,
context.results[i].endRemaining, writeVersion, &writeContext, this);
context.results[i].endRemaining, writeVersion, &writeContext);
context.results[i].insertionPoint = beginNode;
context.results[i].endInsertionPoint = endNode;
}
@@ -5022,7 +5003,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
iter->endInsertionPoint = iter->endInsertionPoint->forwardTo;
}
eraseInRange(iter->insertionPoint, iter->endInsertionPoint,
&writeContext, this);
&writeContext);
}
}
@@ -5053,11 +5034,11 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
auto begin = TrivialSpan(w.begin.p, w.begin.len);
auto end = TrivialSpan(w.end.p, w.end.len);
if (w.end.len > 0) {
addWriteRange(root, begin, end, InternalVersionT(writeVersion),
&writeContext, this);
addWriteRange(rootParent->children[0], begin, end,
InternalVersionT(writeVersion), &writeContext);
} else {
addPointWrite(root, begin, InternalVersionT(writeVersion),
&writeContext, this);
addPointWrite(rootParent->children[0], begin,
InternalVersionT(writeVersion), &writeContext);
}
}
}
@@ -5078,7 +5059,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
if (oldestExtantVersion < writeVersion - kMaxCorrectVersionWindow)
[[unlikely]] {
if (writeVersion > newestVersionFullPrecision + kNominalVersionWindow) {
eraseTree(root, &writeContext);
eraseTree(rootParent->children[0], &writeContext);
init(writeVersion - kNominalVersionWindow);
}
@@ -5106,6 +5087,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
}
if (count > 0) {
rootParent->childMaxVersion[0] = InternalVersionT(writeVersion);
int firstNotInserted = 0;
bool batchHasOnlyPointWrites = writes[0].end.len == 0;
bool batchIsSorted = true;
@@ -5153,11 +5135,14 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
// Spends up to `fuel` gc'ing, and returns its unused fuel. Reclaims memory
// and updates oldestExtantVersion after spending enough fuel.
int64_t gcScanStep(int64_t fuel) {
Node *n = firstGeqPhysical(root, removalKey);
Node *n = firstGeqPhysical(rootParent->children[0], removalKey);
// There's no way to erase removalKey without introducing a key after it
assert(n != nullptr);
// Don't erase the root
if (n == root) {
if (n == rootParent->children[0]) {
for (auto &v : rootParent->childMaxVersion) {
v = std::max(v, oldestVersion);
}
rezero(n, oldestVersion);
n = nextPhysical(n);
}
@@ -5176,7 +5161,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
// There's no way to insert a range such that range version of the
// right node is greater than the point version of the left node
assert(n->entry.rangeVersion <= oldestVersion);
n = erase(n, &writeContext, this, /*logical*/ false);
n = erase(n, &writeContext, /*logical*/ false);
} else {
n = nextPhysical(n);
}
@@ -5227,7 +5212,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
cursor -= partialKey.size();
memcpy(cursor, partialKey.data(), partialKey.size());
if (n->parent == nullptr) {
if (n->parent == rootParent) {
break;
}
@@ -5297,17 +5282,28 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
keyUpdates = 10;
// Insert ""
root = writeContext.allocate<Node0>(0);
root->numChildren = 0;
root->parent = nullptr;
root->entryPresent = false;
root->partialKeyLen = 0;
rootParent->children[0] = writeContext.allocate<Node0>(0);
rootParent->children[0]->numChildren = 0;
rootParent->children[0]->parent = rootParent;
rootParent->children[0]->parentsIndex = 0;
rootParent->children[0]->entryPresent = false;
rootParent->children[0]->partialKeyLen = 0;
rootParent->entryPresent = false;
rootParent->numChildren = 1;
rootParent->parent = nullptr;
rootParent->parentsIndex = 0;
for (auto &v : rootParent->childMaxVersion) {
v = this->oldestVersion;
}
rootParent->index[0] = 0;
rootParent->partialKeyLen = 0;
rootParent->releaseDeferred = false;
addKey(root);
addKey(rootParent->children[0]);
root->entryPresent = true;
root->entry.pointVersion = this->oldestVersion;
root->entry.rangeVersion = this->oldestVersion;
rootParent->children[0]->entryPresent = true;
rootParent->children[0]->entry.pointVersion = this->oldestVersion;
rootParent->children[0]->entry.rangeVersion = this->oldestVersion;
#if !USE_64_BIT
InternalVersionT::zero = writeContext.zero = this->oldestVersion;
@@ -5318,13 +5314,15 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
explicit Impl(int64_t oldestVersion) {
assert(oldestVersion >= 0);
rootParent = writeContext.allocate<Node3>(0);
init(oldestVersion);
metrics = initMetrics(metricsList, metricsCount);
}
~Impl() {
eraseTree(root, &writeContext);
eraseTree(rootParent->children[0], &writeContext);
safe_free(metrics, metricsCount * sizeof(metrics[0]));
safe_free(removalBuffer, removalBufferSize);
writeContext.release(rootParent);
}
WriteContext writeContext;
@@ -5337,7 +5335,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
TrivialSpan removalKey;
int64_t keyUpdates;
TaggedNodePointer root;
Node3 *rootParent;
InternalVersionT oldestVersion;
int64_t oldestVersionFullPrecision;
int64_t oldestExtantVersion;
@@ -5419,9 +5417,8 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
}
};
TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *impl) {
return n->parent == nullptr ? impl->root
: getChildExists(n->parent, n->parentsIndex);
TaggedNodePointer &getInTree(Node *n) {
return getChildExists(n->parent, n->parentsIndex);
}
// Internal entry points. Public entry points should just delegate to these
@@ -5634,11 +5631,13 @@ std::string getSearchPathPrintable(Node *n) {
for (int i = n->partialKeyLen - 1; i >= 0; --i) {
result.push_back(n->partialKey()[i]);
}
if (n->parent == nullptr) {
break;
}
result.push_back(n->parentsIndex);
n = n->parent;
if (n->parent == nullptr) {
// Implicit byte from rootParent
result.pop_back();
break;
}
}
std::reverse(result.begin(), result.end());
if (result.size() > 0) {
@@ -5686,15 +5685,13 @@ std::string getSearchPath(Node *n) {
return std::string((const char *)result.data(), result.size());
}
[[maybe_unused]] void debugPrintDot(FILE *file, Node *node,
ConflictSet::Impl *impl) {
[[maybe_unused]] void debugPrintDot(FILE *file, Node *node) {
constexpr int kSeparation = 3;
struct DebugDotPrinter {
explicit DebugDotPrinter(FILE *file, ConflictSet::Impl *impl)
: file(file), impl(impl) {}
explicit DebugDotPrinter(FILE *file) : file(file) {}
void print(Node *n, int y = 0) {
assert(n != nullptr);
@@ -5720,13 +5717,12 @@ std::string getSearchPath(Node *n) {
}
int x = 0;
FILE *file;
ConflictSet::Impl *impl;
};
fprintf(file, "digraph ConflictSet {\n");
fprintf(file, " node [shape = box];\n");
assert(node != nullptr);
DebugDotPrinter printer{file, impl};
DebugDotPrinter printer{file};
printer.print(node);
fprintf(file, "}\n");
}
@@ -6114,7 +6110,7 @@ void printTree() {
write.begin = "art"_s;
write.end = ""_s;
cs.addWrites(&write, 1, ++writeVersion);
debugPrintDot(stdout, cs.root, &cs);
debugPrintDot(stdout, cs.rootParent->children[0]);
}
int main(void) { benchHorizontal16(); }
@@ -6132,13 +6128,13 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
if (!done1) {
done1 = driver1.next();
if (!driver1.ok) {
debugPrintDot(stdout, driver1.cs.root, &driver1.cs);
debugPrintDot(stdout, driver1.cs.rootParent->children[0]);
fflush(stdout);
abort();
}
if (!checkCorrectness(driver1.cs.root, driver1.cs.oldestVersion,
&driver1.cs)) {
debugPrintDot(stdout, driver1.cs.root, &driver1.cs);
if (!checkCorrectness(driver1.cs.rootParent->children[0],
driver1.cs.oldestVersion, &driver1.cs)) {
debugPrintDot(stdout, driver1.cs.rootParent->children[0]);
fflush(stdout);
abort();
}
@@ -6146,13 +6142,13 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
if (!done2) {
done2 = driver2.next();
if (!driver2.ok) {
debugPrintDot(stdout, driver2.cs.root, &driver2.cs);
debugPrintDot(stdout, driver2.cs.rootParent->children[0]);
fflush(stdout);
abort();
}
if (!checkCorrectness(driver2.cs.root, driver2.cs.oldestVersion,
&driver2.cs)) {
debugPrintDot(stdout, driver2.cs.root, &driver2.cs);
if (!checkCorrectness(driver2.cs.rootParent->children[0],
driver2.cs.oldestVersion, &driver2.cs)) {
debugPrintDot(stdout, driver2.cs.rootParent->children[0]);
fflush(stdout);
abort();
}