diff --git a/ConflictSet.cpp b/ConflictSet.cpp index 841787e..98f5208 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -1114,8 +1114,9 @@ InternalVersionT exchangeMaxVersion(Node *n, InternalVersionT newMax) { } } -// Precondition `n` is not the root void setMaxVersion(Node *n, InternalVersionT newMax) { + // This should not be the rootParent + assert(n->parent != nullptr); assert(newMax >= InternalVersionT::zero); int index = n->parentsIndex; n = n->parent; @@ -1158,8 +1159,7 @@ void setMaxVersion(Node *n, InternalVersionT newMax) { } } -// If impl is nullptr, then n->parent must not be nullptr -TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *impl); +TaggedNodePointer &getInTree(Node *n); TaggedNodePointer getChild(Node0 *, uint8_t) { return nullptr; } TaggedNodePointer getChild(Node3 *self, uint8_t index) { @@ -1375,13 +1375,11 @@ bool checkRangeVersionOfFirstGeq(Node *node, InternalVersionT readVersion) { } // self must not be the root -void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext, - ConflictSet::Impl *impl); +void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext); void consumePartialKeyFull(TaggedNodePointer &self, TrivialSpan &key, InternalVersionT writeVersion, - WriteContext *writeContext, - ConflictSet::Impl *impl) { + WriteContext *writeContext) { // Handle an existing partial key int commonLen = std::min(self->partialKeyLen, key.size()); int partialKeyIndex = @@ -1419,7 +1417,7 @@ void consumePartialKeyFull(TaggedNodePointer &self, TrivialSpan &key, partialKeyIndex + 1); // Maintain memory capacity invariant - maybeDecreaseCapacity(old, writeContext, impl); + maybeDecreaseCapacity(old, writeContext); } key = key.subspan(partialKeyIndex, key.size() - partialKeyIndex); } @@ -1428,10 +1426,9 @@ void consumePartialKeyFull(TaggedNodePointer &self, TrivialSpan &key, // `key` such that `self` is along the search path of `key` inline __attribute__((always_inline)) void consumePartialKey(TaggedNodePointer &self, TrivialSpan &key, - InternalVersionT writeVersion, WriteContext *writeContext, - ConflictSet::Impl *impl) { + InternalVersionT writeVersion, WriteContext *writeContext) { if (self->partialKeyLen > 0) { - consumePartialKeyFull(self, key, writeVersion, writeContext, impl); + consumePartialKeyFull(self, key, writeVersion, writeContext); } } @@ -1441,8 +1438,7 @@ consumePartialKey(TaggedNodePointer &self, TrivialSpan &key, // `maxVersion` for result. TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key, InternalVersionT newMaxVersion, - WriteContext *writeContext, - ConflictSet::Impl *impl) { + WriteContext *writeContext) { int index = key.front(); key = key.subspan(1, key.size() - 1); @@ -1455,8 +1451,7 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key, auto *self3 = static_cast(self); int i = getNodeIndex(self3, index); if (i >= 0) { - consumePartialKey(self3->children[i], key, newMaxVersion, writeContext, - impl); + consumePartialKey(self3->children[i], key, newMaxVersion, writeContext); self3->childMaxVersion[i] = newMaxVersion; return self3->children[i]; } @@ -1465,8 +1460,7 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key, auto *self16 = static_cast(self); int i = getNodeIndex(self16, index); if (i >= 0) { - consumePartialKey(self16->children[i], key, newMaxVersion, writeContext, - impl); + consumePartialKey(self16->children[i], key, newMaxVersion, writeContext); self16->childMaxVersion[i] = newMaxVersion; return self16->children[i]; } @@ -1476,7 +1470,7 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key, int secondIndex = self48->index[index]; if (secondIndex >= 0) { consumePartialKey(self48->children[secondIndex], key, newMaxVersion, - writeContext, impl); + writeContext); self48->childMaxVersion[secondIndex] = newMaxVersion; self48->maxOfMax[secondIndex >> Node48::kMaxOfMaxShift] = std::max(self48->maxOfMax[secondIndex >> Node48::kMaxOfMaxShift], @@ -1487,7 +1481,7 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key, case Type_Node256: { auto *self256 = static_cast(self); if (auto &result = self256->children[index]; result != nullptr) { - consumePartialKey(result, key, newMaxVersion, writeContext, impl); + consumePartialKey(result, key, newMaxVersion, writeContext); self256->childMaxVersion[index] = newMaxVersion; self256->maxOfMax[index >> Node256::kMaxOfMaxShift] = std::max( self256->maxOfMax[index >> Node256::kMaxOfMaxShift], newMaxVersion); @@ -1666,15 +1660,15 @@ downLeftSpine: return node; } -void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext, - ConflictSet::Impl *impl) { +void freeAndMakeCapacity(Node *&self, int capacity, + WriteContext *writeContext) { ++writeContext->accum.nodes_resized; switch (self->getType()) { case Type_Node0: { auto *self0 = (Node0 *)self; auto *newSelf = writeContext->allocate(capacity); newSelf->copyChildrenAndKeyFrom(*self0); - getInTree(self, impl) = newSelf; + getInTree(self) = newSelf; writeContext->deferRelease(self0, newSelf); self = newSelf; } break; @@ -1682,7 +1676,7 @@ void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext, auto *self3 = (Node3 *)self; auto *newSelf = writeContext->allocate(capacity); newSelf->copyChildrenAndKeyFrom(*self3); - getInTree(self, impl) = newSelf; + getInTree(self) = newSelf; writeContext->deferRelease(self3, newSelf); self = newSelf; } break; @@ -1690,7 +1684,7 @@ void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext, auto *self16 = (Node16 *)self; auto *newSelf = writeContext->allocate(capacity); newSelf->copyChildrenAndKeyFrom(*self16); - getInTree(self, impl) = newSelf; + getInTree(self) = newSelf; writeContext->deferRelease(self16, newSelf); self = newSelf; } break; @@ -1698,7 +1692,7 @@ void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext, auto *self48 = (Node48 *)self; auto *newSelf = writeContext->allocate(capacity); newSelf->copyChildrenAndKeyFrom(*self48); - getInTree(self, impl) = newSelf; + getInTree(self) = newSelf; writeContext->deferRelease(self48, newSelf); self = newSelf; } break; @@ -1706,7 +1700,7 @@ void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext, auto *self256 = (Node256 *)self; auto *newSelf = writeContext->allocate(capacity); newSelf->copyChildrenAndKeyFrom(*self256); - getInTree(self, impl) = newSelf; + getInTree(self) = newSelf; writeContext->deferRelease(self256, newSelf); self = newSelf; } break; @@ -1716,8 +1710,7 @@ void freeAndMakeCapacity(Node *&self, int capacity, WriteContext *writeContext, } // Fix larger-than-desired capacities. self must not be the root -void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext, - ConflictSet::Impl *impl) { +void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext) { const int maxCapacity = (self->numChildren + int(self->entryPresent)) * (self->partialKeyLen + 1); @@ -1729,7 +1722,7 @@ void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext, if (self->getCapacity() <= maxCapacity) { return; } - freeAndMakeCapacity(self, self->partialKeyLen, writeContext, impl); + freeAndMakeCapacity(self, self->partialKeyLen, writeContext); } #if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__) @@ -1799,13 +1792,13 @@ void rezero(Node *n, InternalVersionT z) { #endif void mergeWithChild(TaggedNodePointer &self, WriteContext *writeContext, - Node3 *self3, ConflictSet::Impl *impl) { + Node3 *self3) { assert(!self3->entryPresent); Node *child = self3->children[0]; const int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen; if (minCapacity > child->getCapacity()) { - freeAndMakeCapacity(child, minCapacity, writeContext, impl); + freeAndMakeCapacity(child, minCapacity, writeContext); } // Merge partial key with child @@ -1826,9 +1819,6 @@ void mergeWithChild(TaggedNodePointer &self, WriteContext *writeContext, child->parent = self->parent; child->parentsIndex = self->parentsIndex; - // Max versions are stored in the parent, so we need to update it now - // that we have a new parent. Safe to call since the root never has a partial - // key. setMaxVersion(child, std::max(childMaxVersion, writeContext->zero)); self = child; @@ -1841,63 +1831,59 @@ bool needsDownsize(Node *n) { return n->numChildren + n->entryPresent < minTable[n->getType()]; } -void downsize(Node3 *self, WriteContext *writeContext, - ConflictSet::Impl *impl) { +void downsize(Node3 *self, WriteContext *writeContext) { if (self->numChildren == 0) { auto *newSelf = writeContext->allocate(self->partialKeyLen); newSelf->copyChildrenAndKeyFrom(*self); - getInTree(self, impl) = newSelf; + getInTree(self) = newSelf; writeContext->deferRelease(self, newSelf); } else { assert(self->numChildren == 1 && !self->entryPresent); - mergeWithChild(getInTree(self, impl), writeContext, self, impl); + mergeWithChild(getInTree(self), writeContext, self); } } -void downsize(Node16 *self, WriteContext *writeContext, - ConflictSet::Impl *impl) { +void downsize(Node16 *self, WriteContext *writeContext) { assert(self->numChildren + int(self->entryPresent) < kMinChildrenNode16); auto *newSelf = writeContext->allocate(self->partialKeyLen); newSelf->copyChildrenAndKeyFrom(*self); - getInTree(self, impl) = newSelf; + getInTree(self) = newSelf; writeContext->deferRelease(self, newSelf); } -void downsize(Node48 *self, WriteContext *writeContext, - ConflictSet::Impl *impl) { +void downsize(Node48 *self, WriteContext *writeContext) { assert(self->numChildren + int(self->entryPresent) < kMinChildrenNode48); auto *newSelf = writeContext->allocate(self->partialKeyLen); newSelf->copyChildrenAndKeyFrom(*self); - getInTree(self, impl) = newSelf; + getInTree(self) = newSelf; writeContext->deferRelease(self, newSelf); } -void downsize(Node256 *self, WriteContext *writeContext, - ConflictSet::Impl *impl) { +void downsize(Node256 *self, WriteContext *writeContext) { assert(self->numChildren + int(self->entryPresent) < kMinChildrenNode256); auto *self256 = (Node256 *)self; auto *newSelf = writeContext->allocate(self->partialKeyLen); newSelf->copyChildrenAndKeyFrom(*self256); - getInTree(self, impl) = newSelf; + getInTree(self) = newSelf; writeContext->deferRelease(self256, newSelf); } -void downsize(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl) { +void downsize(Node *self, WriteContext *writeContext) { switch (self->getType()) { case Type_Node0: // GCOVR_EXCL_LINE __builtin_unreachable(); // GCOVR_EXCL_LINE case Type_Node3: - downsize(static_cast(self), writeContext, impl); + downsize(static_cast(self), writeContext); break; case Type_Node16: - downsize(static_cast(self), writeContext, impl); + downsize(static_cast(self), writeContext); break; case Type_Node48: - downsize(static_cast(self), writeContext, impl); + downsize(static_cast(self), writeContext); break; case Type_Node256: - downsize(static_cast(self), writeContext, impl); + downsize(static_cast(self), writeContext); break; default: // GCOVR_EXCL_LINE __builtin_unreachable(); // GCOVR_EXCL_LINE @@ -1908,8 +1894,7 @@ void downsize(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl) { // path to self. May invalidate children of self->parent. Returns a pointer to // the node after self. Precondition: `self->entryPresent` -Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl, - bool logical) { +Node *erase(Node *self, WriteContext *writeContext, bool logical) { ++writeContext->accum.entries_erased; assert(self->parent != nullptr); @@ -1928,12 +1913,12 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl, if (self->numChildren != 0) { if (needsDownsize(self)) { - downsize(self, writeContext, impl); + downsize(self, writeContext); } while (self->releaseDeferred) { self = self->forwardTo; } - maybeDecreaseCapacity(self, writeContext, impl); + maybeDecreaseCapacity(self, writeContext); if (result != nullptr) { while (result->releaseDeferred) { result = result->forwardTo; @@ -1964,7 +1949,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl, sizeof(parent3->children[0])); if (needsDownsize(parent3)) { - downsize(parent3, writeContext, impl); + downsize(parent3, writeContext); } } break; case Type_Node16: { @@ -1983,7 +1968,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl, sizeof(parent16->children[0])); if (needsDownsize(parent16)) { - downsize(parent16, writeContext, impl); + downsize(parent16, writeContext); } } break; case Type_Node48: { @@ -2013,7 +1998,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl, sizeof(parent48->children[0])); if (needsDownsize(parent48)) { - downsize(parent48, writeContext, impl); + downsize(parent48, writeContext); } } break; case Type_Node256: { @@ -2024,7 +2009,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl, --parent->numChildren; if (needsDownsize(parent256)) { - downsize(parent256, writeContext, impl); + downsize(parent256, writeContext); } } break; default: // GCOVR_EXCL_LINE @@ -2034,7 +2019,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl, while (parent->releaseDeferred) { parent = parent->forwardTo; } - maybeDecreaseCapacity(parent, writeContext, impl); + maybeDecreaseCapacity(parent, writeContext); if (result != nullptr) { while (result->releaseDeferred) { @@ -2631,11 +2616,13 @@ TrivialSpan getSearchPath(Arena &arena, Node *n) { for (int i = n->partialKeyLen - 1; i >= 0; --i) { result.push_back(n->partialKey()[i]); } - if (n->parent == nullptr) { - break; - } result.push_back(n->parentsIndex); n = n->parent; + if (n->parent == nullptr) { + // Implicit byte from rootParent + result.pop_back(); + break; + } } std::reverse(result.begin(), result.end()); return {result.begin(), result.size()}; @@ -2735,12 +2722,13 @@ checkMaxBetweenExclusiveImpl(Node256 *n, int begin, int end, // of the result will have `maxVersion` set to `writeVersion` as a // postcondition. Nodes along the search path may be invalidated. Callers must // ensure that the max version of the self argument is updated. -[[nodiscard]] TaggedNodePointer * -insert(TaggedNodePointer *self, TrivialSpan key, InternalVersionT writeVersion, - WriteContext *writeContext, ConflictSet::Impl *impl) { +[[nodiscard]] TaggedNodePointer *insert(TaggedNodePointer *self, + TrivialSpan key, + InternalVersionT writeVersion, + WriteContext *writeContext) { for (; key.size() != 0; ++writeContext->accum.insert_iterations) { - self = &getOrCreateChild(*self, key, writeVersion, writeContext, impl); + self = &getOrCreateChild(*self, key, writeVersion, writeContext); } return self; } @@ -2798,10 +2786,9 @@ void eraseTree(Node *root, WriteContext *writeContext) { } void addPointWrite(TaggedNodePointer &root, TrivialSpan key, - InternalVersionT writeVersion, WriteContext *writeContext, - ConflictSet::Impl *impl) { + InternalVersionT writeVersion, WriteContext *writeContext) { ++writeContext->accum.point_writes; - auto n = *insert(&root, key, writeVersion, writeContext, impl); + auto n = *insert(&root, key, writeVersion, writeContext); if (!n->entryPresent) { ++writeContext->accum.entries_inserted; auto *p = nextLogical(n); @@ -2930,13 +2917,12 @@ struct AddedWriteRange { AddedWriteRange addWriteRange(Node *beginRoot, TrivialSpan begin, Node *endRoot, TrivialSpan end, InternalVersionT writeVersion, - WriteContext *writeContext, - ConflictSet::Impl *impl) { + WriteContext *writeContext) { ++writeContext->accum.range_writes; - Node *beginNode = *insert(&getInTree(beginRoot, impl), begin, writeVersion, - writeContext, impl); + Node *beginNode = + *insert(&getInTree(beginRoot), begin, writeVersion, writeContext); addKey(beginNode); if (!beginNode->entryPresent) { ++writeContext->accum.entries_inserted; @@ -2951,8 +2937,7 @@ AddedWriteRange addWriteRange(Node *beginRoot, TrivialSpan begin, Node *endRoot, } beginNode->entry.pointVersion = writeVersion; - Node *endNode = - *insert(&getInTree(endRoot, impl), end, writeVersion, writeContext, impl); + Node *endNode = *insert(&getInTree(endRoot), end, writeVersion, writeContext); addKey(endNode); if (!endNode->entryPresent) { @@ -2971,8 +2956,7 @@ AddedWriteRange addWriteRange(Node *beginRoot, TrivialSpan begin, Node *endRoot, return {beginNode, endNode}; } -void eraseInRange(Node *beginNode, Node *endNode, WriteContext *writeContext, - ConflictSet::Impl *impl) { +void eraseInRange(Node *beginNode, Node *endNode, WriteContext *writeContext) { auto checkNodesEq = [](Node *lhs, Node *&rhs) { assert(!lhs->releaseDeferred); while (rhs->releaseDeferred) [[unlikely]] { @@ -2984,7 +2968,7 @@ void eraseInRange(Node *beginNode, Node *endNode, WriteContext *writeContext, // Erase nodes in range Node *iter = nextLogical(beginNode); for (; !checkNodesEq(iter, endNode); - iter = erase(iter, writeContext, impl, /*logical*/ true)) { + iter = erase(iter, writeContext, /*logical*/ true)) { } // Inserting end trashed the last node's maxVersion. Fix that. Safe to call @@ -2993,22 +2977,21 @@ void eraseInRange(Node *beginNode, Node *endNode, WriteContext *writeContext, } void addWriteRange(TaggedNodePointer &root, TrivialSpan begin, TrivialSpan end, - InternalVersionT writeVersion, WriteContext *writeContext, - ConflictSet::Impl *impl) { + InternalVersionT writeVersion, WriteContext *writeContext) { int lcp = longestCommonPrefix(begin.data(), end.data(), std::min(begin.size(), end.size())); if (lcp == begin.size() && end.size() == begin.size() + 1 && end.back() == 0) { - return addPointWrite(root, begin, writeVersion, writeContext, impl); + return addPointWrite(root, begin, writeVersion, writeContext); } auto useAsRoot = - insert(&root, begin.subspan(0, lcp), writeVersion, writeContext, impl); + insert(&root, begin.subspan(0, lcp), writeVersion, writeContext); auto [beginNode, endNode] = addWriteRange( *useAsRoot, begin.subspan(lcp, begin.size() - lcp), *useAsRoot, - end.subspan(lcp, end.size() - lcp), writeVersion, writeContext, impl); + end.subspan(lcp, end.size() - lcp), writeVersion, writeContext); - eraseInRange(beginNode, endNode, writeContext, impl); + eraseInRange(beginNode, endNode, writeContext); } Node *firstGeqPhysical(Node *n, const TrivialSpan key) { @@ -4780,10 +4763,6 @@ bool checkRangeRead(Node *n, TrivialSpan begin, TrivialSpan end, return checkRangeRightSide(n, end, lcp, readVersion, readContext); } - // This makes it safe to check maxVersion within checkRangeLeftSide. If this - // were false, then we would have returned above since lcp == begin.size(). - assert(!(n->parent == nullptr && begin.size() == 0)); - return checkRangeStartsWith(n, begin.subspan(0, lcp), begin[lcp], end[lcp], readVersion, readContext) && checkRangeLeftSide(n, begin, lcp + 1, readVersion, readContext) && @@ -4803,14 +4782,16 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { } else { bool ok; if (reads[i].end.len == 0) { - ok = checkPointRead( - root, TrivialSpan(reads[i].begin.p, reads[i].begin.len), - InternalVersionT(reads[i].readVersion), &context.readContext); + ok = checkPointRead(rootParent->children[0], + TrivialSpan(reads[i].begin.p, reads[i].begin.len), + InternalVersionT(reads[i].readVersion), + &context.readContext); } else { - ok = checkRangeRead( - root, TrivialSpan(reads[i].begin.p, reads[i].begin.len), - TrivialSpan(reads[i].end.p, reads[i].end.len), - InternalVersionT(reads[i].readVersion), &context.readContext); + ok = checkRangeRead(rootParent->children[0], + TrivialSpan(reads[i].begin.p, reads[i].begin.len), + TrivialSpan(reads[i].end.p, reads[i].end.len), + InternalVersionT(reads[i].readVersion), + &context.readContext); } result[i] = ok ? Commit : Conflict; } @@ -4837,13 +4818,13 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { check::Job inProgress[kConcurrent]; context.count = count; context.oldestVersionFullPrecision = oldestVersionFullPrecision; - context.root = root; + context.root = rootParent->children[0]; context.queries = reads; context.results = result; int64_t started = std::min(kConcurrent, count); context.started = started; for (int i = 0; i < started; i++) { - inProgress[i].init(reads + i, result + i, root, + inProgress[i].init(reads + i, result + i, rootParent->children[0], oldestVersionFullPrecision); } for (int i = 0; i < started - 1; i++) { @@ -4921,7 +4902,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { interleaved_insert::Context context; context.writeVersion = writeVersion; context.count = count; - context.root = root; + context.root = rootParent->children[0]; context.writes = writes; context.results = stackResults; if (count > kStackResultMax) [[unlikely]] { @@ -4983,9 +4964,9 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { context.results[i].insertionPoint->forwardTo; } if (context.results[i].endInsertionPoint == nullptr) { - addPointWrite(getInTree(context.results[i].insertionPoint, this), - context.results[i].remaining, writeVersion, &writeContext, - this); + addPointWrite(getInTree(context.results[i].insertionPoint), + context.results[i].remaining, writeVersion, + &writeContext); } else { if (firstRangeWrite == nullptr) { firstRangeWrite = context.results + i; @@ -5000,7 +4981,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { auto [beginNode, endNode] = addWriteRange( context.results[i].insertionPoint, context.results[i].remaining, context.results[i].endInsertionPoint, - context.results[i].endRemaining, writeVersion, &writeContext, this); + context.results[i].endRemaining, writeVersion, &writeContext); context.results[i].insertionPoint = beginNode; context.results[i].endInsertionPoint = endNode; } @@ -5022,7 +5003,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { iter->endInsertionPoint = iter->endInsertionPoint->forwardTo; } eraseInRange(iter->insertionPoint, iter->endInsertionPoint, - &writeContext, this); + &writeContext); } } @@ -5053,11 +5034,11 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { auto begin = TrivialSpan(w.begin.p, w.begin.len); auto end = TrivialSpan(w.end.p, w.end.len); if (w.end.len > 0) { - addWriteRange(root, begin, end, InternalVersionT(writeVersion), - &writeContext, this); + addWriteRange(rootParent->children[0], begin, end, + InternalVersionT(writeVersion), &writeContext); } else { - addPointWrite(root, begin, InternalVersionT(writeVersion), - &writeContext, this); + addPointWrite(rootParent->children[0], begin, + InternalVersionT(writeVersion), &writeContext); } } } @@ -5078,7 +5059,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { if (oldestExtantVersion < writeVersion - kMaxCorrectVersionWindow) [[unlikely]] { if (writeVersion > newestVersionFullPrecision + kNominalVersionWindow) { - eraseTree(root, &writeContext); + eraseTree(rootParent->children[0], &writeContext); init(writeVersion - kNominalVersionWindow); } @@ -5106,6 +5087,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { } if (count > 0) { + rootParent->childMaxVersion[0] = InternalVersionT(writeVersion); int firstNotInserted = 0; bool batchHasOnlyPointWrites = writes[0].end.len == 0; bool batchIsSorted = true; @@ -5153,11 +5135,14 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { // Spends up to `fuel` gc'ing, and returns its unused fuel. Reclaims memory // and updates oldestExtantVersion after spending enough fuel. int64_t gcScanStep(int64_t fuel) { - Node *n = firstGeqPhysical(root, removalKey); + Node *n = firstGeqPhysical(rootParent->children[0], removalKey); // There's no way to erase removalKey without introducing a key after it assert(n != nullptr); // Don't erase the root - if (n == root) { + if (n == rootParent->children[0]) { + for (auto &v : rootParent->childMaxVersion) { + v = std::max(v, oldestVersion); + } rezero(n, oldestVersion); n = nextPhysical(n); } @@ -5176,7 +5161,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { // There's no way to insert a range such that range version of the // right node is greater than the point version of the left node assert(n->entry.rangeVersion <= oldestVersion); - n = erase(n, &writeContext, this, /*logical*/ false); + n = erase(n, &writeContext, /*logical*/ false); } else { n = nextPhysical(n); } @@ -5227,7 +5212,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { cursor -= partialKey.size(); memcpy(cursor, partialKey.data(), partialKey.size()); - if (n->parent == nullptr) { + if (n->parent == rootParent) { break; } @@ -5297,17 +5282,28 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { keyUpdates = 10; // Insert "" - root = writeContext.allocate(0); - root->numChildren = 0; - root->parent = nullptr; - root->entryPresent = false; - root->partialKeyLen = 0; + rootParent->children[0] = writeContext.allocate(0); + rootParent->children[0]->numChildren = 0; + rootParent->children[0]->parent = rootParent; + rootParent->children[0]->parentsIndex = 0; + rootParent->children[0]->entryPresent = false; + rootParent->children[0]->partialKeyLen = 0; + rootParent->entryPresent = false; + rootParent->numChildren = 1; + rootParent->parent = nullptr; + rootParent->parentsIndex = 0; + for (auto &v : rootParent->childMaxVersion) { + v = this->oldestVersion; + } + rootParent->index[0] = 0; + rootParent->partialKeyLen = 0; + rootParent->releaseDeferred = false; - addKey(root); + addKey(rootParent->children[0]); - root->entryPresent = true; - root->entry.pointVersion = this->oldestVersion; - root->entry.rangeVersion = this->oldestVersion; + rootParent->children[0]->entryPresent = true; + rootParent->children[0]->entry.pointVersion = this->oldestVersion; + rootParent->children[0]->entry.rangeVersion = this->oldestVersion; #if !USE_64_BIT InternalVersionT::zero = writeContext.zero = this->oldestVersion; @@ -5318,13 +5314,15 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { explicit Impl(int64_t oldestVersion) { assert(oldestVersion >= 0); + rootParent = writeContext.allocate(0); init(oldestVersion); metrics = initMetrics(metricsList, metricsCount); } ~Impl() { - eraseTree(root, &writeContext); + eraseTree(rootParent->children[0], &writeContext); safe_free(metrics, metricsCount * sizeof(metrics[0])); safe_free(removalBuffer, removalBufferSize); + writeContext.release(rootParent); } WriteContext writeContext; @@ -5337,7 +5335,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { TrivialSpan removalKey; int64_t keyUpdates; - TaggedNodePointer root; + Node3 *rootParent; InternalVersionT oldestVersion; int64_t oldestVersionFullPrecision; int64_t oldestExtantVersion; @@ -5419,9 +5417,8 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { } }; -TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *impl) { - return n->parent == nullptr ? impl->root - : getChildExists(n->parent, n->parentsIndex); +TaggedNodePointer &getInTree(Node *n) { + return getChildExists(n->parent, n->parentsIndex); } // Internal entry points. Public entry points should just delegate to these @@ -5634,11 +5631,13 @@ std::string getSearchPathPrintable(Node *n) { for (int i = n->partialKeyLen - 1; i >= 0; --i) { result.push_back(n->partialKey()[i]); } - if (n->parent == nullptr) { - break; - } result.push_back(n->parentsIndex); n = n->parent; + if (n->parent == nullptr) { + // Implicit byte from rootParent + result.pop_back(); + break; + } } std::reverse(result.begin(), result.end()); if (result.size() > 0) { @@ -5686,15 +5685,13 @@ std::string getSearchPath(Node *n) { return std::string((const char *)result.data(), result.size()); } -[[maybe_unused]] void debugPrintDot(FILE *file, Node *node, - ConflictSet::Impl *impl) { +[[maybe_unused]] void debugPrintDot(FILE *file, Node *node) { constexpr int kSeparation = 3; struct DebugDotPrinter { - explicit DebugDotPrinter(FILE *file, ConflictSet::Impl *impl) - : file(file), impl(impl) {} + explicit DebugDotPrinter(FILE *file) : file(file) {} void print(Node *n, int y = 0) { assert(n != nullptr); @@ -5720,13 +5717,12 @@ std::string getSearchPath(Node *n) { } int x = 0; FILE *file; - ConflictSet::Impl *impl; }; fprintf(file, "digraph ConflictSet {\n"); fprintf(file, " node [shape = box];\n"); assert(node != nullptr); - DebugDotPrinter printer{file, impl}; + DebugDotPrinter printer{file}; printer.print(node); fprintf(file, "}\n"); } @@ -6114,7 +6110,7 @@ void printTree() { write.begin = "art"_s; write.end = ""_s; cs.addWrites(&write, 1, ++writeVersion); - debugPrintDot(stdout, cs.root, &cs); + debugPrintDot(stdout, cs.rootParent->children[0]); } int main(void) { benchHorizontal16(); } @@ -6132,13 +6128,13 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { if (!done1) { done1 = driver1.next(); if (!driver1.ok) { - debugPrintDot(stdout, driver1.cs.root, &driver1.cs); + debugPrintDot(stdout, driver1.cs.rootParent->children[0]); fflush(stdout); abort(); } - if (!checkCorrectness(driver1.cs.root, driver1.cs.oldestVersion, - &driver1.cs)) { - debugPrintDot(stdout, driver1.cs.root, &driver1.cs); + if (!checkCorrectness(driver1.cs.rootParent->children[0], + driver1.cs.oldestVersion, &driver1.cs)) { + debugPrintDot(stdout, driver1.cs.rootParent->children[0]); fflush(stdout); abort(); } @@ -6146,13 +6142,13 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { if (!done2) { done2 = driver2.next(); if (!driver2.ok) { - debugPrintDot(stdout, driver2.cs.root, &driver2.cs); + debugPrintDot(stdout, driver2.cs.rootParent->children[0]); fflush(stdout); abort(); } - if (!checkCorrectness(driver2.cs.root, driver2.cs.oldestVersion, - &driver2.cs)) { - debugPrintDot(stdout, driver2.cs.root, &driver2.cs); + if (!checkCorrectness(driver2.cs.rootParent->children[0], + driver2.cs.oldestVersion, &driver2.cs)) { + debugPrintDot(stdout, driver2.cs.rootParent->children[0]); fflush(stdout); abort(); }