diff --git a/ConflictSet.cpp b/ConflictSet.cpp index 704fe99..13fdda3 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -1211,6 +1211,53 @@ void maybeDecreaseCapacity(Node *&self, NodeAllocators *allocators, freeAndMakeCapacityAtLeast(self, maxCapacity, allocators, impl, false); } +void rezero(Node *n, InternalVersionT z) { +#if DEBUG_VERBOSE && !defined(NDEBUG) + fprintf(stderr, "rezero to %" PRId64 ": %s\n", z.toInt64(), + getSearchPathPrintable(n).c_str()); +#endif + if (n->entryPresent) { + n->entry.pointVersion = std::max(n->entry.pointVersion, z); + n->entry.rangeVersion = std::max(n->entry.rangeVersion, z); + } + switch (n->getType()) { + case Type_Node0: { + } break; + case Type_Node3: { + auto *self = static_cast(n); + for (int i = 0; i < 3; ++i) { + self->childMaxVersion[i] = std::max(self->childMaxVersion[i], z); + } + } break; + case Type_Node16: { + auto *self = static_cast(n); + for (int i = 0; i < 16; ++i) { + self->childMaxVersion[i] = std::max(self->childMaxVersion[i], z); + } + } break; + case Type_Node48: { + auto *self = static_cast(n); + for (int i = 0; i < 48; ++i) { + self->childMaxVersion[i] = std::max(self->childMaxVersion[i], z); + } + for (auto &m : self->maxOfMax) { + m = std::max(m, z); + } + } break; + case Type_Node256: { + auto *self = static_cast(n); + for (int i = 0; i < 256; ++i) { + self->childMaxVersion[i] = std::max(self->childMaxVersion[i], z); + } + for (auto &m : self->maxOfMax) { + m = std::max(m, z); + } + } break; + default: // GCOVR_EXCL_LINE + __builtin_unreachable(); // GCOVR_EXCL_LINE + } +} + void maybeDownsize(Node *self, NodeAllocators *allocators, ConflictSet::Impl *impl, Node *&dontInvalidate) { @@ -1262,6 +1309,9 @@ void maybeDownsize(Node *self, NodeAllocators *allocators, // Max versions are stored in the parent, so we need to update it now // that we have a new parent. setMaxVersion(child, impl, childMaxVersion); + if (child->parent) { + rezero(child->parent, InternalVersionT::zero); + } getInTree(self, impl) = child; allocators->node3.release(self3); @@ -1304,7 +1354,7 @@ void maybeDownsize(Node *self, NodeAllocators *allocators, // the node after self. If erase invalidates the pointee of `dontInvalidate`, it // will update it to its new pointee as well. Node *erase(Node *self, NodeAllocators *allocators, ConflictSet::Impl *impl, - Node *&dontInvalidate) { + bool logical, Node *&dontInvalidate) { assert(self->parent != nullptr); #if DEBUG_VERBOSE && !defined(NDEBUG) @@ -1314,7 +1364,7 @@ Node *erase(Node *self, NodeAllocators *allocators, ConflictSet::Impl *impl, Node *parent = self->parent; uint8_t parentsIndex = self->parentsIndex; - auto *result = nextLogical(self); + auto *result = logical ? nextLogical(self) : nextPhysical(self); removeKey(self); self->entryPresent = false; @@ -2659,7 +2709,8 @@ void addPointWrite(Node *&root, std::span key, n->entry.pointVersion = writeVersion; setMaxVersion(n, impl, writeVersion); n->entry.rangeVersion = - p != nullptr ? p->entry.rangeVersion : InternalVersionT::zero; + p == nullptr ? InternalVersionT::zero + : std::max(p->entry.rangeVersion, InternalVersionT::zero); } else { assert(writeVersion >= n->entry.pointVersion); n->entry.pointVersion = writeVersion; @@ -2722,7 +2773,8 @@ void addWriteRange(Node *&root, std::span begin, if (insertedBegin) { auto *p = nextLogical(beginNode); beginNode->entry.rangeVersion = - p != nullptr ? p->entry.rangeVersion : InternalVersionT::zero; + p == nullptr ? InternalVersionT::zero + : std::max(p->entry.rangeVersion, InternalVersionT::zero); beginNode->entry.pointVersion = writeVersion; assert(maxVersion(beginNode, impl) <= writeVersion); setMaxVersion(beginNode, impl, writeVersion); @@ -2741,7 +2793,8 @@ void addWriteRange(Node *&root, std::span begin, if (insertedEnd) { auto *p = nextLogical(endNode); endNode->entry.pointVersion = - p != nullptr ? p->entry.rangeVersion : InternalVersionT::zero; + p == nullptr ? InternalVersionT::zero + : std::max(p->entry.rangeVersion, InternalVersionT::zero); auto m = maxVersion(endNode, impl); setMaxVersion(endNode, impl, std::max(m, endNode->entry.pointVersion)); @@ -2756,21 +2809,16 @@ void addWriteRange(Node *&root, std::span begin, } for (beginNode = nextLogical(beginNode); beginNode != endNode; - beginNode = erase(beginNode, allocators, impl, endNode)) { + beginNode = + erase(beginNode, allocators, impl, /*logical*/ true, endNode)) { } } -Iterator firstGeq(Node *n, const std::span key) { +Node *firstGeqPhysical(Node *n, const std::span key) { auto remaining = key; for (;;) { if (remaining.size() == 0) { - if (n->entryPresent) { - return {n, 0}; - } - int c = getChildGeq(n, 0); - assert(c >= 0); - n = getChildExists(n, c); - goto downLeftSpine; + return n; } auto *child = getChild(n, remaining[0]); @@ -2778,7 +2826,7 @@ Iterator firstGeq(Node *n, const std::span key) { int c = getChildGeq(n, remaining[0]); if (c >= 0) { n = getChildExists(n, c); - goto downLeftSpine; + return n; } else { n = nextSibling(n); if (n == nullptr) { @@ -2786,9 +2834,9 @@ Iterator firstGeq(Node *n, const std::span key) { // final library, since we can't remove a key without introducing a // key after it, and the only production caller of firstGeq is for // resuming the setOldestVersion scan. - return {nullptr, 1}; // GCOVR_EXCL_LINE + return nullptr; // GCOVR_EXCL_LINE } - goto downLeftSpine; + return n; } } @@ -2801,10 +2849,10 @@ Iterator firstGeq(Node *n, const std::span key) { if (i < commonLen) { auto c = n->partialKey()[i] <=> remaining[i]; if (c > 0) { - goto downLeftSpine; + return n; } else { n = nextSibling(n); - goto downLeftSpine; + return n; } } if (commonLen == n->partialKeyLen) { @@ -2813,19 +2861,10 @@ Iterator firstGeq(Node *n, const std::span key) { } else if (n->partialKeyLen > int(remaining.size())) { // n is the first physical node greater than remaining, and there's no // eq node - goto downLeftSpine; + return n; } } } -downLeftSpine: - for (;;) { - if (n->entryPresent) { - return {n, 1}; - } - int c = getChildGeq(n, 0); - assert(c >= 0); - n = getChildExists(n, c); - } } struct __attribute__((visibility("hidden"))) ConflictSet::Impl { @@ -2852,8 +2891,19 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { void addWrites(const WriteRange *writes, int count, int64_t writeVersion) { assert(writeVersion >= newestVersionFullPrecision); + + // TODO allow this condition + assert(writeVersion < newestVersionFullPrecision + kNominalVersionWindow); + newestVersionFullPrecision = writeVersion; - InternalVersionT::zero = oldestVersion; + setOldestVersion( + std::max(oldestVersionFullPrecision, + newestVersionFullPrecision - kNominalVersionWindow)); + while (oldestExtantVersion < + newestVersionFullPrecision - kMaxCorrectVersionWindow) { + gcScanStep(1000); + } + for (int i = 0; i < count; ++i) { const auto &w = writes[i]; auto begin = std::span(w.begin.p, w.begin.len); @@ -2870,16 +2920,20 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { } } - // Spends up to `fuel` gc'ing, and returns its unused fuel + // Spends up to `fuel` gc'ing, and returns its unused fuel. Reclaims memory + // and updates oldestExtantVersion after spending enough fuel. int64_t gcScanStep(int64_t fuel) { - Node *n = firstGeq(root, removalKey).n; + Node *n = firstGeqPhysical(root, removalKey); // There's no way to erase removalKey without introducing a key after it assert(n != nullptr); // Don't erase the root if (n == root) { + rezero(n, oldestVersion); + rootMaxVersion = std::max(rootMaxVersion, oldestVersion); n = nextPhysical(n); } for (; fuel > 0 && n != nullptr; --fuel) { + rezero(n, oldestVersion); if (n->entryPresent && std::max(n->entry.pointVersion, n->entry.rangeVersion) <= oldestVersion) { // Any transaction n would have prevented from committing is @@ -2889,7 +2943,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { // node is greater than the point version of the left node assert(n->entry.rangeVersion <= oldestVersion); Node *dummy = nullptr; - n = erase(n, &allocators, this, dummy); + n = erase(n, &allocators, this, /*logical*/ false, dummy); } else { maybeDecreaseCapacity(n, &allocators, this); n = nextPhysical(n); @@ -2897,6 +2951,14 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { } if (n == nullptr) { removalKey = {}; + oldestExtantVersion = oldestVersionAtGcBegin; + oldestVersionAtGcBegin = oldestVersionFullPrecision; +#if DEBUG_VERBOSE && !defined(NDEBUG) + fprintf(stderr, + "new oldestExtantVersion: %" PRId64 + ", new oldestVersionAtGcBegin: %" PRId64 "\n", + oldestExtantVersion, oldestVersionAtGcBegin); +#endif } else { removalKeyArena = Arena(); removalKey = getSearchPath(removalKeyArena, n); @@ -2905,8 +2967,10 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { } void setOldestVersion(int64_t o) { + if (o <= oldestVersionFullPrecision) { + return; + } InternalVersionT oldestVersion{o}; - assert(o >= oldestVersionFullPrecision); this->oldestVersionFullPrecision = o; this->oldestVersion = oldestVersion; InternalVersionT::zero = oldestVersion; @@ -2923,6 +2987,8 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { explicit Impl(int64_t oldestVersion) : oldestVersion(oldestVersion), oldestVersionFullPrecision(oldestVersion), + oldestExtantVersion(oldestVersion), + oldestVersionAtGcBegin(oldestVersion), newestVersionFullPrecision(oldestVersion) { #if DEBUG_VERBOSE fprintf(stderr, "radix_tree: create\n"); @@ -2941,6 +3007,8 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { root->entryPresent = true; root->entry.pointVersion = this->oldestVersion; root->entry.rangeVersion = this->oldestVersion; + + InternalVersionT::zero = this->oldestVersion; } ~Impl() { #if DEBUG_VERBOSE @@ -2962,6 +3030,8 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { // need to make sure we clean up versions in the tree before they fall out of // the `kMaxCorrectVersionWindow` window. int64_t oldestVersionFullPrecision; + int64_t oldestExtantVersion; + int64_t oldestVersionAtGcBegin; int64_t newestVersionFullPrecision; int64_t totalBytes = 0; }; @@ -3099,6 +3169,74 @@ int64_t internal_getBytes(ConflictSet::Impl *impl) { return impl->totalBytes; } // GCOVR_EXCL_START +Iterator firstGeqLogical(Node *n, const std::span key) { + auto remaining = key; + for (;;) { + if (remaining.size() == 0) { + if (n->entryPresent) { + return {n, 0}; + } + int c = getChildGeq(n, 0); + assert(c >= 0); + n = getChildExists(n, c); + goto downLeftSpine; + } + + auto *child = getChild(n, remaining[0]); + if (child == nullptr) { + int c = getChildGeq(n, remaining[0]); + if (c >= 0) { + n = getChildExists(n, c); + goto downLeftSpine; + } else { + n = nextSibling(n); + if (n == nullptr) { + // This line is genuinely unreachable from any entry point of the + // final library, since we can't remove a key without introducing a + // key after it, and the only production caller of firstGeq is for + // resuming the setOldestVersion scan. + return {nullptr, 1}; // GCOVR_EXCL_LINE + } + goto downLeftSpine; + } + } + + n = child; + remaining = remaining.subspan(1, remaining.size() - 1); + + if (n->partialKeyLen > 0) { + int commonLen = std::min(n->partialKeyLen, remaining.size()); + int i = longestCommonPrefix(n->partialKey(), remaining.data(), commonLen); + if (i < commonLen) { + auto c = n->partialKey()[i] <=> remaining[i]; + if (c > 0) { + goto downLeftSpine; + } else { + n = nextSibling(n); + goto downLeftSpine; + } + } + if (commonLen == n->partialKeyLen) { + // partial key matches + remaining = remaining.subspan(commonLen, remaining.size() - commonLen); + } else if (n->partialKeyLen > int(remaining.size())) { + // n is the first physical node greater than remaining, and there's no + // eq node + goto downLeftSpine; + } + } + } +downLeftSpine: + for (;;) { + if (n->entryPresent) { + return {n, 1}; + } + int c = getChildGeq(n, 0); + assert(c >= 0); + n = getChildExists(n, c); + } +} + void ConflictSet::check(const ReadRange *reads, Result *results, int count) const { internal_check(impl, reads, results, count); @@ -3295,13 +3433,59 @@ void checkParentPointers(Node *node, bool &success) { } Iterator firstGeq(Node *n, std::string_view key) { - return firstGeq( + return firstGeqLogical( n, std::span((const uint8_t *)key.data(), key.size())); } +void checkVersionsGeqOldestExtant(Node *n, + InternalVersionT oldestExtantVersion) { + if (n->entryPresent) { + assert(n->entry.pointVersion >= oldestExtantVersion); + assert(n->entry.rangeVersion >= oldestExtantVersion); + } + switch (n->getType()) { + case Type_Node0: { + } break; + case Type_Node3: { + auto *self = static_cast(n); + for (int i = 0; i < 3; ++i) { + assert(self->childMaxVersion[i] >= oldestExtantVersion); + } + } break; + case Type_Node16: { + auto *self = static_cast(n); + for (int i = 0; i < 16; ++i) { + assert(self->childMaxVersion[i] >= oldestExtantVersion); + } + } break; + case Type_Node48: { + auto *self = static_cast(n); + for (int i = 0; i < 48; ++i) { + assert(self->childMaxVersion[i] >= oldestExtantVersion); + } + for (auto m : self->maxOfMax) { + assert(m >= oldestExtantVersion); + } + } break; + case Type_Node256: { + auto *self = static_cast(n); + for (int i = 0; i < 256; ++i) { + assert(self->childMaxVersion[i] >= oldestExtantVersion); + } + for (auto m : self->maxOfMax) { + assert(m >= oldestExtantVersion); + } + } break; + default: // GCOVR_EXCL_LINE + __builtin_unreachable(); // GCOVR_EXCL_LINE + } +} + [[maybe_unused]] InternalVersionT checkMaxVersion(Node *root, Node *node, InternalVersionT oldestVersion, bool &success, ConflictSet::Impl *impl) { + checkVersionsGeqOldestExtant(node, + InternalVersionT(impl->oldestExtantVersion)); auto expected = InternalVersionT::zero; if (node->entryPresent) { expected = std::max(expected, node->entry.pointVersion);