diff --git a/ConflictSet.cpp b/ConflictSet.cpp index 2455faf..6ce57cf 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -1891,6 +1891,8 @@ template struct Iterator; // an entry associated with a node or a leaf. struct IteratorBase { + IteratorBase() = default; + explicit IteratorBase(Node *node) : node(node), type(node ? node->getType() : Type_Node0) {} @@ -1928,6 +1930,8 @@ struct IteratorBase { IteratorBase nextLogical(); IteratorBase nextPhysical(); InternalVersionT getMaxVersion(); + IteratorBase parent() { return IteratorBase{node->parent}; } + uint8_t parentsIndex() { return node->parentsIndex; } InternalVersionT exchangeMaxVersion(InternalVersionT); InternalVersionT setMaxVersion(); TrivialSpan partialKey(); @@ -1936,7 +1940,11 @@ struct IteratorBase { struct ChildAndMaxVersion; ChildAndMaxVersion getChildAndMaxVersion(int index); IteratorBase getChildGeq(int index); + IteratorBase getChild(int index); IteratorBase nextSibling(); + TrivialSpan getSearchPath(Arena &arena); + + Node *escapeHatch() { return node; } protected: Node *node; @@ -1963,9 +1971,14 @@ template struct Iterator : IteratorBase { IteratorBase getChildGeq(int index) { return IteratorBase{::getChildGeq(static_cast(node), index)}; } + IteratorBase getChild(int index) { + return IteratorBase{::getChild(static_cast(node), index)}; + } TrivialSpan partialKey() { return {static_cast(node)->partialKey(), node->partialKeyLen}; } + + T *escapeHatch() { return static_cast(node); } }; TrivialSpan IteratorBase::partialKey() { @@ -2037,6 +2050,23 @@ IteratorBase IteratorBase::getChildGeq(int index) { } } +IteratorBase IteratorBase::getChild(int index) { + switch (type) { + case Type_Node0: + return as().getChild(index); + case Type_Node3: + return as().getChild(index); + case Type_Node16: + return as().getChild(index); + case Type_Node48: + return as().getChild(index); + case Type_Node256: + return as().getChild(index); + default: // GCOVR_EXCL_LINE + __builtin_unreachable(); // GCOVR_EXCL_LINE + } +} + InternalVersionT IteratorBase::getMaxVersion() { return ::maxVersion(node); } bool IteratorBase::checkRangeVersionOfFirstGeq(InternalVersionT readVersion) { @@ -2049,6 +2079,26 @@ IteratorBase IteratorBase::nextSibling() { return IteratorBase{::nextSibling(node)}; } +TrivialSpan IteratorBase::getSearchPath(Arena &arena) { + Node *n = node; + assert(n != nullptr); + auto result = vector(arena); + for (;;) { + for (int i = n->partialKeyLen - 1; i >= 0; --i) { + result.push_back(n->partialKey()[i]); + } + result.push_back(n->parentsIndex); + n = n->parent; + if (n->parent == nullptr) { + // Implicit byte from rootParent + result.pop_back(); + break; + } + } + std::reverse(result.begin(), result.end()); + return {result.begin(), result.size()}; +} + // Precondition: self is not the root. May invalidate nodes along the search // path to self. May invalidate children of self->parent. Returns a pointer to // the node after self. Precondition: `self->entryPresent` @@ -2792,58 +2842,58 @@ TrivialSpan getSearchPath(Arena &arena, Node *n) { // Precondition: transitively, no child of n has a search path that's a longer // prefix of key than n template -bool checkRangeStartsWith(NodeT *nTyped, TrivialSpan key, int begin, int end, - InternalVersionT readVersion, +bool checkRangeStartsWith(Iterator nTyped, TrivialSpan key, int begin, + int end, InternalVersionT readVersion, ReadContext *readContext) { - Node *n; + IteratorBase n; #if DEBUG_VERBOSE && !defined(NDEBUG) fprintf(stderr, "%s(%02x,%02x)*\n", printable(key).c_str(), begin, end); #endif auto remaining = key; if (remaining.size() == 0) { - return checkMaxBetweenExclusive(nTyped, begin, end, readVersion, - readContext); + return checkMaxBetweenExclusive(nTyped.escapeHatch(), begin, end, + readVersion, readContext); } - auto cAndV = getChildAndMaxVersion(nTyped, remaining[0]); - Node *child = cAndV.child; - if (child == nullptr) { - auto c = getChildGeq(nTyped, remaining[0]); - if (c != nullptr) { - n = c; - return checkRangeVersionOfFirstGeq(n, readVersion); + auto cAndV = nTyped.getChildAndMaxVersion(remaining[0]); + IteratorBase child = cAndV.child; + if (!child.valid()) { + auto c = nTyped.getChildGeq(remaining[0]); + if (c.valid()) { + return c.checkRangeVersionOfFirstGeq(readVersion); } else { - n = nextSibling(nTyped); - if (n == nullptr) { + n = nTyped.nextSibling(); + if (!n.valid()) { return true; } - return checkRangeVersionOfFirstGeq(n, readVersion); + return n.checkRangeVersionOfFirstGeq(readVersion); } } n = child; remaining = remaining.subspan(1, remaining.size() - 1); - assert(n->partialKeyLen > 0); + auto partialKey = n.partialKey(); + assert(partialKey.size() > 0); { - int commonLen = std::min(n->partialKeyLen, remaining.size()); - int i = longestCommonPrefix(n->partialKey(), remaining.data(), commonLen); + int commonLen = std::min(partialKey.size(), remaining.size()); + int i = longestCommonPrefix(partialKey.data(), remaining.data(), commonLen); if (i < commonLen) { - auto c = n->partialKey()[i] <=> remaining[i]; + auto c = partialKey[i] <=> remaining[i]; if (c > 0) { - return checkRangeVersionOfFirstGeq(n, readVersion); + return n.checkRangeVersionOfFirstGeq(readVersion); } else { - n = nextSibling(n); - if (n == nullptr) { + n = n.nextSibling(); + if (!n.valid()) { return true; } - return checkRangeVersionOfFirstGeq(n, readVersion); + return n.checkRangeVersionOfFirstGeq(readVersion); } } - assert(n->partialKeyLen > remaining.size()); - if (begin < n->partialKey()[remaining.size()] && - n->partialKey()[remaining.size()] < end) { - if (n->entryPresent && n->entry.rangeVersion > readVersion) { + assert(partialKey.size() > remaining.size()); + if (begin < partialKey[remaining.size()] && + partialKey[remaining.size()] < end) { + if (n.entryPresent() && n.getEntry().rangeVersion > readVersion) { return false; } return cAndV.maxVersion <= readVersion; @@ -2854,6 +2904,30 @@ bool checkRangeStartsWith(NodeT *nTyped, TrivialSpan key, int begin, int end, __builtin_unreachable(); // GCOVR_EXCL_LINE } +bool checkRangeStartsWith(IteratorBase n, TrivialSpan key, int begin, int end, + InternalVersionT readVersion, + ReadContext *readContext) { + switch (n.getType()) { + case Type_Node0: + return checkRangeStartsWith(n.as(), key, begin, end, readVersion, + readContext); + case Type_Node3: + return checkRangeStartsWith(n.as(), key, begin, end, readVersion, + readContext); + case Type_Node16: + return checkRangeStartsWith(n.as(), key, begin, end, readVersion, + readContext); + case Type_Node48: + return checkRangeStartsWith(n.as(), key, begin, end, readVersion, + readContext); + case Type_Node256: + return checkRangeStartsWith(n.as(), key, begin, end, readVersion, + readContext); + default: // GCOVR_EXCL_LINE + __builtin_unreachable(); // GCOVR_EXCL_LINE + } +} + #ifdef __x86_64__ // Explicitly instantiate with target avx512f attribute so the compiler can // inline compare16_32bit_avx512, and generally use avx512f within more @@ -3723,7 +3797,8 @@ PRESERVE_NONE void done_common_prefix_iter(Job *job, Context *context) { // If this were not true we would have returned above assert(job->begin.size() > 0); - if (!checkRangeStartsWith(n, job->begin.subspan(0, job->lcp), + if (!checkRangeStartsWith(IteratorBase{n}.as(), + job->begin.subspan(0, job->lcp), job->begin[job->lcp], job->end[job->lcp], job->readVersion, &context->readContext)) { job->setResult(false); @@ -4689,7 +4764,7 @@ bool checkPrefixRead(IteratorBase n, const TrivialSpan key, // Return true if the max version among all keys that start with key[:prefixLen] // that are >= key is <= readVersion -bool checkRangeLeftSide(Node *n, TrivialSpan key, int prefixLen, +bool checkRangeLeftSide(IteratorBase n, TrivialSpan key, int prefixLen, InternalVersionT readVersion, ReadContext *readContext) { auto remaining = key; @@ -4697,33 +4772,32 @@ bool checkRangeLeftSide(Node *n, TrivialSpan key, int prefixLen, for (;; ++readContext->range_read_iterations_accum) { if (remaining.size() == 0) { assert(searchPathLen >= prefixLen); - return maxVersion(n) <= readVersion; + return n.getMaxVersion() <= readVersion; } if (searchPathLen >= prefixLen) { - if (!checkMaxBetweenExclusive(n, remaining[0], 256, readVersion, - readContext)) { + if (!checkMaxBetweenExclusive(n.escapeHatch(), remaining[0], 256, + readVersion, readContext)) { return false; } } - auto [c, maxV] = getChildAndMaxVersion(n, remaining[0]); - Node *child = c; - if (child == nullptr) { - auto c = getChildGeq(n, remaining[0]); - if (c != nullptr) { + auto [c, maxV] = n.getChildAndMaxVersion(remaining[0]); + IteratorBase child = c; + if (!child.valid()) { + auto c = n.getChildGeq(remaining[0]); + if (c.valid()) { if (searchPathLen < prefixLen) { - n = c; - return checkRangeVersionOfFirstGeq(n, readVersion); + return c.checkRangeVersionOfFirstGeq(readVersion); } n = c; - return maxVersion(n) <= readVersion; + return n.getMaxVersion() <= readVersion; } else { - n = nextSibling(n); - if (n == nullptr) { + n = n.nextSibling(); + if (!n.valid()) { return true; } - return checkRangeVersionOfFirstGeq(n, readVersion); + return n.checkRangeVersionOfFirstGeq(readVersion); } } @@ -4731,34 +4805,36 @@ bool checkRangeLeftSide(Node *n, TrivialSpan key, int prefixLen, remaining = remaining.subspan(1, remaining.size() - 1); ++searchPathLen; - if (n->partialKeyLen > 0) { - int commonLen = std::min(n->partialKeyLen, remaining.size()); - int i = longestCommonPrefix(n->partialKey(), remaining.data(), commonLen); + auto partialKey = n.partialKey(); + if (partialKey.size() > 0) { + int commonLen = std::min(partialKey.size(), remaining.size()); + int i = + longestCommonPrefix(partialKey.data(), remaining.data(), commonLen); searchPathLen += i; if (i < commonLen) { - auto c = n->partialKey()[i] <=> remaining[i]; + auto c = partialKey[i] <=> remaining[i]; if (c > 0) { if (searchPathLen < prefixLen) { - return checkRangeVersionOfFirstGeq(n, readVersion); + return n.checkRangeVersionOfFirstGeq(readVersion); } - if (n->entryPresent && n->entry.rangeVersion > readVersion) { + if (n.entryPresent() && n.getEntry().rangeVersion > readVersion) { return false; } return maxV <= readVersion; } else { - n = nextSibling(n); - if (n == nullptr) { + n = n.nextSibling(); + if (!n.valid()) { return true; } - return checkRangeVersionOfFirstGeq(n, readVersion); + return n.checkRangeVersionOfFirstGeq(readVersion); } } - if (commonLen == n->partialKeyLen) { + if (commonLen == partialKey.size()) { // partial key matches remaining = remaining.subspan(commonLen, remaining.size() - commonLen); - } else if (n->partialKeyLen > remaining.size()) { + } else if (partialKey.size() > remaining.size()) { assert(searchPathLen >= prefixLen); - if (n->entryPresent && n->entry.rangeVersion > readVersion) { + if (n.entryPresent() && n.getEntry().rangeVersion > readVersion) { return false; } return maxV <= readVersion; @@ -4772,7 +4848,7 @@ bool checkRangeLeftSide(Node *n, TrivialSpan key, int prefixLen, // Return true if the max version among all keys that start with key[:prefixLen] // that are < key is <= readVersion -bool checkRangeRightSide(Node *n, TrivialSpan key, int prefixLen, +bool checkRangeRightSide(IteratorBase n, TrivialSpan key, int prefixLen, InternalVersionT readVersion, ReadContext *readContext) { auto remaining = key; @@ -4781,31 +4857,30 @@ bool checkRangeRightSide(Node *n, TrivialSpan key, int prefixLen, for (;; ++readContext->range_read_iterations_accum) { assert(searchPathLen <= key.size()); if (remaining.size() == 0) { - return checkRangeVersionOfFirstGeq(n, readVersion); + return n.checkRangeVersionOfFirstGeq(readVersion); } if (searchPathLen >= prefixLen) { - if (n->entryPresent && n->entry.pointVersion > readVersion) { + if (n.entryPresent() && n.getEntry().pointVersion > readVersion) { return false; } - if (!checkMaxBetweenExclusive(n, -1, remaining[0], readVersion, - readContext)) { + if (!checkMaxBetweenExclusive(n.escapeHatch(), -1, remaining[0], + readVersion, readContext)) { return false; } } - if (searchPathLen > prefixLen && n->entryPresent && - n->entry.rangeVersion > readVersion) { + if (searchPathLen > prefixLen && n.entryPresent() && + n.getEntry().rangeVersion > readVersion) { return false; } - Node *child = getChild(n, remaining[0]); - if (child == nullptr) { - auto c = getChildGeq(n, remaining[0]); - if (c != nullptr) { - n = c; - return checkRangeVersionOfFirstGeq(n, readVersion); + IteratorBase child = n.getChild(remaining[0]); + if (!child.valid()) { + auto c = n.getChildGeq(remaining[0]); + if (c.valid()) { + return c.checkRangeVersionOfFirstGeq(readVersion); } else { goto backtrack; } @@ -4815,87 +4890,91 @@ bool checkRangeRightSide(Node *n, TrivialSpan key, int prefixLen, remaining = remaining.subspan(1, remaining.size() - 1); ++searchPathLen; - if (n->partialKeyLen > 0) { - int commonLen = std::min(n->partialKeyLen, remaining.size()); - int i = longestCommonPrefix(n->partialKey(), remaining.data(), commonLen); + auto partialKey = n.partialKey(); + if (partialKey.size() > 0) { + int commonLen = std::min(partialKey.size(), remaining.size()); + int i = + longestCommonPrefix(partialKey.data(), remaining.data(), commonLen); searchPathLen += i; if (i < commonLen) { ++searchPathLen; - auto c = n->partialKey()[i] <=> remaining[i]; + auto c = partialKey[i] <=> remaining[i]; if (c > 0) { - return checkRangeVersionOfFirstGeq(n, readVersion); + return n.checkRangeVersionOfFirstGeq(readVersion); } else { - if (searchPathLen > prefixLen && n->entryPresent && - n->entry.rangeVersion > readVersion) { + if (searchPathLen > prefixLen && n.entryPresent() && + n.getEntry().rangeVersion > readVersion) { return false; } goto backtrack; } } - if (commonLen == n->partialKeyLen) { + if (commonLen == partialKey.size()) { // partial key matches remaining = remaining.subspan(commonLen, remaining.size() - commonLen); - } else if (n->partialKeyLen > remaining.size()) { - return checkRangeVersionOfFirstGeq(n, readVersion); + } else if (partialKey.size() > remaining.size()) { + return n.checkRangeVersionOfFirstGeq(readVersion); } } } backtrack: for (;;) { // searchPathLen > prefixLen implies n is not the root - if (searchPathLen > prefixLen && maxVersion(n) > readVersion) { + if (searchPathLen > prefixLen && n.getMaxVersion() > readVersion) { return false; } - if (n->parent == nullptr) { + if (!n.parent().valid()) { return true; } - auto next = getChildGeq(n->parent, n->parentsIndex + 1); - if (next == nullptr) { - searchPathLen -= 1 + n->partialKeyLen; - n = n->parent; + auto next = n.parent().getChildGeq(n.parentsIndex() + 1); + if (!next.valid()) { + searchPathLen -= 1 + n.partialKey().size(); + n = n.parent(); } else { n = next; - return checkRangeVersionOfFirstGeq(n, readVersion); + return n.checkRangeVersionOfFirstGeq(readVersion); } } } -bool checkRangeRead(Node *n, TrivialSpan begin, TrivialSpan end, +bool checkRangeRead(IteratorBase n, TrivialSpan begin, TrivialSpan end, InternalVersionT readVersion, ReadContext *readContext) { int lcp = longestCommonPrefix(begin.data(), end.data(), std::min(begin.size(), end.size())); if (lcp == begin.size() && end.size() == begin.size() + 1 && end.back() == 0) { - return checkPointRead(IteratorBase{n}, begin, readVersion, readContext); + return checkPointRead(n, begin, readVersion, readContext); } if (lcp == begin.size() - 1 && end.size() == begin.size() && begin.back() + 1 == end.back()) { - return checkPrefixRead(IteratorBase{n}, begin, readVersion, readContext); + return checkPrefixRead(n, begin, readVersion, readContext); } ++readContext->range_read_accum; auto remaining = begin.subspan(0, lcp); +#ifndef NDEBUG Arena arena; +#endif // Advance down common prefix, but stay on a physical path in the tree for (;; ++readContext->range_read_iterations_accum) { - assert(getSearchPath(arena, n) <=> + assert(n.getSearchPath(arena) <=> begin.subspan(0, lcp - remaining.size()) == 0); if (remaining.size() == 0) { break; } - auto [c, v] = getChildAndMaxVersion(n, remaining[0]); - Node *child = c; - if (child == nullptr) { + auto [c, v] = n.getChildAndMaxVersion(remaining[0]); + IteratorBase child = c; + if (!child.valid()) { break; } - if (child->partialKeyLen > 0) { - int cl = std::min(child->partialKeyLen, remaining.size() - 1); - int i = - longestCommonPrefix(child->partialKey(), remaining.data() + 1, cl); - if (i != child->partialKeyLen) { + auto partialKey = child.partialKey(); + if (partialKey.size() > 0) { + int cl = std::min(partialKey.size(), remaining.size() - 1); + int i = longestCommonPrefix(partialKey.data(), remaining.data() + 1, cl); + if (i != partialKey.size()) { break; } } @@ -4904,11 +4983,10 @@ bool checkRangeRead(Node *n, TrivialSpan begin, TrivialSpan end, return true; } n = child; - remaining = - remaining.subspan(1 + child->partialKeyLen, - remaining.size() - (1 + child->partialKeyLen)); + remaining = remaining.subspan(1 + partialKey.size(), + remaining.size() - (1 + partialKey.size())); } - assert(getSearchPath(arena, n) <=> begin.subspan(0, lcp - remaining.size()) == + assert(n.getSearchPath(arena) <=> begin.subspan(0, lcp - remaining.size()) == 0); const int consumed = lcp - remaining.size(); @@ -4946,7 +5024,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { InternalVersionT(reads[i].readVersion), &context.readContext); } else { - ok = checkRangeRead(rootParent->children[0], + ok = checkRangeRead(IteratorBase{rootParent->children[0]}, TrivialSpan(reads[i].begin.p, reads[i].begin.len), TrivialSpan(reads[i].end.p, reads[i].end.len), InternalVersionT(reads[i].readVersion),