From 44afb8be002c41520938c45b50d0fffaba9449dd Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Fri, 11 Oct 2024 17:26:34 -0700 Subject: [PATCH] Interleave the common prefix search in range check --- ConflictSet.cpp | 154 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 104 insertions(+), 50 deletions(-) diff --git a/ConflictSet.cpp b/ConflictSet.cpp index 9eec877..c37be0f 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -2669,49 +2669,16 @@ downLeftSpine: } } // namespace -bool checkRangeRead(int lcp, Node *n, std::span begin, - std::span end, InternalVersionT readVersion, - ReadContext *tls) { - ++tls->range_read_accum; - - auto remaining = begin.subspan(0, lcp); +bool finishCheckRangeRead(int lcp, Node *n, std::span begin, + std::span end, + std::span commonPrefixRemaining, + InternalVersionT readVersion, ReadContext *tls) { Arena arena; - - // Advance down common prefix, but stay on a physical path in the tree - for (;; ++tls->range_read_iterations_accum) { - assert(getSearchPath(arena, n) <=> - begin.subspan(0, lcp - remaining.size()) == - 0); - if (remaining.size() == 0) { - break; - } - auto [c, v] = getChildAndMaxVersion(n, remaining[0]); - Node *child = c; - if (child == nullptr) { - break; - } - - if (child->partialKeyLen > 0) { - int cl = std::min(child->partialKeyLen, remaining.size() - 1); - int i = - longestCommonPrefix(child->partialKey(), remaining.data() + 1, cl); - if (i != child->partialKeyLen) { - break; - } - } - if (v <= readVersion) { - ++tls->range_read_short_circuit_accum; - return true; - } - n = child; - remaining = - remaining.subspan(1 + child->partialKeyLen, - remaining.size() - (1 + child->partialKeyLen)); - } - assert(getSearchPath(arena, n) <=> begin.subspan(0, lcp - remaining.size()) == + assert(getSearchPath(arena, n) <=> + begin.subspan(0, lcp - commonPrefixRemaining.size()) == 0); - const int consumed = lcp - remaining.size(); + const int consumed = lcp - commonPrefixRemaining.size(); assume(consumed >= 0); begin = begin.subspan(consumed, int(begin.size()) - consumed); @@ -3087,7 +3054,10 @@ struct CheckJob { Node *n; std::span begin; - std::span end; // range read only + std::span end; // range read only + std::span commonPrefixRemaining; // range read only + Node *child; // range read only + int lcp; // range read only InternalVersionT readVersion; ConflictSet::Result *result; Continuation continuation; @@ -3403,16 +3373,26 @@ void down_left_spine(CheckJob *job, CheckContext *context) { namespace check_range_read_state_machine { FLATTEN PRESERVE_NONE void begin(CheckJob *, CheckContext *); +template +FLATTEN PRESERVE_NONE void common_prefix_iter(CheckJob *, CheckContext *); + +FLATTEN PRESERVE_NONE void down_left_spine(CheckJob *, CheckContext *); + +static Continuation commonPrefixIterTable[] = { + common_prefix_iter, common_prefix_iter, + common_prefix_iter, common_prefix_iter, + common_prefix_iter}; + FLATTEN PRESERVE_NONE void begin(CheckJob *job, CheckContext *context) { - int lcp = longestCommonPrefix(job->begin.data(), job->end.data(), - std::min(job->begin.size(), job->end.size())); - if (lcp == int(job->begin.size()) && + job->lcp = longestCommonPrefix(job->begin.data(), job->end.data(), + std::min(job->begin.size(), job->end.size())); + if (job->lcp == int(job->begin.size()) && job->end.size() == job->begin.size() + 1 && job->end.back() == 0) { job->continuation = check_point_read_state_machine::begin; // Call directly since we have nothing to prefetch MUSTTAIL return job->continuation(job, context); } - if (lcp == int(job->begin.size() - 1) && + if (job->lcp == int(job->begin.size() - 1) && job->end.size() == job->begin.size() && int(job->begin.back()) + 1 == int(job->end.back())) { job->continuation = check_prefix_read_state_machine::begin; @@ -3420,11 +3400,85 @@ FLATTEN PRESERVE_NONE void begin(CheckJob *job, CheckContext *context) { MUSTTAIL return job->continuation(job, context); } - *job->result = checkRangeRead(lcp, job->n, job->begin, job->end, - job->readVersion, context->tls) - ? ConflictSet::Commit - : ConflictSet::Conflict; - MUSTTAIL return complete(job, context); + ++context->tls->range_read_accum; + job->commonPrefixRemaining = job->begin.subspan(0, job->lcp); + + if (job->commonPrefixRemaining.size() == 0) { + *job->result = finishCheckRangeRead(job->lcp, job->n, job->begin, job->end, + job->commonPrefixRemaining, + job->readVersion, context->tls) + ? ConflictSet::Commit + : ConflictSet::Conflict; + MUSTTAIL return complete(job, context); + } + + auto c = getChild(job->n, job->commonPrefixRemaining[0]); + job->child = c; + if (job->child == nullptr) { + *job->result = finishCheckRangeRead(job->lcp, job->n, job->begin, job->end, + job->commonPrefixRemaining, + job->readVersion, context->tls) + ? ConflictSet::Commit + : ConflictSet::Conflict; + MUSTTAIL return complete(job, context); + } + + job->continuation = commonPrefixIterTable[c.getType()]; + __builtin_prefetch(job->child); + MUSTTAIL return keepGoing(job, context); +} + +// Advance down common prefix, but stay on a physical path in the tree +template +void common_prefix_iter(CheckJob *job, CheckContext *context) { + + assert(NodeT::kType == job->child->getType()); + NodeT *child = static_cast(job->child); + + if (child->partialKeyLen > 0) { + int cl = std::min(child->partialKeyLen, + job->commonPrefixRemaining.size() - 1); + int i = longestCommonPrefix(child->partialKey(), + job->commonPrefixRemaining.data() + 1, cl); + if (i != child->partialKeyLen) { + *job->result = finishCheckRangeRead(job->lcp, job->n, job->begin, + job->end, job->commonPrefixRemaining, + job->readVersion, context->tls) + ? ConflictSet::Commit + : ConflictSet::Conflict; + MUSTTAIL return complete(job, context); + } + } + job->n = job->child; + job->commonPrefixRemaining = job->commonPrefixRemaining.subspan( + 1 + child->partialKeyLen, + job->commonPrefixRemaining.size() - (1 + child->partialKeyLen)); + + ++context->tls->range_read_iterations_accum; + + if (job->commonPrefixRemaining.size() == 0) { + *job->result = finishCheckRangeRead(job->lcp, job->n, job->begin, job->end, + job->commonPrefixRemaining, + job->readVersion, context->tls) + ? ConflictSet::Commit + : ConflictSet::Conflict; + MUSTTAIL return complete(job, context); + } + + auto c = getChild(job->n, job->commonPrefixRemaining[0]); + job->child = c; + if (job->child == nullptr) { + *job->result = finishCheckRangeRead(job->lcp, job->n, job->begin, job->end, + job->commonPrefixRemaining, + job->readVersion, context->tls) + ? ConflictSet::Commit + : ConflictSet::Conflict; + MUSTTAIL return complete(job, context); + } + + job->continuation = commonPrefixIterTable[c.getType()]; + __builtin_prefetch(job->child); + MUSTTAIL return keepGoing(job, context); } } // namespace check_range_read_state_machine