diff --git a/ConflictSet.cpp b/ConflictSet.cpp index 3d02172..7106784 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -2489,91 +2489,6 @@ downLeftSpine: } namespace { -// Return true if the max version among all keys that start with key[:prefixLen] -// that are >= key is <= readVersion -bool checkRangeLeftSide(Node *n, std::span key, int prefixLen, - InternalVersionT readVersion, ReadContext *tls) { - auto remaining = key; - int searchPathLen = 0; - for (;; ++tls->range_read_iterations_accum) { - if (remaining.size() == 0) { - assert(searchPathLen >= prefixLen); - return maxVersion(n) <= readVersion; - } - - if (searchPathLen >= prefixLen) { - if (!checkMaxBetweenExclusive(n, remaining[0], 256, readVersion, tls)) { - return false; - } - } - - auto [c, maxV] = getChildAndMaxVersion(n, remaining[0]); - Node *child = c; - if (child == nullptr) { - auto c = getChildGeq(n, remaining[0]); - if (c != nullptr) { - if (searchPathLen < prefixLen) { - n = c; - goto downLeftSpine; - } - n = c; - return maxVersion(n) <= readVersion; - } else { - n = nextSibling(n); - if (n == nullptr) { - return true; - } - goto downLeftSpine; - } - } - - n = child; - remaining = remaining.subspan(1, remaining.size() - 1); - ++searchPathLen; - - if (n->partialKeyLen > 0) { - int commonLen = std::min(n->partialKeyLen, remaining.size()); - int i = longestCommonPrefix(n->partialKey(), remaining.data(), commonLen); - searchPathLen += i; - if (i < commonLen) { - auto c = n->partialKey()[i] <=> remaining[i]; - if (c > 0) { - if (searchPathLen < prefixLen) { - goto downLeftSpine; - } - if (n->entryPresent && n->entry.rangeVersion > readVersion) { - return false; - } - return maxVersion(n) <= readVersion; - } else { - n = nextSibling(n); - if (n == nullptr) { - return true; - } - goto downLeftSpine; - } - } - if (commonLen == n->partialKeyLen) { - // partial key matches - remaining = remaining.subspan(commonLen, remaining.size() - commonLen); - } else if (n->partialKeyLen > int(remaining.size())) { - assert(searchPathLen >= prefixLen); - if (n->entryPresent && n->entry.rangeVersion > readVersion) { - return false; - } - return maxVersion(n) <= readVersion; - } - } - if (maxV <= readVersion) { - return true; - } - } -downLeftSpine: - for (; !n->entryPresent; n = getFirstChildExists(n)) { - } - return n->entry.rangeVersion <= readVersion; -} - // Return true if the max version among all keys that start with key[:prefixLen] // that are < key is <= readVersion bool checkRangeRightSide(Node *n, std::span key, int prefixLen, @@ -2669,14 +2584,6 @@ downLeftSpine: } } // namespace -bool finishCheckRangeRead(int lcp, Node *n, std::span begin, - std::span end, - InternalVersionT readVersion, ReadContext *tls) { - - return checkRangeLeftSide(n, begin, lcp + 1, readVersion, tls) && - checkRangeRightSide(n, end, lcp + 1, readVersion, tls); -} - #ifdef __x86_64__ // Explicitly instantiate with target avx512f attribute so the compiler can // inline compare16_32bit_avx512, and generally use avx512f within more @@ -3032,10 +2939,13 @@ struct CheckJob { Node *n; std::span begin; - std::span end; // range read only - std::span commonPrefixRemaining; // range read only - Node *child; // range read only - int lcp; // range read only + std::span end; // range read only + std::span remaining; // range read only + Node *child; // range read only + int lcp; // range read only + int prefixLen; // range read only + int searchPathLen; // range read only + Node *commonPrefixNode; // range read only InternalVersionT readVersion; ConflictSet::Result *result; Continuation continuation; @@ -3361,6 +3271,18 @@ static Continuation commonPrefixIterTable[] = { common_prefix_iter, common_prefix_iter, common_prefix_iter}; +template +FLATTEN PRESERVE_NONE void left_side_iter(CheckJob *, CheckContext *); + +FLATTEN PRESERVE_NONE void left_side_down_left_spine(CheckJob *, + CheckContext *); + +FLATTEN PRESERVE_NONE void done_left_side_iter(CheckJob *, CheckContext *); + +static Continuation leftSideIterTable[] = { + left_side_iter, left_side_iter, left_side_iter, + left_side_iter, left_side_iter}; + FLATTEN PRESERVE_NONE void begin(CheckJob *job, CheckContext *context) { job->lcp = longestCommonPrefix(job->begin.data(), job->end.data(), std::min(job->begin.size(), job->end.size())); @@ -3379,13 +3301,13 @@ FLATTEN PRESERVE_NONE void begin(CheckJob *job, CheckContext *context) { } ++context->tls->range_read_accum; - job->commonPrefixRemaining = job->begin.subspan(0, job->lcp); + job->remaining = job->begin.subspan(0, job->lcp); - if (job->commonPrefixRemaining.size() == 0) { + if (job->remaining.size() == 0) { MUSTTAIL return done_common_prefix_iter(job, context); } - auto c = getChild(job->n, job->commonPrefixRemaining[0]); + auto c = getChild(job->n, job->remaining[0]); job->child = c; if (job->child == nullptr) { MUSTTAIL return done_common_prefix_iter(job, context); @@ -3404,26 +3326,25 @@ void common_prefix_iter(CheckJob *job, CheckContext *context) { NodeT *child = static_cast(job->child); if (child->partialKeyLen > 0) { - int cl = std::min(child->partialKeyLen, - job->commonPrefixRemaining.size() - 1); - int i = longestCommonPrefix(child->partialKey(), - job->commonPrefixRemaining.data() + 1, cl); + int cl = std::min(child->partialKeyLen, job->remaining.size() - 1); + int i = + longestCommonPrefix(child->partialKey(), job->remaining.data() + 1, cl); if (i != child->partialKeyLen) { MUSTTAIL return done_common_prefix_iter(job, context); } } job->n = job->child; - job->commonPrefixRemaining = job->commonPrefixRemaining.subspan( - 1 + child->partialKeyLen, - job->commonPrefixRemaining.size() - (1 + child->partialKeyLen)); + job->remaining = job->remaining.subspan(1 + child->partialKeyLen, + job->remaining.size() - + (1 + child->partialKeyLen)); ++context->tls->range_read_iterations_accum; - if (job->commonPrefixRemaining.size() == 0) { + if (job->remaining.size() == 0) { MUSTTAIL return done_common_prefix_iter(job, context); } - auto c = getChild(job->n, job->commonPrefixRemaining[0]); + auto c = getChild(job->n, job->remaining[0]); job->child = c; if (job->child == nullptr) { MUSTTAIL return done_common_prefix_iter(job, context); @@ -3440,17 +3361,17 @@ FLATTEN PRESERVE_NONE void done_common_prefix_iter(CheckJob *job, { Arena arena; assert(getSearchPath(arena, job->n) <=> - job->begin.subspan(0, job->lcp - - job->commonPrefixRemaining.size()) == + job->begin.subspan(0, job->lcp - job->remaining.size()) == 0); } - const int consumed = job->lcp - job->commonPrefixRemaining.size(); + const int consumed = job->lcp - job->remaining.size(); assume(consumed >= 0); job->begin = job->begin.subspan(consumed, int(job->begin.size()) - consumed); job->end = job->end.subspan(consumed, int(job->end.size()) - consumed); job->lcp -= consumed; + job->commonPrefixNode = job->n; if (job->lcp == int(job->begin.size())) { *job->result = checkRangeRightSide(job->n, job->end, job->lcp, @@ -3471,13 +3392,206 @@ FLATTEN PRESERVE_NONE void done_common_prefix_iter(CheckJob *job, MUSTTAIL return complete(job, context); } - *job->result = finishCheckRangeRead(job->lcp, job->n, job->begin, job->end, - job->readVersion, context->tls) - ? ConflictSet::Commit - : ConflictSet::Conflict; + job->remaining = job->begin; + job->searchPathLen = 0; + job->prefixLen = job->lcp + 1; + + if (job->remaining.size() == 0) { + assert(job->searchPathLen >= job->prefixLen); + if (maxVersion(job->n) > job->readVersion) { + job->setResult(false); + MUSTTAIL return complete(job, context); + } else { + job->continuation = done_left_side_iter; + MUSTTAIL return job->continuation(job, context); + } + } + + if (job->searchPathLen >= job->prefixLen) { + if (!checkMaxBetweenExclusive(job->n, job->remaining[0], 256, + job->readVersion, context->tls)) { + job->setResult(false); + MUSTTAIL return complete(job, context); + } + } + + auto c = getChild(job->n, job->remaining[0]); + Node *child = c; + if (child == nullptr) { + auto c = getChildGeq(job->n, job->remaining[0]); + if (c != nullptr) { + if (job->searchPathLen < job->prefixLen) { + job->n = c; + job->continuation = left_side_down_left_spine; + __builtin_prefetch(job->n); + MUSTTAIL return keepGoing(job, context); + } + job->n = c; + if (maxVersion(job->n) > job->readVersion) { + job->setResult(false); + MUSTTAIL return complete(job, context); + } + job->continuation = done_left_side_iter; + MUSTTAIL return job->continuation(job, context); + } else { + job->n = nextSibling(job->n); + if (job->n == nullptr) { + job->continuation = done_left_side_iter; + MUSTTAIL return job->continuation(job, context); + } + job->continuation = left_side_down_left_spine; + __builtin_prefetch(job->n); + MUSTTAIL return keepGoing(job, context); + } + } + + job->n = child; + job->continuation = leftSideIterTable[c.getType()]; + __builtin_prefetch(job->n); + MUSTTAIL return keepGoing(job, context); +} + +// Return true if the max version among all keys that start with key[:prefixLen] +// that are >= key is <= readVersion +template +FLATTEN PRESERVE_NONE void left_side_iter(CheckJob *job, + CheckContext *context) { + job->remaining = job->remaining.subspan(1, job->remaining.size() - 1); + ++job->searchPathLen; + + if (job->n->partialKeyLen > 0) { + int commonLen = std::min(job->n->partialKeyLen, job->remaining.size()); + int i = longestCommonPrefix(job->n->partialKey(), job->remaining.data(), + commonLen); + job->searchPathLen += i; + if (i < commonLen) { + auto c = job->n->partialKey()[i] <=> job->remaining[i]; + if (c > 0) { + if (job->searchPathLen < job->prefixLen) { + job->continuation = left_side_down_left_spine; + MUSTTAIL return job->continuation(job, context); + } + if (job->n->entryPresent && + job->n->entry.rangeVersion > job->readVersion) { + job->setResult(false); + MUSTTAIL return complete(job, context); + } + if (maxVersion(job->n) > job->readVersion) { + job->setResult(false); + MUSTTAIL return complete(job, context); + } + job->continuation = done_left_side_iter; + MUSTTAIL return job->continuation(job, context); + } else { + job->n = nextSibling(job->n); + if (job->n == nullptr) { + job->continuation = done_left_side_iter; + MUSTTAIL return job->continuation(job, context); + } + job->continuation = left_side_down_left_spine; + MUSTTAIL return job->continuation(job, context); + } + } + if (commonLen == job->n->partialKeyLen) { + // partial key matches + job->remaining = + job->remaining.subspan(commonLen, job->remaining.size() - commonLen); + } else if (job->n->partialKeyLen > int(job->remaining.size())) { + assert(job->searchPathLen >= job->prefixLen); + if (job->n->entryPresent && + job->n->entry.rangeVersion > job->readVersion) { + job->setResult(false); + MUSTTAIL return complete(job, context); + } + if (maxVersion(job->n) > job->readVersion) { + job->setResult(false); + MUSTTAIL return complete(job, context); + } + job->continuation = done_left_side_iter; + MUSTTAIL return job->continuation(job, context); + } + } + + ++context->tls->range_read_iterations_accum; + + if (job->remaining.size() == 0) { + assert(job->searchPathLen >= job->prefixLen); + if (maxVersion(job->n) > job->readVersion) { + *job->result = ConflictSet::Conflict; + MUSTTAIL return complete(job, context); + } else { + job->continuation = done_left_side_iter; + MUSTTAIL return job->continuation(job, context); + } + } + + if (job->searchPathLen >= job->prefixLen) { + if (!checkMaxBetweenExclusive(job->n, job->remaining[0], 256, + job->readVersion, context->tls)) { + *job->result = ConflictSet::Conflict; + MUSTTAIL return complete(job, context); + } + } + + auto c = getChild(job->n, job->remaining[0]); + Node *child = c; + if (child == nullptr) { + auto c = getChildGeq(job->n, job->remaining[0]); + if (c != nullptr) { + if (job->searchPathLen < job->prefixLen) { + job->n = c; + job->continuation = left_side_down_left_spine; + __builtin_prefetch(job->n); + MUSTTAIL return keepGoing(job, context); + } + job->n = c; + if (maxVersion(job->n) > job->readVersion) { + *job->result = ConflictSet::Conflict; + MUSTTAIL return complete(job, context); + } + job->continuation = done_left_side_iter; + MUSTTAIL return job->continuation(job, context); + } else { + job->n = nextSibling(job->n); + if (job->n == nullptr) { + job->continuation = done_left_side_iter; + MUSTTAIL return job->continuation(job, context); + } + job->continuation = left_side_down_left_spine; + __builtin_prefetch(job->n); + MUSTTAIL return keepGoing(job, context); + } + } + + job->n = child; + job->continuation = leftSideIterTable[c.getType()]; + __builtin_prefetch(job->n); + MUSTTAIL return keepGoing(job, context); +} + +FLATTEN PRESERVE_NONE void done_left_side_iter(CheckJob *job, + CheckContext *context) { + + job->setResult(checkRangeRightSide(job->commonPrefixNode, job->end, + job->lcp + 1, job->readVersion, + context->tls)); MUSTTAIL return complete(job, context); } +void left_side_down_left_spine(CheckJob *job, CheckContext *context) { + if (job->n->entryPresent) { + if (job->n->entry.rangeVersion > job->readVersion) { + job->setResult(false); + MUSTTAIL return complete(job, context); + } + job->continuation = done_left_side_iter; + MUSTTAIL return job->continuation(job, context); + } + job->n = getFirstChildExists(job->n); + __builtin_prefetch(job->n); + MUSTTAIL return keepGoing(job, context); +} + } // namespace check_range_read_state_machine void CheckJob::init(const ConflictSet::ReadRange *read,