From 767dacc74201576849dbeec4bab121219949e003 Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Sun, 13 Oct 2024 20:05:06 -0700 Subject: [PATCH] Fully interleaved range read Would be great to find a way to make this all more readable --- ConflictSet.cpp | 273 ++++++++++++++++++++++++++++++------------------ 1 file changed, 170 insertions(+), 103 deletions(-) diff --git a/ConflictSet.cpp b/ConflictSet.cpp index f3d48cf..381c964 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -2488,102 +2488,6 @@ downLeftSpine: return n->entry.rangeVersion <= readVersion; } -namespace { -// Return true if the max version among all keys that start with key[:prefixLen] -// that are < key is <= readVersion -bool checkRangeRightSide(Node *n, std::span key, int prefixLen, - InternalVersionT readVersion, ReadContext *tls) { - auto remaining = key; - - assert(remaining.size() > 0); - - if (prefixLen == 0) { - if (n->entryPresent && n->entry.pointVersion > readVersion) { - return false; - } - - if (!checkMaxBetweenExclusive(n, -1, remaining[0], readVersion, tls)) { - return false; - } - } - - Node *commonPrefixNode = n; - - for (;;) { - Node *child = getChild(n, remaining[0]); - if (child == nullptr) { - auto c = getChildGeq(n, remaining[0]); - if (c != nullptr) { - n = c; - goto downLeftSpine; - } else { - if (n != commonPrefixNode && maxVersion(n) > readVersion) { - return false; - } - n = nextSibling(n); - if (n == nullptr) { - return true; - } - goto downLeftSpine; - } - } - - n = child; - remaining = remaining.subspan(1, remaining.size() - 1); - - if (n->partialKeyLen > 0) { - int commonLen = std::min(n->partialKeyLen, remaining.size()); - int i = longestCommonPrefix(n->partialKey(), remaining.data(), commonLen); - if (i < commonLen) { - auto c = n->partialKey()[i] <=> remaining[i]; - if (c > 0) { - goto downLeftSpine; - } else { - if ((n->parent != commonPrefixNode || i + 1 >= prefixLen) && - n->entryPresent && n->entry.rangeVersion > readVersion) { - return false; - } - if ((n->parent != commonPrefixNode || i + 1 >= prefixLen) && - maxVersion(n) > readVersion) { - return false; - } - n = nextSibling(n); - if (n == nullptr) { - return true; - } - goto downLeftSpine; - } - } - if (commonLen == n->partialKeyLen) { - // partial key matches - remaining = remaining.subspan(commonLen, remaining.size() - commonLen); - } else if (n->partialKeyLen > int(remaining.size())) { - goto downLeftSpine; - } - } - - ++tls->range_read_iterations_accum; - - if (remaining.size() == 0) { - goto downLeftSpine; - } - - if (n->entryPresent && (n->entry.pointVersion > readVersion || - n->entry.rangeVersion > readVersion)) { - return false; - } - - if (!checkMaxBetweenExclusive(n, -1, remaining[0], readVersion, tls)) { - return false; - } - } -downLeftSpine: - for (; !n->entryPresent; n = getFirstChildExists(n)) { - } - return n->entry.rangeVersion <= readVersion; -} -} // namespace - #ifdef __x86_64__ // Explicitly instantiate with target avx512f attribute so the compiler can // inline compare16_32bit_avx512, and generally use avx512f within more @@ -3289,6 +3193,13 @@ static Continuation leftSideIterTable[] = { left_side_iter, left_side_iter, left_side_iter, left_side_iter, left_side_iter}; +template +PRESERVE_NONE void right_side_iter(CheckJob *, CheckContext *); + +static Continuation rightSideIterTable[] = { + right_side_iter, right_side_iter, right_side_iter, + right_side_iter, right_side_iter}; + PRESERVE_NONE void begin(CheckJob *job, CheckContext *context) { job->lcp = longestCommonPrefix(job->begin.data(), job->end.data(), std::min(job->begin.size(), job->end.size())); @@ -3388,9 +3299,47 @@ PRESERVE_NONE void done_common_prefix_iter(CheckJob *job, job->commonPrefixNode = job->n; if (job->lcp == int(job->begin.size())) { - job->setResult(checkRangeRightSide(job->n, job->end, job->lcp, - job->readVersion, context->tls)); - MUSTTAIL return complete(job, context); + job->remaining = job->end; + if (job->lcp == 0) { + if (job->n->entryPresent && + job->n->entry.pointVersion > job->readVersion) { + job->setResult(false); + MUSTTAIL return complete(job, context); + } + + if (!checkMaxBetweenExclusive(job->n, -1, job->remaining[0], + job->readVersion, context->tls)) { + job->setResult(false); + MUSTTAIL return complete(job, context); + } + } + + // This is a hack + --job->lcp; + + auto c = getChild(job->n, job->remaining[0]); + Node *child = c; + if (child == nullptr) { + auto c = getChildGeq(job->n, job->remaining[0]); + if (c != nullptr) { + job->n = c; + job->continuation = check_point_read_state_machine::down_left_spine; + MUSTTAIL return job->continuation(job, context); + } else { + job->n = nextSibling(job->n); + if (job->n == nullptr) { + job->setResult(true); + MUSTTAIL return complete(job, context); + } + job->continuation = check_point_read_state_machine::down_left_spine; + MUSTTAIL return job->continuation(job, context); + } + } + + job->n = child; + job->continuation = rightSideIterTable[c.getType()]; + __builtin_prefetch(job->n); + MUSTTAIL return keepGoing(job, context); } // If this were not true we would have returned above @@ -3541,10 +3490,32 @@ PRESERVE_NONE void left_side_iter(CheckJob *job, CheckContext *context) { PRESERVE_NONE void done_left_side_iter(CheckJob *job, CheckContext *context) { - job->setResult(checkRangeRightSide(job->commonPrefixNode, job->end, - job->lcp + 1, job->readVersion, - context->tls)); - MUSTTAIL return complete(job, context); + job->n = job->commonPrefixNode; + job->remaining = job->end; + + auto c = getChild(job->n, job->remaining[0]); + Node *child = c; + if (child == nullptr) { + auto c = getChildGeq(job->n, job->remaining[0]); + if (c != nullptr) { + job->n = c; + job->continuation = check_point_read_state_machine::down_left_spine; + MUSTTAIL return job->continuation(job, context); + } else { + job->n = nextSibling(job->n); + if (job->n == nullptr) { + job->setResult(true); + MUSTTAIL return complete(job, context); + } + job->continuation = check_point_read_state_machine::down_left_spine; + MUSTTAIL return job->continuation(job, context); + } + } + + job->n = child; + job->continuation = rightSideIterTable[c.getType()]; + __builtin_prefetch(job->n); + MUSTTAIL return keepGoing(job, context); } void left_side_down_left_spine(CheckJob *job, CheckContext *context) { @@ -3561,6 +3532,102 @@ void left_side_down_left_spine(CheckJob *job, CheckContext *context) { MUSTTAIL return keepGoing(job, context); } +// Return true if the max version among all keys that start with key[:prefixLen] +// that are < key is <= readVersion +template +PRESERVE_NONE void right_side_iter(CheckJob *job, CheckContext *context) { + assert(NodeT::kType == job->n->getType()); + NodeT *n = static_cast(job->n); + + job->remaining = job->remaining.subspan(1, job->remaining.size() - 1); + + if (n->partialKeyLen > 0) { + int commonLen = std::min(n->partialKeyLen, job->remaining.size()); + int i = + longestCommonPrefix(n->partialKey(), job->remaining.data(), commonLen); + if (i < commonLen) { + auto c = n->partialKey()[i] <=> job->remaining[i]; + if (c > 0) { + job->continuation = check_point_read_state_machine::down_left_spine; + MUSTTAIL return job->continuation(job, context); + } else { + if ((n->parent != job->commonPrefixNode || i >= job->lcp) && + n->entryPresent && n->entry.rangeVersion > job->readVersion) { + job->setResult(false); + MUSTTAIL return complete(job, context); + } + if ((n->parent != job->commonPrefixNode || i >= job->lcp) && + maxVersion(n) > job->readVersion) { + job->setResult(false); + MUSTTAIL return complete(job, context); + } + job->n = nextSibling(job->n); + if (job->n == nullptr) { + job->setResult(true); + MUSTTAIL return complete(job, context); + } + job->continuation = check_point_read_state_machine::down_left_spine; + MUSTTAIL return job->continuation(job, context); + } + } + if (commonLen == n->partialKeyLen) { + // partial key matches + job->remaining = + job->remaining.subspan(commonLen, job->remaining.size() - commonLen); + } else if (n->partialKeyLen > int(job->remaining.size())) { + job->continuation = check_point_read_state_machine::down_left_spine; + MUSTTAIL return job->continuation(job, context); + } + } + + ++context->tls->range_read_iterations_accum; + + if (job->remaining.size() == 0) { + job->continuation = check_point_read_state_machine::down_left_spine; + MUSTTAIL return job->continuation(job, context); + } + + if (n->entryPresent && (n->entry.pointVersion > job->readVersion || + n->entry.rangeVersion > job->readVersion)) { + job->setResult(false); + MUSTTAIL return complete(job, context); + } + + if (!checkMaxBetweenExclusive(n, -1, job->remaining[0], job->readVersion, + context->tls)) { + job->setResult(false); + MUSTTAIL return complete(job, context); + } + + auto c = getChild(job->n, job->remaining[0]); + Node *child = c; + if (child == nullptr) { + auto c = getChildGeq(n, job->remaining[0]); + if (c != nullptr) { + job->n = c; + job->continuation = check_point_read_state_machine::down_left_spine; + MUSTTAIL return job->continuation(job, context); + } else { + if (n != job->commonPrefixNode && maxVersion(n) > job->readVersion) { + job->setResult(false); + MUSTTAIL return complete(job, context); + } + job->n = nextSibling(job->n); + if (job->n == nullptr) { + job->setResult(true); + MUSTTAIL return complete(job, context); + } + job->continuation = check_point_read_state_machine::down_left_spine; + MUSTTAIL return job->continuation(job, context); + } + } + + job->n = child; + job->continuation = rightSideIterTable[c.getType()]; + __builtin_prefetch(job->n); + MUSTTAIL return keepGoing(job, context); +} + } // namespace check_range_read_state_machine void CheckJob::init(const ConflictSet::ReadRange *read,