Interleave the common prefix search in range check

ConflictSet.cpp

@@ -2669,49 +2669,16 @@ downLeftSpine:
 }
 } // namespace
 
-bool checkRangeRead(int lcp, Node *n, std::span<const uint8_t> begin,
+bool finishCheckRangeRead(int lcp, Node *n, std::span<const uint8_t> begin,
-                    std::span<const uint8_t> end, InternalVersionT readVersion,
+                          std::span<const uint8_t> end,
-                    ReadContext *tls) {
+                          std::span<const uint8_t> commonPrefixRemaining,
-  ++tls->range_read_accum;
+                          InternalVersionT readVersion, ReadContext *tls) {
-
-  auto remaining = begin.subspan(0, lcp);
   Arena arena;
-
-  // Advance down common prefix, but stay on a physical path in the tree
-  for (;; ++tls->range_read_iterations_accum) {
   assert(getSearchPath(arena, n) <=>
-               begin.subspan(0, lcp - remaining.size()) ==
+             begin.subspan(0, lcp - commonPrefixRemaining.size()) ==
-           0);
-    if (remaining.size() == 0) {
-      break;
-    }
-    auto [c, v] = getChildAndMaxVersion(n, remaining[0]);
-    Node *child = c;
-    if (child == nullptr) {
-      break;
-    }
-
-    if (child->partialKeyLen > 0) {
-      int cl = std::min<int>(child->partialKeyLen, remaining.size() - 1);
-      int i =
-          longestCommonPrefix(child->partialKey(), remaining.data() + 1, cl);
-      if (i != child->partialKeyLen) {
-        break;
-      }
-    }
-    if (v <= readVersion) {
-      ++tls->range_read_short_circuit_accum;
-      return true;
-    }
-    n = child;
-    remaining =
-        remaining.subspan(1 + child->partialKeyLen,
-                          remaining.size() - (1 + child->partialKeyLen));
-  }
-  assert(getSearchPath(arena, n) <=> begin.subspan(0, lcp - remaining.size()) ==
          0);
 
-  const int consumed = lcp - remaining.size();
+  const int consumed = lcp - commonPrefixRemaining.size();
   assume(consumed >= 0);
 
   begin = begin.subspan(consumed, int(begin.size()) - consumed);
@@ -3088,6 +3055,9 @@ struct CheckJob {
   Node *n;
   std::span<const uint8_t> begin;
   std::span<const uint8_t> end;                   // range read only
+  std::span<const uint8_t> commonPrefixRemaining; // range read only
+  Node *child;                                    // range read only
+  int lcp;                                        // range read only
   InternalVersionT readVersion;
   ConflictSet::Result *result;
   Continuation continuation;
@@ -3403,16 +3373,26 @@ void down_left_spine(CheckJob *job, CheckContext *context) {
 namespace check_range_read_state_machine {
 FLATTEN PRESERVE_NONE void begin(CheckJob *, CheckContext *);
 
+template <class NodeT>
+FLATTEN PRESERVE_NONE void common_prefix_iter(CheckJob *, CheckContext *);
+
+FLATTEN PRESERVE_NONE void down_left_spine(CheckJob *, CheckContext *);
+
+static Continuation commonPrefixIterTable[] = {
+    common_prefix_iter<Node0>, common_prefix_iter<Node3>,
+    common_prefix_iter<Node16>, common_prefix_iter<Node48>,
+    common_prefix_iter<Node256>};
+
 FLATTEN PRESERVE_NONE void begin(CheckJob *job, CheckContext *context) {
-  int lcp = longestCommonPrefix(job->begin.data(), job->end.data(),
+  job->lcp = longestCommonPrefix(job->begin.data(), job->end.data(),
                                  std::min(job->begin.size(), job->end.size()));
-  if (lcp == int(job->begin.size()) &&
+  if (job->lcp == int(job->begin.size()) &&
       job->end.size() == job->begin.size() + 1 && job->end.back() == 0) {
     job->continuation = check_point_read_state_machine::begin;
     // Call directly since we have nothing to prefetch
     MUSTTAIL return job->continuation(job, context);
   }
-  if (lcp == int(job->begin.size() - 1) &&
+  if (job->lcp == int(job->begin.size() - 1) &&
       job->end.size() == job->begin.size() &&
       int(job->begin.back()) + 1 == int(job->end.back())) {
     job->continuation = check_prefix_read_state_machine::begin;
@@ -3420,13 +3400,87 @@ FLATTEN PRESERVE_NONE void begin(CheckJob *job, CheckContext *context) {
     MUSTTAIL return job->continuation(job, context);
   }
 
-  *job->result = checkRangeRead(lcp, job->n, job->begin, job->end,
+  ++context->tls->range_read_accum;
+  job->commonPrefixRemaining = job->begin.subspan(0, job->lcp);
+
+  if (job->commonPrefixRemaining.size() == 0) {
+    *job->result = finishCheckRangeRead(job->lcp, job->n, job->begin, job->end,
+                                        job->commonPrefixRemaining,
                                         job->readVersion, context->tls)
                        ? ConflictSet::Commit
                        : ConflictSet::Conflict;
     MUSTTAIL return complete(job, context);
   }
 
+  auto c = getChild(job->n, job->commonPrefixRemaining[0]);
+  job->child = c;
+  if (job->child == nullptr) {
+    *job->result = finishCheckRangeRead(job->lcp, job->n, job->begin, job->end,
+                                        job->commonPrefixRemaining,
+                                        job->readVersion, context->tls)
+                       ? ConflictSet::Commit
+                       : ConflictSet::Conflict;
+    MUSTTAIL return complete(job, context);
+  }
+
+  job->continuation = commonPrefixIterTable[c.getType()];
+  __builtin_prefetch(job->child);
+  MUSTTAIL return keepGoing(job, context);
+}
+
+// Advance down common prefix, but stay on a physical path in the tree
+template <class NodeT>
+void common_prefix_iter(CheckJob *job, CheckContext *context) {
+
+  assert(NodeT::kType == job->child->getType());
+  NodeT *child = static_cast<NodeT *>(job->child);
+
+  if (child->partialKeyLen > 0) {
+    int cl = std::min<int>(child->partialKeyLen,
+                           job->commonPrefixRemaining.size() - 1);
+    int i = longestCommonPrefix(child->partialKey(),
+                                job->commonPrefixRemaining.data() + 1, cl);
+    if (i != child->partialKeyLen) {
+      *job->result = finishCheckRangeRead(job->lcp, job->n, job->begin,
+                                          job->end, job->commonPrefixRemaining,
+                                          job->readVersion, context->tls)
+                         ? ConflictSet::Commit
+                         : ConflictSet::Conflict;
+      MUSTTAIL return complete(job, context);
+    }
+  }
+  job->n = job->child;
+  job->commonPrefixRemaining = job->commonPrefixRemaining.subspan(
+      1 + child->partialKeyLen,
+      job->commonPrefixRemaining.size() - (1 + child->partialKeyLen));
+
+  ++context->tls->range_read_iterations_accum;
+
+  if (job->commonPrefixRemaining.size() == 0) {
+    *job->result = finishCheckRangeRead(job->lcp, job->n, job->begin, job->end,
+                                        job->commonPrefixRemaining,
+                                        job->readVersion, context->tls)
+                       ? ConflictSet::Commit
+                       : ConflictSet::Conflict;
+    MUSTTAIL return complete(job, context);
+  }
+
+  auto c = getChild(job->n, job->commonPrefixRemaining[0]);
+  job->child = c;
+  if (job->child == nullptr) {
+    *job->result = finishCheckRangeRead(job->lcp, job->n, job->begin, job->end,
+                                        job->commonPrefixRemaining,
+                                        job->readVersion, context->tls)
+                       ? ConflictSet::Commit
+                       : ConflictSet::Conflict;
+    MUSTTAIL return complete(job, context);
+  }
+
+  job->continuation = commonPrefixIterTable[c.getType()];
+  __builtin_prefetch(job->child);
+  MUSTTAIL return keepGoing(job, context);
+}
+
 } // namespace check_range_read_state_machine
 
 void CheckJob::init(const ConflictSet::ReadRange *read,