Add to corpus

For the previous approach to work, we would have had to lock in the
generated code for every function that needs to make tail calls in
assembly.

CMakeLists.txt

@@ -95,12 +95,23 @@ target_compile_options(${PROJECT_NAME}-object PRIVATE -fno-exceptions
                                                       -fvisibility=hidden)
 target_include_directories(${PROJECT_NAME}-object
                            PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
+if(NOT LD_EXE)
+  set(LD_EXE ld)
+endif()
+add_custom_command(
+  OUTPUT ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o
+  COMMAND ${LD_EXE} -r $<TARGET_OBJECTS:${PROJECT_NAME}-object> -o
+          ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o
+  DEPENDS $<TARGET_OBJECTS:${PROJECT_NAME}-object>
+  COMMAND_EXPAND_LISTS)
 
-add_library(${PROJECT_NAME} SHARED $<TARGET_OBJECTS:${PROJECT_NAME}-object>)
+add_library(${PROJECT_NAME} SHARED ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o)
 set_target_properties(
   ${PROJECT_NAME} PROPERTIES LIBRARY_OUTPUT_DIRECTORY
                              "${CMAKE_CURRENT_BINARY_DIR}/radix_tree")
-if(NOT CMAKE_BUILD_TYPE STREQUAL Debug)
+if(CMAKE_BUILD_TYPE STREQUAL Debug)
+  set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX)
+else()
   set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE C)
 endif()
 
@@ -110,19 +121,13 @@ if(HAS_VERSION_SCRIPT)
     LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/linker.map)
 endif()
 
-add_library(${PROJECT_NAME}-static STATIC
-            $<TARGET_OBJECTS:${PROJECT_NAME}-object>)
-if(NOT CMAKE_BUILD_TYPE STREQUAL Debug)
+add_library(${PROJECT_NAME}-static STATIC ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o)
+if(CMAKE_BUILD_TYPE STREQUAL Debug)
+  set_target_properties(${PROJECT_NAME}-static PROPERTIES LINKER_LANGUAGE CXX)
+else()
   set_target_properties(${PROJECT_NAME}-static PROPERTIES LINKER_LANGUAGE C)
 endif()
-
-if(APPLE)
-  add_custom_command(
-    TARGET ${PROJECT_NAME}-static
-    PRE_LINK
-    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/privatize_symbols_macos.sh
-            $<TARGET_OBJECTS:${PROJECT_NAME}-object>)
-else()
+if(NOT APPLE)
   add_custom_command(
     TARGET ${PROJECT_NAME}-static
     POST_BUILD

ConflictSet.cpp

@@ -205,28 +205,28 @@ template <class T> struct BoundedFreeListAllocator;
 
 struct TaggedNodePointer {
   TaggedNodePointer() = default;
-  operator struct Node *() { return (struct Node *)(uintptr_t)p; }
+  operator struct Node *() { return (struct Node *)withoutType(); }
   operator struct Node0 *() {
-    assert(t == Type_Node0);
-    return (struct Node0 *)(uintptr_t)p;
+    assert(getType() == Type_Node0);
+    return (struct Node0 *)withoutType();
   }
   operator struct Node3 *() {
-    assert(t == Type_Node3);
-    return (struct Node3 *)(uintptr_t)p;
+    assert(getType() == Type_Node3);
+    return (struct Node3 *)withoutType();
   }
   operator struct Node16 *() {
-    assert(t == Type_Node16);
-    return (struct Node16 *)(uintptr_t)p;
+    assert(getType() == Type_Node16);
+    return (struct Node16 *)withoutType();
   }
   operator struct Node48 *() {
-    assert(t == Type_Node48);
-    return (struct Node48 *)(uintptr_t)p;
+    assert(getType() == Type_Node48);
+    return (struct Node48 *)withoutType();
   }
   operator struct Node256 *() {
-    assert(t == Type_Node256);
-    return (struct Node256 *)(uintptr_t)p;
+    assert(getType() == Type_Node256);
+    return (struct Node256 *)withoutType();
   }
-  /*implicit*/ TaggedNodePointer(std::nullptr_t) : p(0), t(Type_Node0) {}
+  /*implicit*/ TaggedNodePointer(std::nullptr_t) : p(0) {}
   /*implicit*/ TaggedNodePointer(Node0 *x)
       : TaggedNodePointer((struct Node *)x, Type_Node0) {}
   /*implicit*/ TaggedNodePointer(Node3 *x)
@@ -238,23 +238,25 @@ struct TaggedNodePointer {
   /*implicit*/ TaggedNodePointer(Node256 *x)
       : TaggedNodePointer((struct Node *)x, Type_Node256) {}
 
-  bool operator!=(std::nullptr_t) { return p != 0 || t != Type_Node0; }
-  bool operator==(std::nullptr_t) { return p == 0 && t == Type_Node0; }
+  bool operator!=(std::nullptr_t) { return p != 0; }
+  bool operator==(std::nullptr_t) { return p == 0; }
   bool operator==(const TaggedNodePointer &) const = default;
-  bool operator==(Node *n) const { return (uintptr_t)n == p; }
-  Node *operator->() { return (Node *)(uintptr_t)p; }
-  Type getType() { return t; }
+  bool operator==(Node *n) const { return (uintptr_t)n == withoutType(); }
+  Node *operator->() { return (Node *)withoutType(); }
+  Type getType();
 
   TaggedNodePointer(const TaggedNodePointer &) = default;
   TaggedNodePointer &operator=(const TaggedNodePointer &) = default;
   /*implicit*/ TaggedNodePointer(Node *n);
 
 private:
-  TaggedNodePointer(struct Node *p, Type t) : p((uintptr_t)p), t(t) {
+  TaggedNodePointer(struct Node *p, Type t) : p((uintptr_t)p) {
+    assert((this->p & 7) == 0);
+    this->p |= t;
     assume(p != 0);
   }
-  uintptr_t p : 56;
-  Type t : 8;
+  uintptr_t withoutType() const { return p & ~uintptr_t(7); }
+  uintptr_t p;
 };
 
 struct Node {
@@ -285,6 +287,11 @@ private:
 TaggedNodePointer::TaggedNodePointer(Node *n)
     : TaggedNodePointer(n, n->getType()) {}
 
+Type TaggedNodePointer::getType() {
+  assert(p != 0);
+  return Type(p & uintptr_t(7));
+}
+
 constexpr int kNodeCopyBegin = offsetof(Node, entry);
 constexpr int kNodeCopySize =
     offsetof(Node, parentsIndex) + sizeof(Node::parentsIndex) - kNodeCopyBegin;
@@ -1097,22 +1104,24 @@ void setMaxVersion(Node *n, InternalVersionT newMax) {
 
 TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *);
 
-Node *getChild(Node0 *, uint8_t) { return nullptr; }
-Node *getChild(Node3 *self, uint8_t index) {
+TaggedNodePointer getChild(Node0 *, uint8_t) { return nullptr; }
+TaggedNodePointer getChild(Node3 *self, uint8_t index) {
   int i = getNodeIndex(self, index);
   return i < 0 ? nullptr : self->children[i];
 }
-Node *getChild(Node16 *self, uint8_t index) {
+TaggedNodePointer getChild(Node16 *self, uint8_t index) {
   int i = getNodeIndex(self, index);
   return i < 0 ? nullptr : self->children[i];
 }
-Node *getChild(Node48 *self, uint8_t index) {
+TaggedNodePointer getChild(Node48 *self, uint8_t index) {
   int i = self->index[index];
   return i < 0 ? nullptr : self->children[i];
 }
-Node *getChild(Node256 *self, uint8_t index) { return self->children[index]; }
+TaggedNodePointer getChild(Node256 *self, uint8_t index) {
+  return self->children[index];
+}
 
-Node *getChild(Node *self, uint8_t index) {
+TaggedNodePointer getChild(Node *self, uint8_t index) {
   switch (self->getType()) {
   case Type_Node0:
     return getChild(static_cast<Node0 *>(self), index);
@@ -1130,9 +1139,8 @@ Node *getChild(Node *self, uint8_t index) {
 }
 
 struct ChildAndMaxVersion {
-  Node *child;
+  TaggedNodePointer child;
   InternalVersionT maxVersion;
-  Type childType;
 };
 
 ChildAndMaxVersion getChildAndMaxVersion(Node0 *, uint8_t) { return {}; }
@@ -1141,28 +1149,24 @@ ChildAndMaxVersion getChildAndMaxVersion(Node3 *self, uint8_t index) {
   if (i < 0) {
     return {};
   }
-  return {self->children[i], self->childMaxVersion[i],
-          self->children[i].getType()};
+  return {self->children[i], self->childMaxVersion[i]};
 }
 ChildAndMaxVersion getChildAndMaxVersion(Node16 *self, uint8_t index) {
   int i = getNodeIndex(self, index);
   if (i < 0) {
     return {};
   }
-  return {self->children[i], self->childMaxVersion[i],
-          self->children[i].getType()};
+  return {self->children[i], self->childMaxVersion[i]};
 }
 ChildAndMaxVersion getChildAndMaxVersion(Node48 *self, uint8_t index) {
   int i = self->index[index];
   if (i < 0) {
     return {};
   }
-  return {self->children[i], self->childMaxVersion[i],
-          self->children[i].getType()};
+  return {self->children[i], self->childMaxVersion[i]};
 }
 ChildAndMaxVersion getChildAndMaxVersion(Node256 *self, uint8_t index) {
-  return {self->children[index], self->childMaxVersion[index],
-          self->children[index].getType()};
+  return {self->children[index], self->childMaxVersion[index]};
 }
 
 ChildAndMaxVersion getChildAndMaxVersion(Node *self, uint8_t index) {
@@ -1291,9 +1295,11 @@ Node *getFirstChildExists(Node256 *self) {
 
 // Precondition: self has a child
 Node *getFirstChildExists(Node *self) {
+  // Only require that the node-specific overloads are covered
+  // GCOVR_EXCL_START
   switch (self->getType()) {
-  case Type_Node0:           // GCOVR_EXCL_LINE
-    __builtin_unreachable(); // GCOVR_EXCL_LINE
+  case Type_Node0:
+    __builtin_unreachable();
   case Type_Node3:
     return getFirstChildExists(static_cast<Node3 *>(self));
   case Type_Node16:
@@ -1302,9 +1308,10 @@ Node *getFirstChildExists(Node *self) {
     return getFirstChildExists(static_cast<Node48 *>(self));
   case Type_Node256:
     return getFirstChildExists(static_cast<Node256 *>(self));
-  default:                   // GCOVR_EXCL_LINE
-    __builtin_unreachable(); // GCOVR_EXCL_LINE
+  default:
+    __builtin_unreachable();
   }
+  // GCOVR_EXCL_STOP
 }
 
 void consumePartialKeyFull(TaggedNodePointer &self,
@@ -1959,153 +1966,6 @@ Node *nextSibling(Node *node) {
   }
 }
 
-// Logically this is the same as performing firstGeq and then checking against
-// point or range version according to cmp, but this version short circuits as
-// soon as it can prove that there's no conflict.
-bool checkPointRead(Node *n, const std::span<const uint8_t> key,
-                    InternalVersionT readVersion, ReadContext *tls) {
-  ++tls->point_read_accum;
-#if DEBUG_VERBOSE && !defined(NDEBUG)
-  fprintf(stderr, "Check point read: %s\n", printable(key).c_str());
-#endif
-  auto remaining = key;
-  for (;; ++tls->point_read_iterations_accum) {
-    if (remaining.size() == 0) {
-      if (n->entryPresent) {
-        return n->entry.pointVersion <= readVersion;
-      }
-      n = getFirstChildExists(n);
-      goto downLeftSpine;
-    }
-
-    auto [child, maxV, childT] = getChildAndMaxVersion(n, remaining[0]);
-    if (child == nullptr) {
-      auto c = getChildGeq(n, remaining[0]);
-      if (c != nullptr) {
-        n = c;
-        goto downLeftSpine;
-      } else {
-        n = nextSibling(n);
-        if (n == nullptr) {
-          return true;
-        }
-        goto downLeftSpine;
-      }
-    }
-
-    n = child;
-    remaining = remaining.subspan(1, remaining.size() - 1);
-
-    if (n->partialKeyLen > 0) {
-      int commonLen = std::min<int>(n->partialKeyLen, remaining.size());
-      int i = longestCommonPrefix(n->partialKey(), remaining.data(), commonLen);
-      if (i < commonLen) {
-        auto c = n->partialKey()[i] <=> remaining[i];
-        if (c > 0) {
-          goto downLeftSpine;
-        } else {
-          n = nextSibling(n);
-          if (n == nullptr) {
-            return true;
-          }
-          goto downLeftSpine;
-        }
-      }
-      if (commonLen == n->partialKeyLen) {
-        // partial key matches
-        remaining = remaining.subspan(commonLen, remaining.size() - commonLen);
-      } else if (n->partialKeyLen > int(remaining.size())) {
-        // n is the first physical node greater than remaining, and there's no
-        // eq node
-        goto downLeftSpine;
-      }
-    }
-
-    if (maxV <= readVersion) {
-      ++tls->point_read_short_circuit_accum;
-      return true;
-    }
-  }
-downLeftSpine:
-  for (; !n->entryPresent; n = getFirstChildExists(n)) {
-  }
-  return n->entry.rangeVersion <= readVersion;
-}
-
-// Logically this is the same as performing firstGeq and then checking against
-// max version or range version if this prefix doesn't exist, but this version
-// short circuits as soon as it can prove that there's no conflict.
-bool checkPrefixRead(Node *n, const std::span<const uint8_t> key,
-                     InternalVersionT readVersion, ReadContext *tls) {
-  ++tls->prefix_read_accum;
-#if DEBUG_VERBOSE && !defined(NDEBUG)
-  fprintf(stderr, "Check prefix read: %s\n", printable(key).c_str());
-#endif
-  auto remaining = key;
-  for (;; ++tls->prefix_read_iterations_accum) {
-    if (remaining.size() == 0) {
-      // There's no way to encode a prefix read of "", so n is not the root
-      return maxVersion(n) <= readVersion;
-    }
-
-    auto [child, maxV, childT] = getChildAndMaxVersion(n, remaining[0]);
-    if (child == nullptr) {
-      auto c = getChildGeq(n, remaining[0]);
-      if (c != nullptr) {
-        n = c;
-        goto downLeftSpine;
-      } else {
-        n = nextSibling(n);
-        if (n == nullptr) {
-          return true;
-        }
-        goto downLeftSpine;
-      }
-    }
-
-    n = child;
-    remaining = remaining.subspan(1, remaining.size() - 1);
-
-    if (n->partialKeyLen > 0) {
-      int commonLen = std::min<int>(n->partialKeyLen, remaining.size());
-      int i = longestCommonPrefix(n->partialKey(), remaining.data(), commonLen);
-      if (i < commonLen) {
-        auto c = n->partialKey()[i] <=> remaining[i];
-        if (c > 0) {
-          goto downLeftSpine;
-        } else {
-          n = nextSibling(n);
-          if (n == nullptr) {
-            return true;
-          }
-          goto downLeftSpine;
-        }
-      }
-      if (commonLen == n->partialKeyLen) {
-        // partial key matches
-        remaining = remaining.subspan(commonLen, remaining.size() - commonLen);
-      } else if (n->partialKeyLen > int(remaining.size())) {
-        // n is the first physical node greater than remaining, and there's no
-        // eq node. All physical nodes that start with prefix are reachable from
-        // n.
-        if (maxVersion(n) > readVersion) {
-          return false;
-        }
-        goto downLeftSpine;
-      }
-    }
-
-    if (maxV <= readVersion) {
-      ++tls->prefix_read_short_circuit_accum;
-      return true;
-    }
-  }
-downLeftSpine:
-  for (; !n->entryPresent; n = getFirstChildExists(n)) {
-  }
-  return n->entry.rangeVersion <= readVersion;
-}
-
 #ifdef HAS_AVX
 uint32_t compare16(const InternalVersionT *vs, InternalVersionT rv) {
 #if USE_64_BIT
@@ -2356,7 +2216,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
     if (!mask) {
       return true;
     }
-    auto *child = self->children[std::countr_zero(mask) >> 2];
+    Node *child = self->children[std::countr_zero(mask) >> 2];
     const bool firstRangeOk =
         !child->entryPresent || child->entry.rangeVersion <= readVersion;
 
@@ -2424,7 +2284,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
     if (!mask) {
       return true;
     }
-    auto *child = self->children[std::countr_zero(mask)];
+    Node *child = self->children[std::countr_zero(mask)];
     const bool firstRangeOk =
         !child->entryPresent || child->entry.rangeVersion <= readVersion;
     uint32_t compared = 0;
@@ -2575,7 +2435,7 @@ bool checkRangeStartsWith(Node *n, std::span<const uint8_t> key, int begin,
     return checkMaxBetweenExclusive(n, begin, end, readVersion, tls);
   }
 
-  auto *child = getChild(n, remaining[0]);
+  Node *child = getChild(n, remaining[0]);
   if (child == nullptr) {
     auto c = getChildGeq(n, remaining[0]);
     if (c != nullptr) {
@@ -2647,7 +2507,8 @@ bool checkRangeLeftSide(Node *n, std::span<const uint8_t> key, int prefixLen,
       }
     }
 
-    auto [child, maxV, childT] = getChildAndMaxVersion(n, remaining[0]);
+    auto [c, maxV] = getChildAndMaxVersion(n, remaining[0]);
+    Node *child = c;
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {
@@ -2741,7 +2602,7 @@ bool checkRangeRightSide(Node *n, std::span<const uint8_t> key, int prefixLen,
       return false;
     }
 
-    auto *child = getChild(n, remaining[0]);
+    Node *child = getChild(n, remaining[0]);
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {
@@ -2808,20 +2669,9 @@ downLeftSpine:
 }
 } // namespace
 
-bool checkRangeRead(Node *n, std::span<const uint8_t> begin,
+bool checkRangeRead(int lcp, Node *n, std::span<const uint8_t> begin,
                     std::span<const uint8_t> end, InternalVersionT readVersion,
                     ReadContext *tls) {
-  int lcp = longestCommonPrefix(begin.data(), end.data(),
-                                std::min(begin.size(), end.size()));
-  if (lcp == int(begin.size()) && end.size() == begin.size() + 1 &&
-      end.back() == 0) {
-    return checkPointRead(n, begin, readVersion, tls);
-  }
-  if (lcp == int(begin.size() - 1) && end.size() == begin.size() &&
-      int(begin.back()) + 1 == int(end.back())) {
-    return checkPrefixRead(n, begin, readVersion, tls);
-  }
-
   ++tls->range_read_accum;
 
   auto remaining = begin.subspan(0, lcp);
@@ -2835,7 +2685,8 @@ bool checkRangeRead(Node *n, std::span<const uint8_t> begin,
     if (remaining.size() == 0) {
       break;
     }
-    auto [child, v, childT] = getChildAndMaxVersion(n, remaining[0]);
+    auto [c, v] = getChildAndMaxVersion(n, remaining[0]);
+    Node *child = c;
     if (child == nullptr) {
       break;
     }
@@ -3154,7 +3005,7 @@ Node *firstGeqPhysical(Node *n, const std::span<const uint8_t> key) {
       return n;
     }
 
-    auto *child = getChild(n, remaining[0]);
+    Node *child = getChild(n, remaining[0]);
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {
@@ -3200,34 +3051,467 @@ Node *firstGeqPhysical(Node *n, const std::span<const uint8_t> key) {
   }
 }
 
+#ifndef __has_attribute
+#define __has_attribute(x) 0
+#endif
+
+#if __has_attribute(musttail)
+#define MUSTTAIL __attribute__((musttail))
+#else
+#define MUSTTAIL
+#endif
+
+#if __has_attribute(preserve_none)
+#define PRESERVE_NONE __attribute__((preserve_none))
+#else
+#define PRESERVE_NONE
+#endif
+
+#if __has_attribute(flatten)
+#define FLATTEN __attribute__((flatten))
+#else
+#define FLATTEN
+#endif
+
+typedef PRESERVE_NONE void (*Continuation)(struct CheckJob *,
+                                           struct CheckContext *);
+
+// State relevant to an individual query
+struct CheckJob {
+  void setResult(bool ok) {
+    *result = ok ? ConflictSet::Commit : ConflictSet::Conflict;
+  }
+
+  void init(const ConflictSet::ReadRange *read, ConflictSet::Result *result,
+            Node *root, int64_t oldestVersionFullPrecision);
+
+  Node *n;
+  std::span<const uint8_t> begin;
+  std::span<const uint8_t> end; // range read only
+  InternalVersionT readVersion;
+  ConflictSet::Result *result;
+  Continuation continuation;
+  CheckJob *prev;
+  CheckJob *next;
+};
+
+// State relevant to every query
+struct CheckContext {
+  int count;
+  int64_t oldestVersionFullPrecision;
+  Node *root;
+  const ConflictSet::ReadRange *queries;
+  ConflictSet::Result *results;
+  int64_t started;
+  ReadContext *tls;
+#if !__has_attribute(musttail)
+  CheckJob *job;
+  bool done;
+#endif
+};
+
+FLATTEN PRESERVE_NONE void keepGoing(CheckJob *job, CheckContext *context) {
+#if __has_attribute(musttail)
+  job = job->next;
+  MUSTTAIL return job->continuation(job, context);
+#else
+  context->job = job->next;
+  return;
+#endif
+}
+
+FLATTEN PRESERVE_NONE void complete(CheckJob *job, CheckContext *context) {
+  if (context->started == context->count) {
+    if (job->prev == job) {
+#if !__has_attribute(musttail)
+      context->done = true;
+#endif
+      return;
+    }
+    job->prev->next = job->next;
+    job->next->prev = job->prev;
+    job = job->prev;
+  } else {
+    int temp = context->started++;
+    job->init(context->queries + temp, context->results + temp, context->root,
+              context->oldestVersionFullPrecision);
+  }
+  MUSTTAIL return keepGoing(job, context);
+}
+
+namespace check_point_read_state_machine {
+
+FLATTEN PRESERVE_NONE void begin(CheckJob *, CheckContext *);
+
+template <class NodeT>
+FLATTEN PRESERVE_NONE void iter(CheckJob *, CheckContext *);
+
+FLATTEN PRESERVE_NONE void down_left_spine(CheckJob *, CheckContext *);
+
+static Continuation iterTable[] = {iter<Node0>, iter<Node3>, iter<Node16>,
+                                   iter<Node48>, iter<Node256>};
+
+void begin(CheckJob *job, CheckContext *context) {
+  ++context->tls->point_read_accum;
+#if DEBUG_VERBOSE && !defined(NDEBUG)
+  fprintf(stderr, "Check point read: %s\n", printable(key).c_str());
+#endif
+
+  if (job->begin.size() == 0) [[unlikely]] {
+    // We don't erase the root
+    assert(job->n->entryPresent);
+    job->setResult(job->n->entry.pointVersion <= job->readVersion);
+    MUSTTAIL return complete(job, context);
+  }
+
+  auto taggedChild = getChild(job->n, job->begin[0]);
+  Node *child = taggedChild;
+  if (child == nullptr) [[unlikely]] {
+    auto c = getChildGeq(job->n, job->begin[0]);
+    if (c != nullptr) {
+      job->n = c;
+      job->continuation = down_left_spine;
+      __builtin_prefetch(job->n);
+      MUSTTAIL return keepGoing(job, context);
+    } else {
+      // The root never has a next sibling
+      job->setResult(true);
+      MUSTTAIL return complete(job, context);
+    }
+  }
+  job->continuation = iterTable[taggedChild.getType()];
+  job->n = child;
+  __builtin_prefetch(child);
+  MUSTTAIL return keepGoing(job, context);
+}
+
+template <class NodeT> void iter(CheckJob *job, CheckContext *context) {
+
+  assert(NodeT::kType == job->n->getType());
+  NodeT *n = static_cast<NodeT *>(job->n);
+  job->begin = job->begin.subspan(1, job->begin.size() - 1);
+
+  if (n->partialKeyLen > 0) {
+    int commonLen = std::min<int>(n->partialKeyLen, job->begin.size());
+    int i = longestCommonPrefix(n->partialKey(), job->begin.data(), commonLen);
+    if (i < commonLen) [[unlikely]] {
+      auto c = n->partialKey()[i] <=> job->begin[i];
+      if (c > 0) {
+        job->continuation = down_left_spine;
+        MUSTTAIL return down_left_spine(job, context);
+      } else {
+        job->n = nextSibling(n);
+        if (job->n == nullptr) {
+          job->setResult(true);
+          MUSTTAIL return complete(job, context);
+        }
+        job->continuation = down_left_spine;
+        __builtin_prefetch(job->n);
+        MUSTTAIL return keepGoing(job, context);
+      }
+    }
+    if (commonLen == n->partialKeyLen) {
+      // partial key matches
+      job->begin = job->begin.subspan(commonLen, job->begin.size() - commonLen);
+    } else if (n->partialKeyLen > int(job->begin.size())) [[unlikely]] {
+      // n is the first physical node greater than remaining, and there's no
+      // eq node
+      job->continuation = down_left_spine;
+      MUSTTAIL return down_left_spine(job, context);
+    }
+  }
+
+  ++context->tls->point_read_iterations_accum;
+
+  if (job->begin.size() == 0) [[unlikely]] {
+    if (n->entryPresent) {
+      job->setResult(n->entry.pointVersion <= job->readVersion);
+      MUSTTAIL return complete(job, context);
+    }
+    job->n = getFirstChildExists(n);
+    job->continuation = down_left_spine;
+    __builtin_prefetch(job->n);
+    MUSTTAIL return keepGoing(job, context);
+  }
+
+  auto taggedChild = getChild(n, job->begin[0]);
+  Node *child = taggedChild;
+  if (child == nullptr) [[unlikely]] {
+    auto c = getChildGeq(n, job->begin[0]);
+    if (c != nullptr) {
+      job->n = c;
+      job->continuation = down_left_spine;
+      __builtin_prefetch(job->n);
+      MUSTTAIL return keepGoing(job, context);
+    } else {
+      job->n = nextSibling(job->n);
+      if (job->n == nullptr) {
+        job->setResult(true);
+        MUSTTAIL return complete(job, context);
+      }
+      job->continuation = down_left_spine;
+      __builtin_prefetch(job->n);
+      MUSTTAIL return keepGoing(job, context);
+    }
+  }
+  job->continuation = iterTable[taggedChild.getType()];
+  job->n = child;
+  __builtin_prefetch(child);
+  MUSTTAIL return keepGoing(job, context);
+}
+
+void down_left_spine(CheckJob *job, CheckContext *context) {
+  if (job->n->entryPresent) {
+    job->setResult(job->n->entry.rangeVersion <= job->readVersion);
+    MUSTTAIL return complete(job, context);
+  }
+  job->n = getFirstChildExists(job->n);
+  __builtin_prefetch(job->n);
+  MUSTTAIL return keepGoing(job, context);
+}
+
+} // namespace check_point_read_state_machine
+
+namespace check_prefix_read_state_machine {
+
+FLATTEN PRESERVE_NONE void begin(CheckJob *, CheckContext *);
+
+template <class NodeT>
+FLATTEN PRESERVE_NONE void iter(CheckJob *, CheckContext *);
+
+FLATTEN PRESERVE_NONE void down_left_spine(CheckJob *, CheckContext *);
+
+static Continuation iterTable[] = {iter<Node0>, iter<Node3>, iter<Node16>,
+                                   iter<Node48>, iter<Node256>};
+
+void begin(CheckJob *job, CheckContext *context) {
+  ++context->tls->prefix_read_accum;
+#if DEBUG_VERBOSE && !defined(NDEBUG)
+  fprintf(stderr, "Check prefix read: %s\n", printable(key).c_str());
+#endif
+
+  // There's no way to encode a prefix read of ""
+  assert(job->begin.size() > 0);
+
+  auto taggedChild = getChild(job->n, job->begin[0]);
+  Node *child = taggedChild;
+  if (child == nullptr) [[unlikely]] {
+    auto c = getChildGeq(job->n, job->begin[0]);
+    if (c != nullptr) {
+      job->n = c;
+      job->continuation = down_left_spine;
+      __builtin_prefetch(job->n);
+      MUSTTAIL return keepGoing(job, context);
+    } else {
+      // The root never has a next sibling
+      job->setResult(true);
+      MUSTTAIL return complete(job, context);
+    }
+  }
+  job->continuation = iterTable[taggedChild.getType()];
+  job->n = child;
+  __builtin_prefetch(child);
+  MUSTTAIL return keepGoing(job, context);
+}
+
+template <class NodeT> void iter(CheckJob *job, CheckContext *context) {
+
+  assert(NodeT::kType == job->n->getType());
+  NodeT *n = static_cast<NodeT *>(job->n);
+  job->begin = job->begin.subspan(1, job->begin.size() - 1);
+
+  if (n->partialKeyLen > 0) {
+    int commonLen = std::min<int>(n->partialKeyLen, job->begin.size());
+    int i = longestCommonPrefix(n->partialKey(), job->begin.data(), commonLen);
+    if (i < commonLen) [[unlikely]] {
+      auto c = n->partialKey()[i] <=> job->begin[i];
+      if (c > 0) {
+        job->continuation = down_left_spine;
+        MUSTTAIL return down_left_spine(job, context);
+      } else {
+        job->n = nextSibling(n);
+        if (job->n == nullptr) {
+          job->setResult(true);
+          MUSTTAIL return complete(job, context);
+        }
+        job->continuation = down_left_spine;
+        __builtin_prefetch(job->n);
+        MUSTTAIL return keepGoing(job, context);
+      }
+    }
+    if (commonLen == n->partialKeyLen) {
+      // partial key matches
+      job->begin = job->begin.subspan(commonLen, job->begin.size() - commonLen);
+    } else if (n->partialKeyLen > int(job->begin.size())) [[unlikely]] {
+      // n is the first physical node greater than remaining, and there's no
+      // eq node. All physical nodes that start with prefix are reachable from
+      // n.
+      if (maxVersion(n) > job->readVersion) {
+        job->setResult(false);
+        MUSTTAIL return complete(job, context);
+      }
+      job->continuation = down_left_spine;
+      MUSTTAIL return down_left_spine(job, context);
+    }
+  }
+
+  ++context->tls->prefix_read_iterations_accum;
+
+  if (job->begin.size() == 0) [[unlikely]] {
+    job->setResult(maxVersion(job->n) <= job->readVersion);
+    MUSTTAIL return complete(job, context);
+  }
+
+  auto taggedChild = getChild(n, job->begin[0]);
+  Node *child = taggedChild;
+  if (child == nullptr) [[unlikely]] {
+    auto c = getChildGeq(n, job->begin[0]);
+    if (c != nullptr) {
+      job->n = c;
+      job->continuation = down_left_spine;
+      __builtin_prefetch(job->n);
+      MUSTTAIL return keepGoing(job, context);
+    } else {
+      job->n = nextSibling(job->n);
+      if (job->n == nullptr) {
+        job->setResult(true);
+        MUSTTAIL return complete(job, context);
+      }
+      job->continuation = down_left_spine;
+      __builtin_prefetch(job->n);
+      MUSTTAIL return keepGoing(job, context);
+    }
+  }
+  job->continuation = iterTable[taggedChild.getType()];
+  job->n = child;
+  __builtin_prefetch(child);
+  MUSTTAIL return keepGoing(job, context);
+}
+
+void down_left_spine(CheckJob *job, CheckContext *context) {
+  if (job->n->entryPresent) {
+    job->setResult(job->n->entry.rangeVersion <= job->readVersion);
+    MUSTTAIL return complete(job, context);
+  }
+  job->n = getFirstChildExists(job->n);
+  __builtin_prefetch(job->n);
+  MUSTTAIL return keepGoing(job, context);
+}
+
+} // namespace check_prefix_read_state_machine
+
+namespace check_range_read_state_machine {
+FLATTEN PRESERVE_NONE void begin(CheckJob *, CheckContext *);
+
+FLATTEN PRESERVE_NONE void begin(CheckJob *job, CheckContext *context) {
+  int lcp = longestCommonPrefix(job->begin.data(), job->end.data(),
+                                std::min(job->begin.size(), job->end.size()));
+  if (lcp == int(job->begin.size()) &&
+      job->end.size() == job->begin.size() + 1 && job->end.back() == 0) {
+    job->continuation = check_point_read_state_machine::begin;
+    // Call directly since we have nothing to prefetch
+    MUSTTAIL return job->continuation(job, context);
+  }
+  if (lcp == int(job->begin.size() - 1) &&
+      job->end.size() == job->begin.size() &&
+      int(job->begin.back()) + 1 == int(job->end.back())) {
+    job->continuation = check_prefix_read_state_machine::begin;
+    // Call directly since we have nothing to prefetch
+    MUSTTAIL return job->continuation(job, context);
+  }
+
+  *job->result = checkRangeRead(lcp, job->n, job->begin, job->end,
+                                job->readVersion, context->tls)
+                     ? ConflictSet::Commit
+                     : ConflictSet::Conflict;
+  return complete(job, context);
+}
+
+} // namespace check_range_read_state_machine
+
+void CheckJob::init(const ConflictSet::ReadRange *read,
+                    ConflictSet::Result *result, Node *root,
+                    int64_t oldestVersionFullPrecision) {
+  auto begin = std::span<const uint8_t>(read->begin.p, read->begin.len);
+  auto end = std::span<const uint8_t>(read->end.p, read->end.len);
+  if (read->readVersion < oldestVersionFullPrecision) [[unlikely]] {
+    *result = ConflictSet::TooOld;
+    continuation = complete;
+  } else if (end.size() == 0) {
+    this->begin = begin;
+    this->n = root;
+    this->readVersion = InternalVersionT(read->readVersion);
+    this->result = result;
+    continuation = check_point_read_state_machine::begin;
+  } else {
+    this->begin = begin;
+    this->end = end;
+    this->n = root;
+    this->readVersion = InternalVersionT(read->readVersion);
+    this->result = result;
+    continuation = check_range_read_state_machine::begin;
+  }
+}
+
 struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
 
   void check(const ReadRange *reads, Result *result, int count) {
+    assert(oldestVersionFullPrecision >=
+           newestVersionFullPrecision - kNominalVersionWindow);
+
+    if (count == 0) {
+      return;
+    }
+
     ReadContext tls;
     tls.impl = this;
     int64_t check_byte_accum = 0;
+    constexpr int kConcurrent = 16;
+    CheckJob inProgress[kConcurrent];
+    CheckContext context;
+    context.count = count;
+    context.oldestVersionFullPrecision = oldestVersionFullPrecision;
+    context.root = root;
+    context.queries = reads;
+    context.results = result;
+    context.tls = &tls;
+    int64_t started = std::min(kConcurrent, count);
+    context.started = started;
+    for (int i = 0; i < started; i++) {
+      inProgress[i].init(reads + i, result + i, root,
+                         oldestVersionFullPrecision);
+    }
+    for (int i = 0; i < started - 1; i++) {
+      inProgress[i].next = inProgress + i + 1;
+    }
+    for (int i = 1; i < started; i++) {
+      inProgress[i].prev = inProgress + i - 1;
+    }
+    inProgress[0].prev = inProgress + started - 1;
+    inProgress[started - 1].next = inProgress;
+
+#if __has_attribute(musttail)
+    // Kick off the sequence of tail calls that finally returns once all jobs
+    // are done
+    inProgress->continuation(inProgress, &context);
+#else
+    context.job = inProgress;
+    context.done = false;
+    while (!context.done) {
+      context.job->continuation(context.job, &context);
+    }
+#endif
+
     for (int i = 0; i < count; ++i) {
       assert(reads[i].readVersion >= 0);
       assert(reads[i].readVersion <= newestVersionFullPrecision);
       const auto &r = reads[i];
       check_byte_accum += r.begin.len + r.end.len;
-      auto begin = std::span<const uint8_t>(r.begin.p, r.begin.len);
-      auto end = std::span<const uint8_t>(r.end.p, r.end.len);
-      assert(oldestVersionFullPrecision >=
-             newestVersionFullPrecision - kNominalVersionWindow);
-      result[i] =
-          reads[i].readVersion < oldestVersionFullPrecision ? TooOld
-          : (end.size() > 0
-                 ? checkRangeRead(root, begin, end,
-                                  InternalVersionT(reads[i].readVersion), &tls)
-                 : checkPointRead(root, begin,
-                                  InternalVersionT(reads[i].readVersion), &tls))
-              ? Commit
-              : Conflict;
       tls.commits_accum += result[i] == Commit;
       tls.conflicts_accum += result[i] == Conflict;
       tls.too_olds_accum += result[i] == TooOld;
     }
+
     point_read_total.add(tls.point_read_accum);
     prefix_read_total.add(tls.prefix_read_accum);
     range_read_total.add(tls.range_read_accum);
@@ -3606,7 +3890,7 @@ Node *firstGeqLogical(Node *n, const std::span<const uint8_t> key) {
       goto downLeftSpine;
     }
 
-    auto *child = getChild(n, remaining[0]);
+    Node *child = getChild(n, remaining[0]);
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {

Dockerfile

@@ -8,6 +8,7 @@ RUN chmod -R 777 /tmp
 RUN apt-get update
 RUN apt-get upgrade -y
 RUN TZ=America/Los_Angeles DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    binutils-aarch64-linux-gnu \
     build-essential \
     ccache \
     clang \

aarch64-toolchain.cmake

@@ -5,3 +5,4 @@ set(CMAKE_CXX_COMPILER "/usr/bin/aarch64-linux-gnu-g++")
 set(CMAKE_FIND_ROOT_PATH /usr/aarch64-linux-gnu)
 set(CMAKE_CROSSCOMPILING_EMULATOR "qemu-aarch64;-L;/usr/aarch64-linux-gnu/")
 set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE arm64)
+set(LD_EXE "/usr/bin/aarch64-linux-gnu-ld")