Add to corpus

CMakeLists.txt

@@ -38,6 +38,11 @@ if(NOT APPLE)
   add_compile_options(-g -fno-omit-frame-pointer)
 endif()
 
+# work around lack of musttail for gcc
+if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_BUILD_TYPE STREQUAL "Debug")
+  add_compile_options(-O1 -foptimize-sibling-calls)
+endif()
+
 set(full_relro_flags "-pie;LINKER:-z,relro,-z,now,-z,noexecstack")
 cmake_push_check_state()
 list(APPEND CMAKE_REQUIRED_LINK_OPTIONS ${full_relro_flags})

ConflictSet.cpp

@@ -203,6 +203,62 @@ enum Type : int8_t {
 
 template <class T> struct BoundedFreeListAllocator;
 
+struct TaggedNodePointer {
+  TaggedNodePointer() = default;
+  operator struct Node *() { return (struct Node *)withoutType(); }
+  operator struct Node0 *() {
+    assert(getType() == Type_Node0);
+    return (struct Node0 *)withoutType();
+  }
+  operator struct Node3 *() {
+    assert(getType() == Type_Node3);
+    return (struct Node3 *)withoutType();
+  }
+  operator struct Node16 *() {
+    assert(getType() == Type_Node16);
+    return (struct Node16 *)withoutType();
+  }
+  operator struct Node48 *() {
+    assert(getType() == Type_Node48);
+    return (struct Node48 *)withoutType();
+  }
+  operator struct Node256 *() {
+    assert(getType() == Type_Node256);
+    return (struct Node256 *)withoutType();
+  }
+  /*implicit*/ TaggedNodePointer(std::nullptr_t) : p(0) {}
+  /*implicit*/ TaggedNodePointer(Node0 *x)
+      : TaggedNodePointer((struct Node *)x, Type_Node0) {}
+  /*implicit*/ TaggedNodePointer(Node3 *x)
+      : TaggedNodePointer((struct Node *)x, Type_Node3) {}
+  /*implicit*/ TaggedNodePointer(Node16 *x)
+      : TaggedNodePointer((struct Node *)x, Type_Node16) {}
+  /*implicit*/ TaggedNodePointer(Node48 *x)
+      : TaggedNodePointer((struct Node *)x, Type_Node48) {}
+  /*implicit*/ TaggedNodePointer(Node256 *x)
+      : TaggedNodePointer((struct Node *)x, Type_Node256) {}
+
+  bool operator!=(std::nullptr_t) { return p != 0; }
+  bool operator==(std::nullptr_t) { return p == 0; }
+  bool operator==(const TaggedNodePointer &) const = default;
+  bool operator==(Node *n) const { return (uintptr_t)n == withoutType(); }
+  Node *operator->() { return (Node *)withoutType(); }
+  Type getType();
+
+  TaggedNodePointer(const TaggedNodePointer &) = default;
+  TaggedNodePointer &operator=(const TaggedNodePointer &) = default;
+  /*implicit*/ TaggedNodePointer(Node *n);
+
+private:
+  TaggedNodePointer(struct Node *p, Type t) : p((uintptr_t)p) {
+    assert((this->p & 7) == 0);
+    this->p |= t;
+    assume(p != 0);
+  }
+  uintptr_t withoutType() const { return p & ~uintptr_t(7); }
+  uintptr_t p;
+};
+
 struct Node {
 
   /* begin section that's copied to the next node */
@@ -228,6 +284,14 @@ private:
   int32_t partialKeyCapacity;
 };
 
+TaggedNodePointer::TaggedNodePointer(Node *n)
+    : TaggedNodePointer(n, n->getType()) {}
+
+Type TaggedNodePointer::getType() {
+  assert(p != 0);
+  return Type(p & uintptr_t(7));
+}
+
 constexpr int kNodeCopyBegin = offsetof(Node, entry);
 constexpr int kNodeCopySize =
     offsetof(Node, parentsIndex) + sizeof(Node::parentsIndex) - kNodeCopyBegin;
@@ -251,7 +315,7 @@ struct Node3 : Node {
   constexpr static auto kMaxNodes = 3;
   constexpr static auto kType = Type_Node3;
 
-  Node *children[kMaxNodes];
+  TaggedNodePointer children[kMaxNodes];
   InternalVersionT childMaxVersion[kMaxNodes];
   // Sorted
   uint8_t index[kMaxNodes];
@@ -267,7 +331,7 @@ struct Node16 : Node {
   constexpr static auto kType = Type_Node16;
   constexpr static auto kMaxNodes = 16;
 
-  Node *children[kMaxNodes];
+  TaggedNodePointer children[kMaxNodes];
   InternalVersionT childMaxVersion[kMaxNodes];
   // Sorted
   uint8_t index[kMaxNodes];
@@ -288,7 +352,7 @@ struct Node48 : Node {
   constexpr static int kMaxOfMaxTotalPages = kMaxNodes / kMaxOfMaxPageSize;
 
   BitSet bitSet;
-  Node *children[kMaxNodes];
+  TaggedNodePointer children[kMaxNodes];
   InternalVersionT childMaxVersion[kMaxNodes];
   InternalVersionT maxOfMax[kMaxOfMaxTotalPages];
   uint8_t reverseIndex[kMaxNodes];
@@ -310,7 +374,7 @@ struct Node256 : Node {
   constexpr static int kMaxOfMaxTotalPages = kMaxNodes / kMaxOfMaxPageSize;
 
   BitSet bitSet;
-  Node *children[kMaxNodes];
+  TaggedNodePointer children[kMaxNodes];
   InternalVersionT childMaxVersion[kMaxNodes];
   InternalVersionT maxOfMax[kMaxOfMaxTotalPages];
 
@@ -888,26 +952,26 @@ int getNodeIndexExists(Node16 *self, uint8_t index) {
 }
 
 // Precondition - an entry for index must exist in the node
-Node *&getChildExists(Node3 *self, uint8_t index) {
+TaggedNodePointer &getChildExists(Node3 *self, uint8_t index) {
   return self->children[getNodeIndexExists(self, index)];
 }
 // Precondition - an entry for index must exist in the node
-Node *&getChildExists(Node16 *self, uint8_t index) {
+TaggedNodePointer &getChildExists(Node16 *self, uint8_t index) {
   return self->children[getNodeIndexExists(self, index)];
 }
 // Precondition - an entry for index must exist in the node
-Node *&getChildExists(Node48 *self, uint8_t index) {
+TaggedNodePointer &getChildExists(Node48 *self, uint8_t index) {
   assert(self->bitSet.test(index));
   return self->children[self->index[index]];
 }
 // Precondition - an entry for index must exist in the node
-Node *&getChildExists(Node256 *self, uint8_t index) {
+TaggedNodePointer &getChildExists(Node256 *self, uint8_t index) {
   assert(self->bitSet.test(index));
   return self->children[index];
 }
 
 // Precondition - an entry for index must exist in the node
-Node *&getChildExists(Node *self, uint8_t index) {
+TaggedNodePointer &getChildExists(Node *self, uint8_t index) {
   switch (self->getType()) {
   case Type_Node0:           // GCOVR_EXCL_LINE
     __builtin_unreachable(); // GCOVR_EXCL_LINE
@@ -1038,24 +1102,26 @@ void setMaxVersion(Node *n, InternalVersionT newMax) {
   }
 }
 
-Node *&getInTree(Node *n, ConflictSet::Impl *);
+TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *);
 
-Node *getChild(Node0 *, uint8_t) { return nullptr; }
+TaggedNodePointer getChild(Node0 *, uint8_t) { return nullptr; }
-Node *getChild(Node3 *self, uint8_t index) {
+TaggedNodePointer getChild(Node3 *self, uint8_t index) {
   int i = getNodeIndex(self, index);
   return i < 0 ? nullptr : self->children[i];
 }
-Node *getChild(Node16 *self, uint8_t index) {
+TaggedNodePointer getChild(Node16 *self, uint8_t index) {
   int i = getNodeIndex(self, index);
   return i < 0 ? nullptr : self->children[i];
 }
-Node *getChild(Node48 *self, uint8_t index) {
+TaggedNodePointer getChild(Node48 *self, uint8_t index) {
   int i = self->index[index];
   return i < 0 ? nullptr : self->children[i];
 }
-Node *getChild(Node256 *self, uint8_t index) { return self->children[index]; }
+TaggedNodePointer getChild(Node256 *self, uint8_t index) {
+  return self->children[index];
+}
 
-Node *getChild(Node *self, uint8_t index) {
+TaggedNodePointer getChild(Node *self, uint8_t index) {
   switch (self->getType()) {
   case Type_Node0:
     return getChild(static_cast<Node0 *>(self), index);
@@ -1073,7 +1139,7 @@ Node *getChild(Node *self, uint8_t index) {
 }
 
 struct ChildAndMaxVersion {
-  Node *child;
+  TaggedNodePointer child;
   InternalVersionT maxVersion;
 };
 
@@ -1245,14 +1311,15 @@ Node *getFirstChildExists(Node *self) {
   }
 }
 
-void consumePartialKeyFull(Node *&self, std::span<const uint8_t> &key,
+void consumePartialKeyFull(TaggedNodePointer &self,
+                           std::span<const uint8_t> &key,
                            InternalVersionT writeVersion, WriteContext *tls) {
   // Handle an existing partial key
   int commonLen = std::min<int>(self->partialKeyLen, key.size());
   int partialKeyIndex =
       longestCommonPrefix(self->partialKey(), key.data(), commonLen);
   if (partialKeyIndex < self->partialKeyLen) {
-    auto *old = self;
+    Node *old = self;
     // Since root cannot have a partial key
     assert(old->parent != nullptr);
     InternalVersionT oldMaxVersion = exchangeMaxVersion(old, writeVersion);
@@ -1290,7 +1357,7 @@ void consumePartialKeyFull(Node *&self, std::span<const uint8_t> &key,
 // Consume any partial key of `self`, and update `self` and
 // `key` such that `self` is along the search path of `key`
 inline __attribute__((always_inline)) void
-consumePartialKey(Node *&self, std::span<const uint8_t> &key,
+consumePartialKey(TaggedNodePointer &self, std::span<const uint8_t> &key,
                   InternalVersionT writeVersion, WriteContext *tls) {
   if (self->partialKeyLen > 0) {
     consumePartialKeyFull(self, key, writeVersion, tls);
@@ -1301,8 +1368,10 @@ consumePartialKey(Node *&self, std::span<const uint8_t> &key,
 // such that the search path of the result + key is the same as the search path
 // of self + key before the call. Creates a node if necessary. Updates
 // `maxVersion` for result.
-Node *&getOrCreateChild(Node *&self, std::span<const uint8_t> &key,
+TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self,
-                        InternalVersionT newMaxVersion, WriteContext *tls) {
+                                    std::span<const uint8_t> &key,
+                                    InternalVersionT newMaxVersion,
+                                    WriteContext *tls) {
 
   int index = key.front();
   key = key.subspan(1, key.size() - 1);
@@ -1660,10 +1729,11 @@ void rezero(Node *n, InternalVersionT z) {
 }
 #endif
 
-void mergeWithChild(Node *&self, WriteContext *tls, ConflictSet::Impl *impl,
+void mergeWithChild(TaggedNodePointer &self, WriteContext *tls,
-                    Node *&dontInvalidate, Node3 *self3) {
+                    ConflictSet::Impl *impl, Node *&dontInvalidate,
+                    Node3 *self3) {
   assert(!self3->entryPresent);
-  auto *child = self3->children[0];
+  Node *child = self3->children[0];
   int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen;
 
   if (minCapacity > child->getCapacity()) {
@@ -1912,7 +1982,8 @@ bool checkPointRead(Node *n, const std::span<const uint8_t> key,
       goto downLeftSpine;
     }
 
-    auto [child, maxV] = getChildAndMaxVersion(n, remaining[0]);
+    auto [c, maxV] = getChildAndMaxVersion(n, remaining[0]);
+    Node *child = c;
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {
@@ -1982,7 +2053,8 @@ bool checkPrefixRead(Node *n, const std::span<const uint8_t> key,
       return maxVersion(n) <= readVersion;
     }
 
-    auto [child, maxV] = getChildAndMaxVersion(n, remaining[0]);
+    auto [c, maxV] = getChildAndMaxVersion(n, remaining[0]);
+    Node *child = c;
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {
@@ -2255,7 +2327,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
     if (!mask) {
       return true;
     }
-    auto *child = self->children[std::countr_zero(mask)];
+    Node *child = self->children[std::countr_zero(mask)];
     const bool firstRangeOk =
         !child->entryPresent || child->entry.rangeVersion <= readVersion;
     uint32_t compared = 0;
@@ -2330,7 +2402,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
     if (!mask) {
       return true;
     }
-    auto *child = self->children[std::countr_zero(mask)];
+    Node *child = self->children[std::countr_zero(mask)];
     const bool firstRangeOk =
         !child->entryPresent || child->entry.rangeVersion <= readVersion;
 
@@ -2375,7 +2447,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
     {
       int c = self->bitSet.firstSetGeq(begin + 1);
       if (c >= 0 && c < end) {
-        auto *child = self->children[self->index[c]];
+        Node *child = self->children[self->index[c]];
         if (child->entryPresent && child->entry.rangeVersion > readVersion) {
           return false;
         }
@@ -2409,7 +2481,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
     {
       int c = self->bitSet.firstSetGeq(begin + 1);
       if (c >= 0 && c < end) {
-        auto *child = self->children[c];
+        Node *child = self->children[c];
         if (child->entryPresent && child->entry.rangeVersion > readVersion) {
           return false;
         }
@@ -2509,7 +2581,7 @@ bool checkRangeStartsWith(Node *n, std::span<const uint8_t> key, int begin,
     return checkMaxBetweenExclusive(n, begin, end, readVersion, tls);
   }
 
-  auto *child = getChild(n, remaining[0]);
+  Node *child = getChild(n, remaining[0]);
   if (child == nullptr) {
     auto c = getChildGeq(n, remaining[0]);
     if (c != nullptr) {
@@ -2581,7 +2653,8 @@ bool checkRangeLeftSide(Node *n, std::span<const uint8_t> key, int prefixLen,
       }
     }
 
-    auto [child, maxV] = getChildAndMaxVersion(n, remaining[0]);
+    auto [c, maxV] = getChildAndMaxVersion(n, remaining[0]);
+    Node *child = c;
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {
@@ -2675,7 +2748,7 @@ bool checkRangeRightSide(Node *n, std::span<const uint8_t> key, int prefixLen,
       return false;
     }
 
-    auto *child = getChild(n, remaining[0]);
+    Node *child = getChild(n, remaining[0]);
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {
@@ -2769,7 +2842,8 @@ bool checkRangeRead(Node *n, std::span<const uint8_t> begin,
     if (remaining.size() == 0) {
       break;
     }
-    auto [child, v] = getChildAndMaxVersion(n, remaining[0]);
+    auto [c, v] = getChildAndMaxVersion(n, remaining[0]);
+    Node *child = c;
     if (child == nullptr) {
       break;
     }
@@ -2835,8 +2909,10 @@ checkMaxBetweenExclusiveImpl<true>(Node *n, int begin, int end,
 // of the result will have `maxVersion` set to `writeVersion` as a
 // postcondition. Nodes along the search path may be invalidated. Callers must
 // ensure that the max version of the self argument is updated.
-[[nodiscard]] Node **insert(Node **self, std::span<const uint8_t> key,
+[[nodiscard]] TaggedNodePointer *insert(TaggedNodePointer *self,
-                            InternalVersionT writeVersion, WriteContext *tls) {
+                                        std::span<const uint8_t> key,
+                                        InternalVersionT writeVersion,
+                                        WriteContext *tls) {
 
   for (; key.size() != 0; ++tls->accum.insert_iterations) {
     self = &getOrCreateChild(*self, key, writeVersion, tls);
@@ -2864,17 +2940,23 @@ void eraseTree(Node *root, WriteContext *tls) {
     } break;
     case Type_Node3: {
       auto *n3 = static_cast<Node3 *>(n);
-      toFree.append(std::span<Node *>(n3->children, n3->numChildren));
+      for (int i = 0; i < n3->numChildren; ++i) {
+        toFree.push_back(n3->children[i]);
+      }
       tls->release(n3);
     } break;
     case Type_Node16: {
       auto *n16 = static_cast<Node16 *>(n);
-      toFree.append(std::span<Node *>(n16->children, n16->numChildren));
+      for (int i = 0; i < n16->numChildren; ++i) {
+        toFree.push_back(n16->children[i]);
+      }
       tls->release(n16);
     } break;
     case Type_Node48: {
       auto *n48 = static_cast<Node48 *>(n);
-      toFree.append(std::span<Node *>(n48->children, n48->numChildren));
+      for (int i = 0; i < n48->numChildren; ++i) {
+        toFree.push_back(n48->children[i]);
+      }
       tls->release(n48);
     } break;
     case Type_Node256: {
@@ -2890,10 +2972,10 @@ void eraseTree(Node *root, WriteContext *tls) {
   }
 }
 
-void addPointWrite(Node *&root, std::span<const uint8_t> key,
+void addPointWrite(TaggedNodePointer &root, std::span<const uint8_t> key,
                    InternalVersionT writeVersion, WriteContext *tls) {
   ++tls->accum.point_writes;
-  auto *n = *insert(&root, key, writeVersion, tls);
+  auto n = *insert(&root, key, writeVersion, tls);
   if (!n->entryPresent) {
     ++tls->accum.entries_inserted;
     auto *p = nextLogical(n);
@@ -3011,7 +3093,7 @@ void fixupMaxVersion(Node *node, WriteContext *tls) {
   setMaxVersion(node, max);
 }
 
-void addWriteRange(Node *&root, std::span<const uint8_t> begin,
+void addWriteRange(TaggedNodePointer &root, std::span<const uint8_t> begin,
                    std::span<const uint8_t> end, InternalVersionT writeVersion,
                    WriteContext *tls, ConflictSet::Impl *impl) {
 
@@ -3024,12 +3106,12 @@ void addWriteRange(Node *&root, std::span<const uint8_t> begin,
   ++tls->accum.range_writes;
   const bool beginIsPrefix = lcp == int(begin.size());
 
-  Node **useAsRoot = insert(&root, begin.subspan(0, lcp), writeVersion, tls);
+  auto useAsRoot = insert(&root, begin.subspan(0, lcp), writeVersion, tls);
 
   begin = begin.subspan(lcp, begin.size() - lcp);
   end = end.subspan(lcp, end.size() - lcp);
 
-  auto *beginNode = *insert(useAsRoot, begin, writeVersion, tls);
+  Node *beginNode = *insert(useAsRoot, begin, writeVersion, tls);
   addKey(beginNode);
   if (!beginNode->entryPresent) {
     ++tls->accum.entries_inserted;
@@ -3040,7 +3122,7 @@ void addWriteRange(Node *&root, std::span<const uint8_t> begin,
   }
   beginNode->entry.pointVersion = writeVersion;
 
-  auto *endNode = *insert(useAsRoot, end, writeVersion, tls);
+  Node *endNode = *insert(useAsRoot, end, writeVersion, tls);
   addKey(endNode);
   if (!endNode->entryPresent) {
     ++tls->accum.entries_inserted;
@@ -3060,7 +3142,7 @@ void addWriteRange(Node *&root, std::span<const uint8_t> begin,
   assert(!beginNode->endOfRange);
   assert(!endNode->endOfRange);
   endNode->endOfRange = true;
-  auto *iter = beginNode;
+  Node *iter = beginNode;
   for (iter = nextLogical(iter); !iter->endOfRange;
        iter = erase(iter, tls, impl, /*logical*/ true)) {
     assert(!iter->endOfRange);
@@ -3080,7 +3162,7 @@ Node *firstGeqPhysical(Node *n, const std::span<const uint8_t> key) {
       return n;
     }
 
-    auto *child = getChild(n, remaining[0]);
+    Node *child = getChild(n, remaining[0]);
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {
@@ -3126,34 +3208,288 @@ Node *firstGeqPhysical(Node *n, const std::span<const uint8_t> key) {
   }
 }
 
+#ifndef __has_attribute
+#define __has_attribute(x) 0
+#endif
+
+#if __has_attribute(musttail)
+#define MUSTTAIL __attribute__((musttail))
+#else
+#define MUSTTAIL
+#endif
+
+#if __has_attribute(preserve_none)
+#define PRESERVE_NONE __attribute__((preserve_none))
+#else
+#define PRESERVE_NONE
+#endif
+
+#if __has_attribute(flatten)
+#define FLATTEN __attribute__((flatten))
+#else
+#define FLATTEN
+#endif
+
+typedef PRESERVE_NONE void (*Continuation)(struct CheckJob *,
+                                           struct CheckContext *);
+
+// State relevant to an individual query
+struct CheckJob {
+  void setResult(bool ok) {
+    *result = ok ? ConflictSet::Commit : ConflictSet::Conflict;
+  }
+
+  void init(const ConflictSet::ReadRange *read, ConflictSet::Result *result,
+            Node *root, int64_t oldestVersionFullPrecision, ReadContext *tls);
+
+  Node *n;
+  std::span<const uint8_t> begin;
+  InternalVersionT readVersion;
+  ConflictSet::Result *result;
+  Continuation continuation;
+  CheckJob *prev;
+  CheckJob *next;
+};
+
+// State relevant to every query
+struct CheckContext {
+  int count;
+  int64_t oldestVersionFullPrecision;
+  Node *root;
+  const ConflictSet::ReadRange *queries;
+  ConflictSet::Result *results;
+  int64_t started;
+  ReadContext *tls;
+};
+
+FLATTEN PRESERVE_NONE void keepGoing(CheckJob *job, CheckContext *context) {
+  job = job->next;
+  MUSTTAIL return job->continuation(job, context);
+}
+
+FLATTEN PRESERVE_NONE void complete(CheckJob *job, CheckContext *context) {
+  if (context->started == context->count) {
+    if (job->prev == job) {
+      return;
+    }
+    job->prev->next = job->next;
+    job->next->prev = job->prev;
+    job = job->prev;
+  } else {
+    int temp = context->started++;
+    job->init(context->queries + temp, context->results + temp, context->root,
+              context->oldestVersionFullPrecision, context->tls);
+  }
+  MUSTTAIL return keepGoing(job, context);
+}
+
+namespace check_point_read_state_machine {
+
+FLATTEN PRESERVE_NONE void begin(CheckJob *, CheckContext *);
+
+template <class NodeT>
+FLATTEN PRESERVE_NONE void iter(CheckJob *, CheckContext *);
+
+FLATTEN PRESERVE_NONE void down_left_spine(CheckJob *, CheckContext *);
+
+static Continuation iterTable[] = {iter<Node0>, iter<Node3>, iter<Node16>,
+                                   iter<Node48>, iter<Node256>};
+
+void begin(CheckJob *job, CheckContext *context) {
+  ++context->tls->point_read_accum;
+#if DEBUG_VERBOSE && !defined(NDEBUG)
+  fprintf(stderr, "Check point read: %s\n", printable(key).c_str());
+#endif
+
+  if (job->begin.size() == 0) [[unlikely]] {
+    // We don't erase the root
+    assert(job->n->entryPresent);
+    job->setResult(job->n->entry.pointVersion <= job->readVersion);
+    MUSTTAIL return complete(job, context);
+  }
+
+  auto taggedChild = getChild(job->n, job->begin[0]);
+  Node *child = taggedChild;
+  if (child == nullptr) [[unlikely]] {
+    auto c = getChildGeq(job->n, job->begin[0]);
+    if (c != nullptr) {
+      job->n = c;
+      job->continuation = down_left_spine;
+      __builtin_prefetch(job->n);
+      MUSTTAIL return keepGoing(job, context);
+    } else {
+      // The root never has a next sibling
+      job->setResult(true);
+      MUSTTAIL return complete(job, context);
+    }
+  }
+  job->continuation = iterTable[taggedChild.getType()];
+  job->n = child;
+  __builtin_prefetch(child);
+  MUSTTAIL return keepGoing(job, context);
+}
+
+template <class NodeT> void iter(CheckJob *job, CheckContext *context) {
+
+  assert(NodeT::kType == job->n->getType());
+  NodeT *n = static_cast<NodeT *>(job->n);
+  job->begin = job->begin.subspan(1, job->begin.size() - 1);
+
+  if (n->partialKeyLen > 0) {
+    int commonLen = std::min<int>(n->partialKeyLen, job->begin.size());
+    int i = longestCommonPrefix(n->partialKey(), job->begin.data(), commonLen);
+    if (i < commonLen) [[unlikely]] {
+      auto c = n->partialKey()[i] <=> job->begin[i];
+      if (c > 0) {
+        job->continuation = down_left_spine;
+        MUSTTAIL return down_left_spine(job, context);
+      } else {
+        job->n = nextSibling(n);
+        if (job->n == nullptr) {
+          job->setResult(true);
+          MUSTTAIL return complete(job, context);
+        }
+        job->continuation = down_left_spine;
+        __builtin_prefetch(job->n);
+        MUSTTAIL return keepGoing(job, context);
+      }
+    }
+    if (commonLen == n->partialKeyLen) {
+      // partial key matches
+      job->begin = job->begin.subspan(commonLen, job->begin.size() - commonLen);
+    } else if (n->partialKeyLen > int(job->begin.size())) [[unlikely]] {
+      // n is the first physical node greater than remaining, and there's no
+      // eq node
+      job->continuation = down_left_spine;
+      MUSTTAIL return down_left_spine(job, context);
+    }
+  }
+
+  ++context->tls->point_read_iterations_accum;
+
+  if (job->begin.size() == 0) [[unlikely]] {
+    if (n->entryPresent) {
+      job->setResult(n->entry.pointVersion <= job->readVersion);
+      MUSTTAIL return complete(job, context);
+    }
+    job->n = getFirstChildExists(n);
+    job->continuation = down_left_spine;
+    __builtin_prefetch(job->n);
+    MUSTTAIL return keepGoing(job, context);
+  }
+
+  auto taggedChild = getChild(n, job->begin[0]);
+  Node *child = taggedChild;
+  if (child == nullptr) [[unlikely]] {
+    auto c = getChildGeq(n, job->begin[0]);
+    if (c != nullptr) {
+      job->n = c;
+      job->continuation = down_left_spine;
+      __builtin_prefetch(job->n);
+      MUSTTAIL return keepGoing(job, context);
+    } else {
+      job->n = nextSibling(job->n);
+      if (job->n == nullptr) {
+        job->setResult(true);
+        MUSTTAIL return complete(job, context);
+      }
+      job->continuation = down_left_spine;
+      __builtin_prefetch(job->n);
+      MUSTTAIL return keepGoing(job, context);
+    }
+  }
+  job->continuation = iterTable[taggedChild.getType()];
+  job->n = child;
+  __builtin_prefetch(child);
+  MUSTTAIL return keepGoing(job, context);
+}
+
+void down_left_spine(CheckJob *job, CheckContext *context) {
+  if (job->n->entryPresent) {
+    job->setResult(job->n->entry.rangeVersion <= job->readVersion);
+    MUSTTAIL return complete(job, context);
+  }
+  job->n = getFirstChildExists(job->n);
+  __builtin_prefetch(job->n);
+  MUSTTAIL return keepGoing(job, context);
+}
+
+} // namespace check_point_read_state_machine
+
+void CheckJob::init(const ConflictSet::ReadRange *read,
+                    ConflictSet::Result *result, Node *root,
+                    int64_t oldestVersionFullPrecision, ReadContext *tls) {
+  auto begin = std::span<const uint8_t>(read->begin.p, read->begin.len);
+  auto end = std::span<const uint8_t>(read->end.p, read->end.len);
+  if (read->readVersion < oldestVersionFullPrecision) [[unlikely]] {
+    *result = ConflictSet::TooOld;
+    continuation = complete;
+  } else if (end.size() == 0) {
+    this->begin = begin;
+    this->n = root;
+    this->readVersion = InternalVersionT(read->readVersion);
+    this->result = result;
+    continuation = check_point_read_state_machine::begin;
+  } else {
+    *result = checkRangeRead(root, begin, end,
+                             InternalVersionT(read->readVersion), tls)
+                  ? ConflictSet::Commit
+                  : ConflictSet::Conflict;
+    continuation = complete;
+  }
+}
+
 struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
 
   void check(const ReadRange *reads, Result *result, int count) {
+    assert(oldestVersionFullPrecision >=
+           newestVersionFullPrecision - kNominalVersionWindow);
+
+    if (count == 0) {
+      return;
+    }
+
     ReadContext tls;
     tls.impl = this;
     int64_t check_byte_accum = 0;
+    constexpr int kConcurrent = 16;
+    CheckJob inProgress[kConcurrent];
+    CheckContext context;
+    context.count = count;
+    context.oldestVersionFullPrecision = oldestVersionFullPrecision;
+    context.root = root;
+    context.queries = reads;
+    context.results = result;
+    context.tls = &tls;
+    int64_t started = std::min(kConcurrent, count);
+    context.started = started;
+    for (int i = 0; i < started; i++) {
+      inProgress[i].init(reads + i, result + i, root,
+                         oldestVersionFullPrecision, &tls);
+    }
+    for (int i = 0; i < started - 1; i++) {
+      inProgress[i].next = inProgress + i + 1;
+    }
+    for (int i = 1; i < started; i++) {
+      inProgress[i].prev = inProgress + i - 1;
+    }
+    inProgress[0].prev = inProgress + started - 1;
+    inProgress[started - 1].next = inProgress;
+
+    // Kick off the sequence of tail calls that finally returns once all jobs
+    // are done
+    inProgress->continuation(inProgress, &context);
+
     for (int i = 0; i < count; ++i) {
       assert(reads[i].readVersion >= 0);
       assert(reads[i].readVersion <= newestVersionFullPrecision);
       const auto &r = reads[i];
       check_byte_accum += r.begin.len + r.end.len;
-      auto begin = std::span<const uint8_t>(r.begin.p, r.begin.len);
-      auto end = std::span<const uint8_t>(r.end.p, r.end.len);
-      assert(oldestVersionFullPrecision >=
-             newestVersionFullPrecision - kNominalVersionWindow);
-      result[i] =
-          reads[i].readVersion < oldestVersionFullPrecision ? TooOld
-          : (end.size() > 0
-                 ? checkRangeRead(root, begin, end,
-                                  InternalVersionT(reads[i].readVersion), &tls)
-                 : checkPointRead(root, begin,
-                                  InternalVersionT(reads[i].readVersion), &tls))
-              ? Commit
-              : Conflict;
       tls.commits_accum += result[i] == Commit;
       tls.conflicts_accum += result[i] == Conflict;
       tls.too_olds_accum += result[i] == TooOld;
     }
+
     point_read_total.add(tls.point_read_accum);
     prefix_read_total.add(tls.prefix_read_accum);
     range_read_total.add(tls.range_read_accum);
@@ -3375,7 +3711,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
   std::span<const uint8_t> removalKey;
   int64_t keyUpdates;
 
-  Node *root;
+  TaggedNodePointer root;
   InternalVersionT oldestVersion;
   int64_t oldestVersionFullPrecision;
   int64_t oldestExtantVersion;
@@ -3456,7 +3792,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
   }
 };
 
-Node *&getInTree(Node *n, ConflictSet::Impl *impl) {
+TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *impl) {
   return n->parent == nullptr ? impl->root
                               : getChildExists(n->parent, n->parentsIndex);
 }
@@ -3532,7 +3868,7 @@ Node *firstGeqLogical(Node *n, const std::span<const uint8_t> key) {
       goto downLeftSpine;
     }
 
-    auto *child = getChild(n, remaining[0]);
+    Node *child = getChild(n, remaining[0]);
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {