Include childType in ChildAndMaxVersion

CMakeLists.txt

@@ -95,23 +95,12 @@ target_compile_options(${PROJECT_NAME}-object PRIVATE -fno-exceptions
                                                       -fvisibility=hidden)
 target_include_directories(${PROJECT_NAME}-object
                            PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
-if(NOT LD_EXE)
-  set(LD_EXE ld)
-endif()
-add_custom_command(
-  OUTPUT ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o
-  COMMAND ${LD_EXE} -r $<TARGET_OBJECTS:${PROJECT_NAME}-object> -o
-          ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o
-  DEPENDS $<TARGET_OBJECTS:${PROJECT_NAME}-object>
-  COMMAND_EXPAND_LISTS)
 
-add_library(${PROJECT_NAME} SHARED ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o)
+add_library(${PROJECT_NAME} SHARED $<TARGET_OBJECTS:${PROJECT_NAME}-object>)
 set_target_properties(
   ${PROJECT_NAME} PROPERTIES LIBRARY_OUTPUT_DIRECTORY
                              "${CMAKE_CURRENT_BINARY_DIR}/radix_tree")
-if(CMAKE_BUILD_TYPE STREQUAL Debug)
-  set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX)
-else()
+if(NOT CMAKE_BUILD_TYPE STREQUAL Debug)
   set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE C)
 endif()
 
@@ -121,13 +110,19 @@ if(HAS_VERSION_SCRIPT)
     LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/linker.map)
 endif()
 
-add_library(${PROJECT_NAME}-static STATIC ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o)
-if(CMAKE_BUILD_TYPE STREQUAL Debug)
-  set_target_properties(${PROJECT_NAME}-static PROPERTIES LINKER_LANGUAGE CXX)
-else()
+add_library(${PROJECT_NAME}-static STATIC
+            $<TARGET_OBJECTS:${PROJECT_NAME}-object>)
+if(NOT CMAKE_BUILD_TYPE STREQUAL Debug)
   set_target_properties(${PROJECT_NAME}-static PROPERTIES LINKER_LANGUAGE C)
 endif()
-if(NOT APPLE)
+
+if(APPLE)
+  add_custom_command(
+    TARGET ${PROJECT_NAME}-static
+    PRE_LINK
+    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/privatize_symbols_macos.sh
+            $<TARGET_OBJECTS:${PROJECT_NAME}-object>)
+else()
   add_custom_command(
     TARGET ${PROJECT_NAME}-static
     POST_BUILD

ConflictSet.cpp

@@ -205,28 +205,28 @@ template <class T> struct BoundedFreeListAllocator;
 
 struct TaggedNodePointer {
   TaggedNodePointer() = default;
-  operator struct Node *() { return (struct Node *)withoutType(); }
+  operator struct Node *() { return (struct Node *)(uintptr_t)p; }
   operator struct Node0 *() {
-    assert(getType() == Type_Node0);
-    return (struct Node0 *)withoutType();
+    assert(t == Type_Node0);
+    return (struct Node0 *)(uintptr_t)p;
   }
   operator struct Node3 *() {
-    assert(getType() == Type_Node3);
-    return (struct Node3 *)withoutType();
+    assert(t == Type_Node3);
+    return (struct Node3 *)(uintptr_t)p;
   }
   operator struct Node16 *() {
-    assert(getType() == Type_Node16);
-    return (struct Node16 *)withoutType();
+    assert(t == Type_Node16);
+    return (struct Node16 *)(uintptr_t)p;
   }
   operator struct Node48 *() {
-    assert(getType() == Type_Node48);
-    return (struct Node48 *)withoutType();
+    assert(t == Type_Node48);
+    return (struct Node48 *)(uintptr_t)p;
   }
   operator struct Node256 *() {
-    assert(getType() == Type_Node256);
-    return (struct Node256 *)withoutType();
+    assert(t == Type_Node256);
+    return (struct Node256 *)(uintptr_t)p;
   }
-  /*implicit*/ TaggedNodePointer(std::nullptr_t) : p(0) {}
+  /*implicit*/ TaggedNodePointer(std::nullptr_t) : p(0), t(Type_Node0) {}
   /*implicit*/ TaggedNodePointer(Node0 *x)
       : TaggedNodePointer((struct Node *)x, Type_Node0) {}
   /*implicit*/ TaggedNodePointer(Node3 *x)
@@ -238,25 +238,23 @@ struct TaggedNodePointer {
   /*implicit*/ TaggedNodePointer(Node256 *x)
       : TaggedNodePointer((struct Node *)x, Type_Node256) {}
 
-  bool operator!=(std::nullptr_t) { return p != 0; }
-  bool operator==(std::nullptr_t) { return p == 0; }
+  bool operator!=(std::nullptr_t) { return p != 0 || t != Type_Node0; }
+  bool operator==(std::nullptr_t) { return p == 0 && t == Type_Node0; }
   bool operator==(const TaggedNodePointer &) const = default;
-  bool operator==(Node *n) const { return (uintptr_t)n == withoutType(); }
-  Node *operator->() { return (Node *)withoutType(); }
-  Type getType();
+  bool operator==(Node *n) const { return (uintptr_t)n == p; }
+  Node *operator->() { return (Node *)(uintptr_t)p; }
+  Type getType() { return t; }
 
   TaggedNodePointer(const TaggedNodePointer &) = default;
   TaggedNodePointer &operator=(const TaggedNodePointer &) = default;
   /*implicit*/ TaggedNodePointer(Node *n);
 
 private:
-  TaggedNodePointer(struct Node *p, Type t) : p((uintptr_t)p) {
-    assert((this->p & 7) == 0);
-    this->p |= t;
+  TaggedNodePointer(struct Node *p, Type t) : p((uintptr_t)p), t(t) {
     assume(p != 0);
   }
-  uintptr_t withoutType() const { return p & ~uintptr_t(7); }
-  uintptr_t p;
+  uintptr_t p : 56;
+  Type t : 8;
 };
 
 struct Node {
@@ -287,11 +285,6 @@ private:
 TaggedNodePointer::TaggedNodePointer(Node *n)
     : TaggedNodePointer(n, n->getType()) {}
 
-Type TaggedNodePointer::getType() {
-  assert(p != 0);
-  return Type(p & uintptr_t(7));
-}
-
 constexpr int kNodeCopyBegin = offsetof(Node, entry);
 constexpr int kNodeCopySize =
     offsetof(Node, parentsIndex) + sizeof(Node::parentsIndex) - kNodeCopyBegin;
@@ -1104,24 +1097,22 @@ void setMaxVersion(Node *n, InternalVersionT newMax) {
 
 TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *);
 
-TaggedNodePointer getChild(Node0 *, uint8_t) { return nullptr; }
-TaggedNodePointer getChild(Node3 *self, uint8_t index) {
+Node *getChild(Node0 *, uint8_t) { return nullptr; }
+Node *getChild(Node3 *self, uint8_t index) {
   int i = getNodeIndex(self, index);
   return i < 0 ? nullptr : self->children[i];
 }
-TaggedNodePointer getChild(Node16 *self, uint8_t index) {
+Node *getChild(Node16 *self, uint8_t index) {
   int i = getNodeIndex(self, index);
   return i < 0 ? nullptr : self->children[i];
 }
-TaggedNodePointer getChild(Node48 *self, uint8_t index) {
+Node *getChild(Node48 *self, uint8_t index) {
   int i = self->index[index];
   return i < 0 ? nullptr : self->children[i];
 }
-TaggedNodePointer getChild(Node256 *self, uint8_t index) {
-  return self->children[index];
-}
+Node *getChild(Node256 *self, uint8_t index) { return self->children[index]; }
 
-TaggedNodePointer getChild(Node *self, uint8_t index) {
+Node *getChild(Node *self, uint8_t index) {
   switch (self->getType()) {
   case Type_Node0:
     return getChild(static_cast<Node0 *>(self), index);
@@ -1139,8 +1130,9 @@ TaggedNodePointer getChild(Node *self, uint8_t index) {
 }
 
 struct ChildAndMaxVersion {
-  TaggedNodePointer child;
+  Node *child;
   InternalVersionT maxVersion;
+  Type childType;
 };
 
 ChildAndMaxVersion getChildAndMaxVersion(Node0 *, uint8_t) { return {}; }
@@ -1149,24 +1141,28 @@ ChildAndMaxVersion getChildAndMaxVersion(Node3 *self, uint8_t index) {
   if (i < 0) {
     return {};
   }
-  return {self->children[i], self->childMaxVersion[i]};
+  return {self->children[i], self->childMaxVersion[i],
+          self->children[i].getType()};
 }
 ChildAndMaxVersion getChildAndMaxVersion(Node16 *self, uint8_t index) {
   int i = getNodeIndex(self, index);
   if (i < 0) {
     return {};
   }
-  return {self->children[i], self->childMaxVersion[i]};
+  return {self->children[i], self->childMaxVersion[i],
+          self->children[i].getType()};
 }
 ChildAndMaxVersion getChildAndMaxVersion(Node48 *self, uint8_t index) {
   int i = self->index[index];
   if (i < 0) {
     return {};
   }
-  return {self->children[i], self->childMaxVersion[i]};
+  return {self->children[i], self->childMaxVersion[i],
+          self->children[i].getType()};
 }
 ChildAndMaxVersion getChildAndMaxVersion(Node256 *self, uint8_t index) {
-  return {self->children[index], self->childMaxVersion[index]};
+  return {self->children[index], self->childMaxVersion[index],
+          self->children[index].getType()};
 }
 
 ChildAndMaxVersion getChildAndMaxVersion(Node *self, uint8_t index) {
@@ -1295,11 +1291,9 @@ Node *getFirstChildExists(Node256 *self) {
 
 // Precondition: self has a child
 Node *getFirstChildExists(Node *self) {
-  // Only require that the node-specific overloads are covered
-  // GCOVR_EXCL_START
   switch (self->getType()) {
-  case Type_Node0:
-    __builtin_unreachable();
+  case Type_Node0:           // GCOVR_EXCL_LINE
+    __builtin_unreachable(); // GCOVR_EXCL_LINE
   case Type_Node3:
     return getFirstChildExists(static_cast<Node3 *>(self));
   case Type_Node16:
@@ -1308,10 +1302,9 @@ Node *getFirstChildExists(Node *self) {
     return getFirstChildExists(static_cast<Node48 *>(self));
   case Type_Node256:
     return getFirstChildExists(static_cast<Node256 *>(self));
-  default:
-    __builtin_unreachable();
+  default:                   // GCOVR_EXCL_LINE
+    __builtin_unreachable(); // GCOVR_EXCL_LINE
   }
-  // GCOVR_EXCL_STOP
 }
 
 void consumePartialKeyFull(TaggedNodePointer &self,
@@ -1966,6 +1959,153 @@ Node *nextSibling(Node *node) {
   }
 }
 
+// Logically this is the same as performing firstGeq and then checking against
+// point or range version according to cmp, but this version short circuits as
+// soon as it can prove that there's no conflict.
+bool checkPointRead(Node *n, const std::span<const uint8_t> key,
+                    InternalVersionT readVersion, ReadContext *tls) {
+  ++tls->point_read_accum;
+#if DEBUG_VERBOSE && !defined(NDEBUG)
+  fprintf(stderr, "Check point read: %s\n", printable(key).c_str());
+#endif
+  auto remaining = key;
+  for (;; ++tls->point_read_iterations_accum) {
+    if (remaining.size() == 0) {
+      if (n->entryPresent) {
+        return n->entry.pointVersion <= readVersion;
+      }
+      n = getFirstChildExists(n);
+      goto downLeftSpine;
+    }
+
+    auto [child, maxV, childT] = getChildAndMaxVersion(n, remaining[0]);
+    if (child == nullptr) {
+      auto c = getChildGeq(n, remaining[0]);
+      if (c != nullptr) {
+        n = c;
+        goto downLeftSpine;
+      } else {
+        n = nextSibling(n);
+        if (n == nullptr) {
+          return true;
+        }
+        goto downLeftSpine;
+      }
+    }
+
+    n = child;
+    remaining = remaining.subspan(1, remaining.size() - 1);
+
+    if (n->partialKeyLen > 0) {
+      int commonLen = std::min<int>(n->partialKeyLen, remaining.size());
+      int i = longestCommonPrefix(n->partialKey(), remaining.data(), commonLen);
+      if (i < commonLen) {
+        auto c = n->partialKey()[i] <=> remaining[i];
+        if (c > 0) {
+          goto downLeftSpine;
+        } else {
+          n = nextSibling(n);
+          if (n == nullptr) {
+            return true;
+          }
+          goto downLeftSpine;
+        }
+      }
+      if (commonLen == n->partialKeyLen) {
+        // partial key matches
+        remaining = remaining.subspan(commonLen, remaining.size() - commonLen);
+      } else if (n->partialKeyLen > int(remaining.size())) {
+        // n is the first physical node greater than remaining, and there's no
+        // eq node
+        goto downLeftSpine;
+      }
+    }
+
+    if (maxV <= readVersion) {
+      ++tls->point_read_short_circuit_accum;
+      return true;
+    }
+  }
+downLeftSpine:
+  for (; !n->entryPresent; n = getFirstChildExists(n)) {
+  }
+  return n->entry.rangeVersion <= readVersion;
+}
+
+// Logically this is the same as performing firstGeq and then checking against
+// max version or range version if this prefix doesn't exist, but this version
+// short circuits as soon as it can prove that there's no conflict.
+bool checkPrefixRead(Node *n, const std::span<const uint8_t> key,
+                     InternalVersionT readVersion, ReadContext *tls) {
+  ++tls->prefix_read_accum;
+#if DEBUG_VERBOSE && !defined(NDEBUG)
+  fprintf(stderr, "Check prefix read: %s\n", printable(key).c_str());
+#endif
+  auto remaining = key;
+  for (;; ++tls->prefix_read_iterations_accum) {
+    if (remaining.size() == 0) {
+      // There's no way to encode a prefix read of "", so n is not the root
+      return maxVersion(n) <= readVersion;
+    }
+
+    auto [child, maxV, childT] = getChildAndMaxVersion(n, remaining[0]);
+    if (child == nullptr) {
+      auto c = getChildGeq(n, remaining[0]);
+      if (c != nullptr) {
+        n = c;
+        goto downLeftSpine;
+      } else {
+        n = nextSibling(n);
+        if (n == nullptr) {
+          return true;
+        }
+        goto downLeftSpine;
+      }
+    }
+
+    n = child;
+    remaining = remaining.subspan(1, remaining.size() - 1);
+
+    if (n->partialKeyLen > 0) {
+      int commonLen = std::min<int>(n->partialKeyLen, remaining.size());
+      int i = longestCommonPrefix(n->partialKey(), remaining.data(), commonLen);
+      if (i < commonLen) {
+        auto c = n->partialKey()[i] <=> remaining[i];
+        if (c > 0) {
+          goto downLeftSpine;
+        } else {
+          n = nextSibling(n);
+          if (n == nullptr) {
+            return true;
+          }
+          goto downLeftSpine;
+        }
+      }
+      if (commonLen == n->partialKeyLen) {
+        // partial key matches
+        remaining = remaining.subspan(commonLen, remaining.size() - commonLen);
+      } else if (n->partialKeyLen > int(remaining.size())) {
+        // n is the first physical node greater than remaining, and there's no
+        // eq node. All physical nodes that start with prefix are reachable from
+        // n.
+        if (maxVersion(n) > readVersion) {
+          return false;
+        }
+        goto downLeftSpine;
+      }
+    }
+
+    if (maxV <= readVersion) {
+      ++tls->prefix_read_short_circuit_accum;
+      return true;
+    }
+  }
+downLeftSpine:
+  for (; !n->entryPresent; n = getFirstChildExists(n)) {
+  }
+  return n->entry.rangeVersion <= readVersion;
+}
+
 #ifdef HAS_AVX
 uint32_t compare16(const InternalVersionT *vs, InternalVersionT rv) {
 #if USE_64_BIT
@@ -2216,7 +2356,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
     if (!mask) {
       return true;
     }
-    Node *child = self->children[std::countr_zero(mask) >> 2];
+    auto *child = self->children[std::countr_zero(mask) >> 2];
     const bool firstRangeOk =
         !child->entryPresent || child->entry.rangeVersion <= readVersion;
 
@@ -2284,7 +2424,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
     if (!mask) {
       return true;
     }
-    Node *child = self->children[std::countr_zero(mask)];
+    auto *child = self->children[std::countr_zero(mask)];
     const bool firstRangeOk =
         !child->entryPresent || child->entry.rangeVersion <= readVersion;
     uint32_t compared = 0;
@@ -2435,7 +2575,7 @@ bool checkRangeStartsWith(Node *n, std::span<const uint8_t> key, int begin,
     return checkMaxBetweenExclusive(n, begin, end, readVersion, tls);
   }
 
-  Node *child = getChild(n, remaining[0]);
+  auto *child = getChild(n, remaining[0]);
   if (child == nullptr) {
     auto c = getChildGeq(n, remaining[0]);
     if (c != nullptr) {
@@ -2507,8 +2647,7 @@ bool checkRangeLeftSide(Node *n, std::span<const uint8_t> key, int prefixLen,
       }
     }
 
-    auto [c, maxV] = getChildAndMaxVersion(n, remaining[0]);
-    Node *child = c;
+    auto [child, maxV, childT] = getChildAndMaxVersion(n, remaining[0]);
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {
@@ -2602,7 +2741,7 @@ bool checkRangeRightSide(Node *n, std::span<const uint8_t> key, int prefixLen,
       return false;
     }
 
-    Node *child = getChild(n, remaining[0]);
+    auto *child = getChild(n, remaining[0]);
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {
@@ -2669,9 +2808,20 @@ downLeftSpine:
 }
 } // namespace
 
-bool checkRangeRead(int lcp, Node *n, std::span<const uint8_t> begin,
+bool checkRangeRead(Node *n, std::span<const uint8_t> begin,
                     std::span<const uint8_t> end, InternalVersionT readVersion,
                     ReadContext *tls) {
+  int lcp = longestCommonPrefix(begin.data(), end.data(),
+                                std::min(begin.size(), end.size()));
+  if (lcp == int(begin.size()) && end.size() == begin.size() + 1 &&
+      end.back() == 0) {
+    return checkPointRead(n, begin, readVersion, tls);
+  }
+  if (lcp == int(begin.size() - 1) && end.size() == begin.size() &&
+      int(begin.back()) + 1 == int(end.back())) {
+    return checkPrefixRead(n, begin, readVersion, tls);
+  }
+
   ++tls->range_read_accum;
 
   auto remaining = begin.subspan(0, lcp);
@@ -2685,8 +2835,7 @@ bool checkRangeRead(int lcp, Node *n, std::span<const uint8_t> begin,
     if (remaining.size() == 0) {
       break;
     }
-    auto [c, v] = getChildAndMaxVersion(n, remaining[0]);
-    Node *child = c;
+    auto [child, v, childT] = getChildAndMaxVersion(n, remaining[0]);
     if (child == nullptr) {
       break;
     }
@@ -3005,7 +3154,7 @@ Node *firstGeqPhysical(Node *n, const std::span<const uint8_t> key) {
       return n;
     }
 
-    Node *child = getChild(n, remaining[0]);
+    auto *child = getChild(n, remaining[0]);
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {
@@ -3051,467 +3200,34 @@ Node *firstGeqPhysical(Node *n, const std::span<const uint8_t> key) {
   }
 }
 
-#ifndef __has_attribute
-#define __has_attribute(x) 0
-#endif
-
-#if __has_attribute(musttail)
-#define MUSTTAIL __attribute__((musttail))
-#else
-#define MUSTTAIL
-#endif
-
-#if __has_attribute(preserve_none)
-#define PRESERVE_NONE __attribute__((preserve_none))
-#else
-#define PRESERVE_NONE
-#endif
-
-#if __has_attribute(flatten)
-#define FLATTEN __attribute__((flatten))
-#else
-#define FLATTEN
-#endif
-
-typedef PRESERVE_NONE void (*Continuation)(struct CheckJob *,
-                                           struct CheckContext *);
-
-// State relevant to an individual query
-struct CheckJob {
-  void setResult(bool ok) {
-    *result = ok ? ConflictSet::Commit : ConflictSet::Conflict;
-  }
-
-  void init(const ConflictSet::ReadRange *read, ConflictSet::Result *result,
-            Node *root, int64_t oldestVersionFullPrecision);
-
-  Node *n;
-  std::span<const uint8_t> begin;
-  std::span<const uint8_t> end; // range read only
-  InternalVersionT readVersion;
-  ConflictSet::Result *result;
-  Continuation continuation;
-  CheckJob *prev;
-  CheckJob *next;
-};
-
-// State relevant to every query
-struct CheckContext {
-  int count;
-  int64_t oldestVersionFullPrecision;
-  Node *root;
-  const ConflictSet::ReadRange *queries;
-  ConflictSet::Result *results;
-  int64_t started;
-  ReadContext *tls;
-#if !__has_attribute(musttail)
-  CheckJob *job;
-  bool done;
-#endif
-};
-
-FLATTEN PRESERVE_NONE void keepGoing(CheckJob *job, CheckContext *context) {
-#if __has_attribute(musttail)
-  job = job->next;
-  MUSTTAIL return job->continuation(job, context);
-#else
-  context->job = job->next;
-  return;
-#endif
-}
-
-FLATTEN PRESERVE_NONE void complete(CheckJob *job, CheckContext *context) {
-  if (context->started == context->count) {
-    if (job->prev == job) {
-#if !__has_attribute(musttail)
-      context->done = true;
-#endif
-      return;
-    }
-    job->prev->next = job->next;
-    job->next->prev = job->prev;
-    job = job->prev;
-  } else {
-    int temp = context->started++;
-    job->init(context->queries + temp, context->results + temp, context->root,
-              context->oldestVersionFullPrecision);
-  }
-  MUSTTAIL return keepGoing(job, context);
-}
-
-namespace check_point_read_state_machine {
-
-FLATTEN PRESERVE_NONE void begin(CheckJob *, CheckContext *);
-
-template <class NodeT>
-FLATTEN PRESERVE_NONE void iter(CheckJob *, CheckContext *);
-
-FLATTEN PRESERVE_NONE void down_left_spine(CheckJob *, CheckContext *);
-
-static Continuation iterTable[] = {iter<Node0>, iter<Node3>, iter<Node16>,
-                                   iter<Node48>, iter<Node256>};
-
-void begin(CheckJob *job, CheckContext *context) {
-  ++context->tls->point_read_accum;
-#if DEBUG_VERBOSE && !defined(NDEBUG)
-  fprintf(stderr, "Check point read: %s\n", printable(key).c_str());
-#endif
-
-  if (job->begin.size() == 0) [[unlikely]] {
-    // We don't erase the root
-    assert(job->n->entryPresent);
-    job->setResult(job->n->entry.pointVersion <= job->readVersion);
-    MUSTTAIL return complete(job, context);
-  }
-
-  auto taggedChild = getChild(job->n, job->begin[0]);
-  Node *child = taggedChild;
-  if (child == nullptr) [[unlikely]] {
-    auto c = getChildGeq(job->n, job->begin[0]);
-    if (c != nullptr) {
-      job->n = c;
-      job->continuation = down_left_spine;
-      __builtin_prefetch(job->n);
-      MUSTTAIL return keepGoing(job, context);
-    } else {
-      // The root never has a next sibling
-      job->setResult(true);
-      MUSTTAIL return complete(job, context);
-    }
-  }
-  job->continuation = iterTable[taggedChild.getType()];
-  job->n = child;
-  __builtin_prefetch(child);
-  MUSTTAIL return keepGoing(job, context);
-}
-
-template <class NodeT> void iter(CheckJob *job, CheckContext *context) {
-
-  assert(NodeT::kType == job->n->getType());
-  NodeT *n = static_cast<NodeT *>(job->n);
-  job->begin = job->begin.subspan(1, job->begin.size() - 1);
-
-  if (n->partialKeyLen > 0) {
-    int commonLen = std::min<int>(n->partialKeyLen, job->begin.size());
-    int i = longestCommonPrefix(n->partialKey(), job->begin.data(), commonLen);
-    if (i < commonLen) [[unlikely]] {
-      auto c = n->partialKey()[i] <=> job->begin[i];
-      if (c > 0) {
-        job->continuation = down_left_spine;
-        MUSTTAIL return down_left_spine(job, context);
-      } else {
-        job->n = nextSibling(n);
-        if (job->n == nullptr) {
-          job->setResult(true);
-          MUSTTAIL return complete(job, context);
-        }
-        job->continuation = down_left_spine;
-        __builtin_prefetch(job->n);
-        MUSTTAIL return keepGoing(job, context);
-      }
-    }
-    if (commonLen == n->partialKeyLen) {
-      // partial key matches
-      job->begin = job->begin.subspan(commonLen, job->begin.size() - commonLen);
-    } else if (n->partialKeyLen > int(job->begin.size())) [[unlikely]] {
-      // n is the first physical node greater than remaining, and there's no
-      // eq node
-      job->continuation = down_left_spine;
-      MUSTTAIL return down_left_spine(job, context);
-    }
-  }
-
-  ++context->tls->point_read_iterations_accum;
-
-  if (job->begin.size() == 0) [[unlikely]] {
-    if (n->entryPresent) {
-      job->setResult(n->entry.pointVersion <= job->readVersion);
-      MUSTTAIL return complete(job, context);
-    }
-    job->n = getFirstChildExists(n);
-    job->continuation = down_left_spine;
-    __builtin_prefetch(job->n);
-    MUSTTAIL return keepGoing(job, context);
-  }
-
-  auto taggedChild = getChild(n, job->begin[0]);
-  Node *child = taggedChild;
-  if (child == nullptr) [[unlikely]] {
-    auto c = getChildGeq(n, job->begin[0]);
-    if (c != nullptr) {
-      job->n = c;
-      job->continuation = down_left_spine;
-      __builtin_prefetch(job->n);
-      MUSTTAIL return keepGoing(job, context);
-    } else {
-      job->n = nextSibling(job->n);
-      if (job->n == nullptr) {
-        job->setResult(true);
-        MUSTTAIL return complete(job, context);
-      }
-      job->continuation = down_left_spine;
-      __builtin_prefetch(job->n);
-      MUSTTAIL return keepGoing(job, context);
-    }
-  }
-  job->continuation = iterTable[taggedChild.getType()];
-  job->n = child;
-  __builtin_prefetch(child);
-  MUSTTAIL return keepGoing(job, context);
-}
-
-void down_left_spine(CheckJob *job, CheckContext *context) {
-  if (job->n->entryPresent) {
-    job->setResult(job->n->entry.rangeVersion <= job->readVersion);
-    MUSTTAIL return complete(job, context);
-  }
-  job->n = getFirstChildExists(job->n);
-  __builtin_prefetch(job->n);
-  MUSTTAIL return keepGoing(job, context);
-}
-
-} // namespace check_point_read_state_machine
-
-namespace check_prefix_read_state_machine {
-
-FLATTEN PRESERVE_NONE void begin(CheckJob *, CheckContext *);
-
-template <class NodeT>
-FLATTEN PRESERVE_NONE void iter(CheckJob *, CheckContext *);
-
-FLATTEN PRESERVE_NONE void down_left_spine(CheckJob *, CheckContext *);
-
-static Continuation iterTable[] = {iter<Node0>, iter<Node3>, iter<Node16>,
-                                   iter<Node48>, iter<Node256>};
-
-void begin(CheckJob *job, CheckContext *context) {
-  ++context->tls->prefix_read_accum;
-#if DEBUG_VERBOSE && !defined(NDEBUG)
-  fprintf(stderr, "Check prefix read: %s\n", printable(key).c_str());
-#endif
-
-  // There's no way to encode a prefix read of ""
-  assert(job->begin.size() > 0);
-
-  auto taggedChild = getChild(job->n, job->begin[0]);
-  Node *child = taggedChild;
-  if (child == nullptr) [[unlikely]] {
-    auto c = getChildGeq(job->n, job->begin[0]);
-    if (c != nullptr) {
-      job->n = c;
-      job->continuation = down_left_spine;
-      __builtin_prefetch(job->n);
-      MUSTTAIL return keepGoing(job, context);
-    } else {
-      // The root never has a next sibling
-      job->setResult(true);
-      MUSTTAIL return complete(job, context);
-    }
-  }
-  job->continuation = iterTable[taggedChild.getType()];
-  job->n = child;
-  __builtin_prefetch(child);
-  MUSTTAIL return keepGoing(job, context);
-}
-
-template <class NodeT> void iter(CheckJob *job, CheckContext *context) {
-
-  assert(NodeT::kType == job->n->getType());
-  NodeT *n = static_cast<NodeT *>(job->n);
-  job->begin = job->begin.subspan(1, job->begin.size() - 1);
-
-  if (n->partialKeyLen > 0) {
-    int commonLen = std::min<int>(n->partialKeyLen, job->begin.size());
-    int i = longestCommonPrefix(n->partialKey(), job->begin.data(), commonLen);
-    if (i < commonLen) [[unlikely]] {
-      auto c = n->partialKey()[i] <=> job->begin[i];
-      if (c > 0) {
-        job->continuation = down_left_spine;
-        MUSTTAIL return down_left_spine(job, context);
-      } else {
-        job->n = nextSibling(n);
-        if (job->n == nullptr) {
-          job->setResult(true);
-          MUSTTAIL return complete(job, context);
-        }
-        job->continuation = down_left_spine;
-        __builtin_prefetch(job->n);
-        MUSTTAIL return keepGoing(job, context);
-      }
-    }
-    if (commonLen == n->partialKeyLen) {
-      // partial key matches
-      job->begin = job->begin.subspan(commonLen, job->begin.size() - commonLen);
-    } else if (n->partialKeyLen > int(job->begin.size())) [[unlikely]] {
-      // n is the first physical node greater than remaining, and there's no
-      // eq node. All physical nodes that start with prefix are reachable from
-      // n.
-      if (maxVersion(n) > job->readVersion) {
-        job->setResult(false);
-        MUSTTAIL return complete(job, context);
-      }
-      job->continuation = down_left_spine;
-      MUSTTAIL return down_left_spine(job, context);
-    }
-  }
-
-  ++context->tls->prefix_read_iterations_accum;
-
-  if (job->begin.size() == 0) [[unlikely]] {
-    job->setResult(maxVersion(job->n) <= job->readVersion);
-    MUSTTAIL return complete(job, context);
-  }
-
-  auto taggedChild = getChild(n, job->begin[0]);
-  Node *child = taggedChild;
-  if (child == nullptr) [[unlikely]] {
-    auto c = getChildGeq(n, job->begin[0]);
-    if (c != nullptr) {
-      job->n = c;
-      job->continuation = down_left_spine;
-      __builtin_prefetch(job->n);
-      MUSTTAIL return keepGoing(job, context);
-    } else {
-      job->n = nextSibling(job->n);
-      if (job->n == nullptr) {
-        job->setResult(true);
-        MUSTTAIL return complete(job, context);
-      }
-      job->continuation = down_left_spine;
-      __builtin_prefetch(job->n);
-      MUSTTAIL return keepGoing(job, context);
-    }
-  }
-  job->continuation = iterTable[taggedChild.getType()];
-  job->n = child;
-  __builtin_prefetch(child);
-  MUSTTAIL return keepGoing(job, context);
-}
-
-void down_left_spine(CheckJob *job, CheckContext *context) {
-  if (job->n->entryPresent) {
-    job->setResult(job->n->entry.rangeVersion <= job->readVersion);
-    MUSTTAIL return complete(job, context);
-  }
-  job->n = getFirstChildExists(job->n);
-  __builtin_prefetch(job->n);
-  MUSTTAIL return keepGoing(job, context);
-}
-
-} // namespace check_prefix_read_state_machine
-
-namespace check_range_read_state_machine {
-FLATTEN PRESERVE_NONE void begin(CheckJob *, CheckContext *);
-
-FLATTEN PRESERVE_NONE void begin(CheckJob *job, CheckContext *context) {
-  int lcp = longestCommonPrefix(job->begin.data(), job->end.data(),
-                                std::min(job->begin.size(), job->end.size()));
-  if (lcp == int(job->begin.size()) &&
-      job->end.size() == job->begin.size() + 1 && job->end.back() == 0) {
-    job->continuation = check_point_read_state_machine::begin;
-    // Call directly since we have nothing to prefetch
-    MUSTTAIL return job->continuation(job, context);
-  }
-  if (lcp == int(job->begin.size() - 1) &&
-      job->end.size() == job->begin.size() &&
-      int(job->begin.back()) + 1 == int(job->end.back())) {
-    job->continuation = check_prefix_read_state_machine::begin;
-    // Call directly since we have nothing to prefetch
-    MUSTTAIL return job->continuation(job, context);
-  }
-
-  *job->result = checkRangeRead(lcp, job->n, job->begin, job->end,
-                                job->readVersion, context->tls)
-                     ? ConflictSet::Commit
-                     : ConflictSet::Conflict;
-  return complete(job, context);
-}
-
-} // namespace check_range_read_state_machine
-
-void CheckJob::init(const ConflictSet::ReadRange *read,
-                    ConflictSet::Result *result, Node *root,
-                    int64_t oldestVersionFullPrecision) {
-  auto begin = std::span<const uint8_t>(read->begin.p, read->begin.len);
-  auto end = std::span<const uint8_t>(read->end.p, read->end.len);
-  if (read->readVersion < oldestVersionFullPrecision) [[unlikely]] {
-    *result = ConflictSet::TooOld;
-    continuation = complete;
-  } else if (end.size() == 0) {
-    this->begin = begin;
-    this->n = root;
-    this->readVersion = InternalVersionT(read->readVersion);
-    this->result = result;
-    continuation = check_point_read_state_machine::begin;
-  } else {
-    this->begin = begin;
-    this->end = end;
-    this->n = root;
-    this->readVersion = InternalVersionT(read->readVersion);
-    this->result = result;
-    continuation = check_range_read_state_machine::begin;
-  }
-}
-
 struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
 
   void check(const ReadRange *reads, Result *result, int count) {
-    assert(oldestVersionFullPrecision >=
-           newestVersionFullPrecision - kNominalVersionWindow);
-
-    if (count == 0) {
-      return;
-    }
-
     ReadContext tls;
     tls.impl = this;
     int64_t check_byte_accum = 0;
-    constexpr int kConcurrent = 16;
-    CheckJob inProgress[kConcurrent];
-    CheckContext context;
-    context.count = count;
-    context.oldestVersionFullPrecision = oldestVersionFullPrecision;
-    context.root = root;
-    context.queries = reads;
-    context.results = result;
-    context.tls = &tls;
-    int64_t started = std::min(kConcurrent, count);
-    context.started = started;
-    for (int i = 0; i < started; i++) {
-      inProgress[i].init(reads + i, result + i, root,
-                         oldestVersionFullPrecision);
-    }
-    for (int i = 0; i < started - 1; i++) {
-      inProgress[i].next = inProgress + i + 1;
-    }
-    for (int i = 1; i < started; i++) {
-      inProgress[i].prev = inProgress + i - 1;
-    }
-    inProgress[0].prev = inProgress + started - 1;
-    inProgress[started - 1].next = inProgress;
-
-#if __has_attribute(musttail)
-    // Kick off the sequence of tail calls that finally returns once all jobs
-    // are done
-    inProgress->continuation(inProgress, &context);
-#else
-    context.job = inProgress;
-    context.done = false;
-    while (!context.done) {
-      context.job->continuation(context.job, &context);
-    }
-#endif
-
     for (int i = 0; i < count; ++i) {
       assert(reads[i].readVersion >= 0);
       assert(reads[i].readVersion <= newestVersionFullPrecision);
       const auto &r = reads[i];
       check_byte_accum += r.begin.len + r.end.len;
+      auto begin = std::span<const uint8_t>(r.begin.p, r.begin.len);
+      auto end = std::span<const uint8_t>(r.end.p, r.end.len);
+      assert(oldestVersionFullPrecision >=
+             newestVersionFullPrecision - kNominalVersionWindow);
+      result[i] =
+          reads[i].readVersion < oldestVersionFullPrecision ? TooOld
+          : (end.size() > 0
+                 ? checkRangeRead(root, begin, end,
+                                  InternalVersionT(reads[i].readVersion), &tls)
+                 : checkPointRead(root, begin,
+                                  InternalVersionT(reads[i].readVersion), &tls))
+              ? Commit
+              : Conflict;
       tls.commits_accum += result[i] == Commit;
       tls.conflicts_accum += result[i] == Conflict;
       tls.too_olds_accum += result[i] == TooOld;
     }
-
     point_read_total.add(tls.point_read_accum);
     prefix_read_total.add(tls.prefix_read_accum);
     range_read_total.add(tls.range_read_accum);
@@ -3890,7 +3606,7 @@ Node *firstGeqLogical(Node *n, const std::span<const uint8_t> key) {
       goto downLeftSpine;
     }
 
-    Node *child = getChild(n, remaining[0]);
+    auto *child = getChild(n, remaining[0]);
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {

Dockerfile

@@ -8,7 +8,6 @@ RUN chmod -R 777 /tmp
 RUN apt-get update
 RUN apt-get upgrade -y
 RUN TZ=America/Los_Angeles DEBIAN_FRONTEND=noninteractive apt-get install -y \
-    binutils-aarch64-linux-gnu \
     build-essential \
     ccache \
     clang \

aarch64-toolchain.cmake

@@ -5,4 +5,3 @@ set(CMAKE_CXX_COMPILER "/usr/bin/aarch64-linux-gnu-g++")
 set(CMAKE_FIND_ROOT_PATH /usr/aarch64-linux-gnu)
 set(CMAKE_CROSSCOMPILING_EMULATOR "qemu-aarch64;-L;/usr/aarch64-linux-gnu/")
 set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE arm64)
-set(LD_EXE "/usr/bin/aarch64-linux-gnu-ld")