Use a trampoline if musttail not available

For the previous approach to work, we would have had to lock in the
generated code for every function that needs to make tail calls in
assembly.

CMakeLists.txt

@@ -95,12 +95,23 @@ target_compile_options(${PROJECT_NAME}-object PRIVATE -fno-exceptions
                                                       -fvisibility=hidden)
 target_include_directories(${PROJECT_NAME}-object
                            PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
+if(NOT LD_EXE)
+  set(LD_EXE ld)
+endif()
+add_custom_command(
+  OUTPUT ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o
+  COMMAND ${LD_EXE} -r $<TARGET_OBJECTS:${PROJECT_NAME}-object> -o
+          ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o
+  DEPENDS $<TARGET_OBJECTS:${PROJECT_NAME}-object>
+  COMMAND_EXPAND_LISTS)
 
-add_library(${PROJECT_NAME} SHARED $<TARGET_OBJECTS:${PROJECT_NAME}-object>)
+add_library(${PROJECT_NAME} SHARED ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o)
 set_target_properties(
   ${PROJECT_NAME} PROPERTIES LIBRARY_OUTPUT_DIRECTORY
                              "${CMAKE_CURRENT_BINARY_DIR}/radix_tree")
-if(NOT CMAKE_BUILD_TYPE STREQUAL Debug)
+if(CMAKE_BUILD_TYPE STREQUAL Debug)
+  set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX)
+else()
   set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE C)
 endif()
 
@@ -110,19 +121,13 @@ if(HAS_VERSION_SCRIPT)
     LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/linker.map)
 endif()
 
-add_library(${PROJECT_NAME}-static STATIC
-            $<TARGET_OBJECTS:${PROJECT_NAME}-object>)
-if(NOT CMAKE_BUILD_TYPE STREQUAL Debug)
+add_library(${PROJECT_NAME}-static STATIC ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o)
+if(CMAKE_BUILD_TYPE STREQUAL Debug)
+  set_target_properties(${PROJECT_NAME}-static PROPERTIES LINKER_LANGUAGE CXX)
+else()
   set_target_properties(${PROJECT_NAME}-static PROPERTIES LINKER_LANGUAGE C)
 endif()
-
-if(APPLE)
-  add_custom_command(
-    TARGET ${PROJECT_NAME}-static
-    PRE_LINK
-    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/privatize_symbols_macos.sh
-            $<TARGET_OBJECTS:${PROJECT_NAME}-object>)
-else()
+if(NOT APPLE)
   add_custom_command(
     TARGET ${PROJECT_NAME}-static
     POST_BUILD

ConflictSet.cpp

@@ -203,6 +203,62 @@ enum Type : int8_t {
 
 template <class T> struct BoundedFreeListAllocator;
 
+struct TaggedNodePointer {
+  TaggedNodePointer() = default;
+  operator struct Node *() { return (struct Node *)withoutType(); }
+  operator struct Node0 *() {
+    assert(getType() == Type_Node0);
+    return (struct Node0 *)withoutType();
+  }
+  operator struct Node3 *() {
+    assert(getType() == Type_Node3);
+    return (struct Node3 *)withoutType();
+  }
+  operator struct Node16 *() {
+    assert(getType() == Type_Node16);
+    return (struct Node16 *)withoutType();
+  }
+  operator struct Node48 *() {
+    assert(getType() == Type_Node48);
+    return (struct Node48 *)withoutType();
+  }
+  operator struct Node256 *() {
+    assert(getType() == Type_Node256);
+    return (struct Node256 *)withoutType();
+  }
+  /*implicit*/ TaggedNodePointer(std::nullptr_t) : p(0) {}
+  /*implicit*/ TaggedNodePointer(Node0 *x)
+      : TaggedNodePointer((struct Node *)x, Type_Node0) {}
+  /*implicit*/ TaggedNodePointer(Node3 *x)
+      : TaggedNodePointer((struct Node *)x, Type_Node3) {}
+  /*implicit*/ TaggedNodePointer(Node16 *x)
+      : TaggedNodePointer((struct Node *)x, Type_Node16) {}
+  /*implicit*/ TaggedNodePointer(Node48 *x)
+      : TaggedNodePointer((struct Node *)x, Type_Node48) {}
+  /*implicit*/ TaggedNodePointer(Node256 *x)
+      : TaggedNodePointer((struct Node *)x, Type_Node256) {}
+
+  bool operator!=(std::nullptr_t) { return p != 0; }
+  bool operator==(std::nullptr_t) { return p == 0; }
+  bool operator==(const TaggedNodePointer &) const = default;
+  bool operator==(Node *n) const { return (uintptr_t)n == withoutType(); }
+  Node *operator->() { return (Node *)withoutType(); }
+  Type getType();
+
+  TaggedNodePointer(const TaggedNodePointer &) = default;
+  TaggedNodePointer &operator=(const TaggedNodePointer &) = default;
+  /*implicit*/ TaggedNodePointer(Node *n);
+
+private:
+  TaggedNodePointer(struct Node *p, Type t) : p((uintptr_t)p) {
+    assert((this->p & 7) == 0);
+    this->p |= t;
+    assume(p != 0);
+  }
+  uintptr_t withoutType() const { return p & ~uintptr_t(7); }
+  uintptr_t p;
+};
+
 struct Node {
 
   /* begin section that's copied to the next node */
@@ -228,6 +284,14 @@ private:
   int32_t partialKeyCapacity;
 };
 
+TaggedNodePointer::TaggedNodePointer(Node *n)
+    : TaggedNodePointer(n, n->getType()) {}
+
+Type TaggedNodePointer::getType() {
+  assert(p != 0);
+  return Type(p & uintptr_t(7));
+}
+
 constexpr int kNodeCopyBegin = offsetof(Node, entry);
 constexpr int kNodeCopySize =
     offsetof(Node, parentsIndex) + sizeof(Node::parentsIndex) - kNodeCopyBegin;
@@ -251,7 +315,7 @@ struct Node3 : Node {
   constexpr static auto kMaxNodes = 3;
   constexpr static auto kType = Type_Node3;
 
-  Node *children[kMaxNodes];
+  TaggedNodePointer children[kMaxNodes];
   InternalVersionT childMaxVersion[kMaxNodes];
   // Sorted
   uint8_t index[kMaxNodes];
@@ -267,7 +331,7 @@ struct Node16 : Node {
   constexpr static auto kType = Type_Node16;
   constexpr static auto kMaxNodes = 16;
 
-  Node *children[kMaxNodes];
+  TaggedNodePointer children[kMaxNodes];
   InternalVersionT childMaxVersion[kMaxNodes];
   // Sorted
   uint8_t index[kMaxNodes];
@@ -288,7 +352,7 @@ struct Node48 : Node {
   constexpr static int kMaxOfMaxTotalPages = kMaxNodes / kMaxOfMaxPageSize;
 
   BitSet bitSet;
-  Node *children[kMaxNodes];
+  TaggedNodePointer children[kMaxNodes];
   InternalVersionT childMaxVersion[kMaxNodes];
   InternalVersionT maxOfMax[kMaxOfMaxTotalPages];
   uint8_t reverseIndex[kMaxNodes];
@@ -310,7 +374,7 @@ struct Node256 : Node {
   constexpr static int kMaxOfMaxTotalPages = kMaxNodes / kMaxOfMaxPageSize;
 
   BitSet bitSet;
-  Node *children[kMaxNodes];
+  TaggedNodePointer children[kMaxNodes];
   InternalVersionT childMaxVersion[kMaxNodes];
   InternalVersionT maxOfMax[kMaxOfMaxTotalPages];
 
@@ -888,26 +952,26 @@ int getNodeIndexExists(Node16 *self, uint8_t index) {
 }
 
 // Precondition - an entry for index must exist in the node
-Node *&getChildExists(Node3 *self, uint8_t index) {
+TaggedNodePointer &getChildExists(Node3 *self, uint8_t index) {
   return self->children[getNodeIndexExists(self, index)];
 }
 // Precondition - an entry for index must exist in the node
-Node *&getChildExists(Node16 *self, uint8_t index) {
+TaggedNodePointer &getChildExists(Node16 *self, uint8_t index) {
   return self->children[getNodeIndexExists(self, index)];
 }
 // Precondition - an entry for index must exist in the node
-Node *&getChildExists(Node48 *self, uint8_t index) {
+TaggedNodePointer &getChildExists(Node48 *self, uint8_t index) {
   assert(self->bitSet.test(index));
   return self->children[self->index[index]];
 }
 // Precondition - an entry for index must exist in the node
-Node *&getChildExists(Node256 *self, uint8_t index) {
+TaggedNodePointer &getChildExists(Node256 *self, uint8_t index) {
   assert(self->bitSet.test(index));
   return self->children[index];
 }
 
 // Precondition - an entry for index must exist in the node
-Node *&getChildExists(Node *self, uint8_t index) {
+TaggedNodePointer &getChildExists(Node *self, uint8_t index) {
   switch (self->getType()) {
   case Type_Node0:           // GCOVR_EXCL_LINE
     __builtin_unreachable(); // GCOVR_EXCL_LINE
@@ -1038,24 +1102,26 @@ void setMaxVersion(Node *n, InternalVersionT newMax) {
   }
 }
 
-Node *&getInTree(Node *n, ConflictSet::Impl *);
+TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *);
 
-Node *getChild(Node0 *, uint8_t) { return nullptr; }
-Node *getChild(Node3 *self, uint8_t index) {
+TaggedNodePointer getChild(Node0 *, uint8_t) { return nullptr; }
+TaggedNodePointer getChild(Node3 *self, uint8_t index) {
   int i = getNodeIndex(self, index);
   return i < 0 ? nullptr : self->children[i];
 }
-Node *getChild(Node16 *self, uint8_t index) {
+TaggedNodePointer getChild(Node16 *self, uint8_t index) {
   int i = getNodeIndex(self, index);
   return i < 0 ? nullptr : self->children[i];
 }
-Node *getChild(Node48 *self, uint8_t index) {
+TaggedNodePointer getChild(Node48 *self, uint8_t index) {
   int i = self->index[index];
   return i < 0 ? nullptr : self->children[i];
 }
-Node *getChild(Node256 *self, uint8_t index) { return self->children[index]; }
+TaggedNodePointer getChild(Node256 *self, uint8_t index) {
+  return self->children[index];
+}
 
-Node *getChild(Node *self, uint8_t index) {
+TaggedNodePointer getChild(Node *self, uint8_t index) {
   switch (self->getType()) {
   case Type_Node0:
     return getChild(static_cast<Node0 *>(self), index);
@@ -1073,7 +1139,7 @@ Node *getChild(Node *self, uint8_t index) {
 }
 
 struct ChildAndMaxVersion {
-  Node *child;
+  TaggedNodePointer child;
   InternalVersionT maxVersion;
 };
 
@@ -1245,14 +1311,15 @@ Node *getFirstChildExists(Node *self) {
   }
 }
 
-void consumePartialKeyFull(Node *&self, std::span<const uint8_t> &key,
+void consumePartialKeyFull(TaggedNodePointer &self,
+                           std::span<const uint8_t> &key,
                            InternalVersionT writeVersion, WriteContext *tls) {
   // Handle an existing partial key
   int commonLen = std::min<int>(self->partialKeyLen, key.size());
   int partialKeyIndex =
       longestCommonPrefix(self->partialKey(), key.data(), commonLen);
   if (partialKeyIndex < self->partialKeyLen) {
-    auto *old = self;
+    Node *old = self;
     // Since root cannot have a partial key
     assert(old->parent != nullptr);
     InternalVersionT oldMaxVersion = exchangeMaxVersion(old, writeVersion);
@@ -1290,7 +1357,7 @@ void consumePartialKeyFull(Node *&self, std::span<const uint8_t> &key,
 // Consume any partial key of `self`, and update `self` and
 // `key` such that `self` is along the search path of `key`
 inline __attribute__((always_inline)) void
-consumePartialKey(Node *&self, std::span<const uint8_t> &key,
+consumePartialKey(TaggedNodePointer &self, std::span<const uint8_t> &key,
                   InternalVersionT writeVersion, WriteContext *tls) {
   if (self->partialKeyLen > 0) {
     consumePartialKeyFull(self, key, writeVersion, tls);
@@ -1301,8 +1368,10 @@ consumePartialKey(Node *&self, std::span<const uint8_t> &key,
 // such that the search path of the result + key is the same as the search path
 // of self + key before the call. Creates a node if necessary. Updates
 // `maxVersion` for result.
-Node *&getOrCreateChild(Node *&self, std::span<const uint8_t> &key,
-                        InternalVersionT newMaxVersion, WriteContext *tls) {
+TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self,
+                                    std::span<const uint8_t> &key,
+                                    InternalVersionT newMaxVersion,
+                                    WriteContext *tls) {
 
   int index = key.front();
   key = key.subspan(1, key.size() - 1);
@@ -1660,10 +1729,11 @@ void rezero(Node *n, InternalVersionT z) {
 }
 #endif
 
-void mergeWithChild(Node *&self, WriteContext *tls, ConflictSet::Impl *impl,
-                    Node *&dontInvalidate, Node3 *self3) {
+void mergeWithChild(TaggedNodePointer &self, WriteContext *tls,
+                    ConflictSet::Impl *impl, Node *&dontInvalidate,
+                    Node3 *self3) {
   assert(!self3->entryPresent);
-  auto *child = self3->children[0];
+  Node *child = self3->children[0];
   int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen;
 
   if (minCapacity > child->getCapacity()) {
@@ -1912,7 +1982,8 @@ bool checkPointRead(Node *n, const std::span<const uint8_t> key,
       goto downLeftSpine;
     }
 
-    auto [child, maxV] = getChildAndMaxVersion(n, remaining[0]);
+    auto [c, maxV] = getChildAndMaxVersion(n, remaining[0]);
+    Node *child = c;
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {
@@ -1982,7 +2053,8 @@ bool checkPrefixRead(Node *n, const std::span<const uint8_t> key,
       return maxVersion(n) <= readVersion;
     }
 
-    auto [child, maxV] = getChildAndMaxVersion(n, remaining[0]);
+    auto [c, maxV] = getChildAndMaxVersion(n, remaining[0]);
+    Node *child = c;
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {
@@ -2255,7 +2327,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
     if (!mask) {
       return true;
     }
-    auto *child = self->children[std::countr_zero(mask)];
+    Node *child = self->children[std::countr_zero(mask)];
     const bool firstRangeOk =
         !child->entryPresent || child->entry.rangeVersion <= readVersion;
     uint32_t compared = 0;
@@ -2290,7 +2362,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
     if (!mask) {
       return true;
     }
-    auto *child = self->children[std::countr_zero(mask) >> 2];
+    Node *child = self->children[std::countr_zero(mask) >> 2];
     const bool firstRangeOk =
         !child->entryPresent || child->entry.rangeVersion <= readVersion;
 
@@ -2330,7 +2402,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
     if (!mask) {
       return true;
     }
-    auto *child = self->children[std::countr_zero(mask)];
+    Node *child = self->children[std::countr_zero(mask)];
     const bool firstRangeOk =
         !child->entryPresent || child->entry.rangeVersion <= readVersion;
 
@@ -2358,7 +2430,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
     if (!mask) {
       return true;
     }
-    auto *child = self->children[std::countr_zero(mask)];
+    Node *child = self->children[std::countr_zero(mask)];
     const bool firstRangeOk =
         !child->entryPresent || child->entry.rangeVersion <= readVersion;
     uint32_t compared = 0;
@@ -2375,7 +2447,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
     {
       int c = self->bitSet.firstSetGeq(begin + 1);
       if (c >= 0 && c < end) {
-        auto *child = self->children[self->index[c]];
+        Node *child = self->children[self->index[c]];
         if (child->entryPresent && child->entry.rangeVersion > readVersion) {
           return false;
         }
@@ -2409,7 +2481,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
     {
       int c = self->bitSet.firstSetGeq(begin + 1);
       if (c >= 0 && c < end) {
-        auto *child = self->children[c];
+        Node *child = self->children[c];
         if (child->entryPresent && child->entry.rangeVersion > readVersion) {
           return false;
         }
@@ -2509,7 +2581,7 @@ bool checkRangeStartsWith(Node *n, std::span<const uint8_t> key, int begin,
     return checkMaxBetweenExclusive(n, begin, end, readVersion, tls);
   }
 
-  auto *child = getChild(n, remaining[0]);
+  Node *child = getChild(n, remaining[0]);
   if (child == nullptr) {
     auto c = getChildGeq(n, remaining[0]);
     if (c != nullptr) {
@@ -2581,7 +2653,8 @@ bool checkRangeLeftSide(Node *n, std::span<const uint8_t> key, int prefixLen,
       }
     }
 
-    auto [child, maxV] = getChildAndMaxVersion(n, remaining[0]);
+    auto [c, maxV] = getChildAndMaxVersion(n, remaining[0]);
+    Node *child = c;
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {
@@ -2675,7 +2748,7 @@ bool checkRangeRightSide(Node *n, std::span<const uint8_t> key, int prefixLen,
       return false;
     }
 
-    auto *child = getChild(n, remaining[0]);
+    Node *child = getChild(n, remaining[0]);
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {
@@ -2769,7 +2842,8 @@ bool checkRangeRead(Node *n, std::span<const uint8_t> begin,
     if (remaining.size() == 0) {
       break;
     }
-    auto [child, v] = getChildAndMaxVersion(n, remaining[0]);
+    auto [c, v] = getChildAndMaxVersion(n, remaining[0]);
+    Node *child = c;
     if (child == nullptr) {
       break;
     }
@@ -2835,8 +2909,10 @@ checkMaxBetweenExclusiveImpl<true>(Node *n, int begin, int end,
 // of the result will have `maxVersion` set to `writeVersion` as a
 // postcondition. Nodes along the search path may be invalidated. Callers must
 // ensure that the max version of the self argument is updated.
-[[nodiscard]] Node **insert(Node **self, std::span<const uint8_t> key,
-                            InternalVersionT writeVersion, WriteContext *tls) {
+[[nodiscard]] TaggedNodePointer *insert(TaggedNodePointer *self,
+                                        std::span<const uint8_t> key,
+                                        InternalVersionT writeVersion,
+                                        WriteContext *tls) {
 
   for (; key.size() != 0; ++tls->accum.insert_iterations) {
     self = &getOrCreateChild(*self, key, writeVersion, tls);
@@ -2864,17 +2940,23 @@ void eraseTree(Node *root, WriteContext *tls) {
     } break;
     case Type_Node3: {
       auto *n3 = static_cast<Node3 *>(n);
-      toFree.append(std::span<Node *>(n3->children, n3->numChildren));
+      for (int i = 0; i < n3->numChildren; ++i) {
+        toFree.push_back(n3->children[i]);
+      }
       tls->release(n3);
     } break;
     case Type_Node16: {
       auto *n16 = static_cast<Node16 *>(n);
-      toFree.append(std::span<Node *>(n16->children, n16->numChildren));
+      for (int i = 0; i < n16->numChildren; ++i) {
+        toFree.push_back(n16->children[i]);
+      }
       tls->release(n16);
     } break;
     case Type_Node48: {
       auto *n48 = static_cast<Node48 *>(n);
-      toFree.append(std::span<Node *>(n48->children, n48->numChildren));
+      for (int i = 0; i < n48->numChildren; ++i) {
+        toFree.push_back(n48->children[i]);
+      }
       tls->release(n48);
     } break;
     case Type_Node256: {
@@ -2890,10 +2972,10 @@ void eraseTree(Node *root, WriteContext *tls) {
   }
 }
 
-void addPointWrite(Node *&root, std::span<const uint8_t> key,
+void addPointWrite(TaggedNodePointer &root, std::span<const uint8_t> key,
                    InternalVersionT writeVersion, WriteContext *tls) {
   ++tls->accum.point_writes;
-  auto *n = *insert(&root, key, writeVersion, tls);
+  auto n = *insert(&root, key, writeVersion, tls);
   if (!n->entryPresent) {
     ++tls->accum.entries_inserted;
     auto *p = nextLogical(n);
@@ -3011,7 +3093,7 @@ void fixupMaxVersion(Node *node, WriteContext *tls) {
   setMaxVersion(node, max);
 }
 
-void addWriteRange(Node *&root, std::span<const uint8_t> begin,
+void addWriteRange(TaggedNodePointer &root, std::span<const uint8_t> begin,
                    std::span<const uint8_t> end, InternalVersionT writeVersion,
                    WriteContext *tls, ConflictSet::Impl *impl) {
 
@@ -3024,12 +3106,12 @@ void addWriteRange(Node *&root, std::span<const uint8_t> begin,
   ++tls->accum.range_writes;
   const bool beginIsPrefix = lcp == int(begin.size());
 
-  Node **useAsRoot = insert(&root, begin.subspan(0, lcp), writeVersion, tls);
+  auto useAsRoot = insert(&root, begin.subspan(0, lcp), writeVersion, tls);
 
   begin = begin.subspan(lcp, begin.size() - lcp);
   end = end.subspan(lcp, end.size() - lcp);
 
-  auto *beginNode = *insert(useAsRoot, begin, writeVersion, tls);
+  Node *beginNode = *insert(useAsRoot, begin, writeVersion, tls);
   addKey(beginNode);
   if (!beginNode->entryPresent) {
     ++tls->accum.entries_inserted;
@@ -3040,7 +3122,7 @@ void addWriteRange(Node *&root, std::span<const uint8_t> begin,
   }
   beginNode->entry.pointVersion = writeVersion;
 
-  auto *endNode = *insert(useAsRoot, end, writeVersion, tls);
+  Node *endNode = *insert(useAsRoot, end, writeVersion, tls);
   addKey(endNode);
   if (!endNode->entryPresent) {
     ++tls->accum.entries_inserted;
@@ -3060,7 +3142,7 @@ void addWriteRange(Node *&root, std::span<const uint8_t> begin,
   assert(!beginNode->endOfRange);
   assert(!endNode->endOfRange);
   endNode->endOfRange = true;
-  auto *iter = beginNode;
+  Node *iter = beginNode;
   for (iter = nextLogical(iter); !iter->endOfRange;
        iter = erase(iter, tls, impl, /*logical*/ true)) {
     assert(!iter->endOfRange);
@@ -3080,7 +3162,7 @@ Node *firstGeqPhysical(Node *n, const std::span<const uint8_t> key) {
       return n;
     }
 
-    auto *child = getChild(n, remaining[0]);
+    Node *child = getChild(n, remaining[0]);
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {
@@ -3126,34 +3208,308 @@ Node *firstGeqPhysical(Node *n, const std::span<const uint8_t> key) {
   }
 }
 
+#ifndef __has_attribute
+#define __has_attribute(x) 0
+#endif
+
+#if __has_attribute(musttail)
+#define MUSTTAIL __attribute__((musttail))
+#else
+#define MUSTTAIL
+#endif
+
+#if __has_attribute(preserve_none)
+#define PRESERVE_NONE __attribute__((preserve_none))
+#else
+#define PRESERVE_NONE
+#endif
+
+#if __has_attribute(flatten)
+#define FLATTEN __attribute__((flatten))
+#else
+#define FLATTEN
+#endif
+
+typedef PRESERVE_NONE void (*Continuation)(struct CheckJob *,
+                                           struct CheckContext *);
+
+// State relevant to an individual query
+struct CheckJob {
+  void setResult(bool ok) {
+    *result = ok ? ConflictSet::Commit : ConflictSet::Conflict;
+  }
+
+  void init(const ConflictSet::ReadRange *read, ConflictSet::Result *result,
+            Node *root, int64_t oldestVersionFullPrecision, ReadContext *tls);
+
+  Node *n;
+  std::span<const uint8_t> begin;
+  InternalVersionT readVersion;
+  ConflictSet::Result *result;
+  Continuation continuation;
+  CheckJob *prev;
+  CheckJob *next;
+};
+
+// State relevant to every query
+struct CheckContext {
+  int count;
+  int64_t oldestVersionFullPrecision;
+  Node *root;
+  const ConflictSet::ReadRange *queries;
+  ConflictSet::Result *results;
+  int64_t started;
+  ReadContext *tls;
+#if !__has_attribute(musttail)
+  CheckJob *job;
+  bool done;
+#endif
+};
+
+FLATTEN PRESERVE_NONE void keepGoing(CheckJob *job, CheckContext *context) {
+#if __has_attribute(musttail)
+  job = job->next;
+  MUSTTAIL return job->continuation(job, context);
+#else
+  context->job = job->next;
+  return;
+#endif
+}
+
+FLATTEN PRESERVE_NONE void complete(CheckJob *job, CheckContext *context) {
+  if (context->started == context->count) {
+    if (job->prev == job) {
+#if !__has_attribute(musttail)
+      context->done = true;
+#endif
+      return;
+    }
+    job->prev->next = job->next;
+    job->next->prev = job->prev;
+    job = job->prev;
+  } else {
+    int temp = context->started++;
+    job->init(context->queries + temp, context->results + temp, context->root,
+              context->oldestVersionFullPrecision, context->tls);
+  }
+  MUSTTAIL return keepGoing(job, context);
+}
+
+namespace check_point_read_state_machine {
+
+FLATTEN PRESERVE_NONE void begin(CheckJob *, CheckContext *);
+
+template <class NodeT>
+FLATTEN PRESERVE_NONE void iter(CheckJob *, CheckContext *);
+
+FLATTEN PRESERVE_NONE void down_left_spine(CheckJob *, CheckContext *);
+
+static Continuation iterTable[] = {iter<Node0>, iter<Node3>, iter<Node16>,
+                                   iter<Node48>, iter<Node256>};
+
+void begin(CheckJob *job, CheckContext *context) {
+  ++context->tls->point_read_accum;
+#if DEBUG_VERBOSE && !defined(NDEBUG)
+  fprintf(stderr, "Check point read: %s\n", printable(key).c_str());
+#endif
+
+  if (job->begin.size() == 0) [[unlikely]] {
+    // We don't erase the root
+    assert(job->n->entryPresent);
+    job->setResult(job->n->entry.pointVersion <= job->readVersion);
+    MUSTTAIL return complete(job, context);
+  }
+
+  auto taggedChild = getChild(job->n, job->begin[0]);
+  Node *child = taggedChild;
+  if (child == nullptr) [[unlikely]] {
+    auto c = getChildGeq(job->n, job->begin[0]);
+    if (c != nullptr) {
+      job->n = c;
+      job->continuation = down_left_spine;
+      __builtin_prefetch(job->n);
+      MUSTTAIL return keepGoing(job, context);
+    } else {
+      // The root never has a next sibling
+      job->setResult(true);
+      MUSTTAIL return complete(job, context);
+    }
+  }
+  job->continuation = iterTable[taggedChild.getType()];
+  job->n = child;
+  __builtin_prefetch(child);
+  MUSTTAIL return keepGoing(job, context);
+}
+
+template <class NodeT> void iter(CheckJob *job, CheckContext *context) {
+
+  assert(NodeT::kType == job->n->getType());
+  NodeT *n = static_cast<NodeT *>(job->n);
+  job->begin = job->begin.subspan(1, job->begin.size() - 1);
+
+  if (n->partialKeyLen > 0) {
+    int commonLen = std::min<int>(n->partialKeyLen, job->begin.size());
+    int i = longestCommonPrefix(n->partialKey(), job->begin.data(), commonLen);
+    if (i < commonLen) [[unlikely]] {
+      auto c = n->partialKey()[i] <=> job->begin[i];
+      if (c > 0) {
+        job->continuation = down_left_spine;
+        MUSTTAIL return down_left_spine(job, context);
+      } else {
+        job->n = nextSibling(n);
+        if (job->n == nullptr) {
+          job->setResult(true);
+          MUSTTAIL return complete(job, context);
+        }
+        job->continuation = down_left_spine;
+        __builtin_prefetch(job->n);
+        MUSTTAIL return keepGoing(job, context);
+      }
+    }
+    if (commonLen == n->partialKeyLen) {
+      // partial key matches
+      job->begin = job->begin.subspan(commonLen, job->begin.size() - commonLen);
+    } else if (n->partialKeyLen > int(job->begin.size())) [[unlikely]] {
+      // n is the first physical node greater than remaining, and there's no
+      // eq node
+      job->continuation = down_left_spine;
+      MUSTTAIL return down_left_spine(job, context);
+    }
+  }
+
+  ++context->tls->point_read_iterations_accum;
+
+  if (job->begin.size() == 0) [[unlikely]] {
+    if (n->entryPresent) {
+      job->setResult(n->entry.pointVersion <= job->readVersion);
+      MUSTTAIL return complete(job, context);
+    }
+    job->n = getFirstChildExists(n);
+    job->continuation = down_left_spine;
+    __builtin_prefetch(job->n);
+    MUSTTAIL return keepGoing(job, context);
+  }
+
+  auto taggedChild = getChild(n, job->begin[0]);
+  Node *child = taggedChild;
+  if (child == nullptr) [[unlikely]] {
+    auto c = getChildGeq(n, job->begin[0]);
+    if (c != nullptr) {
+      job->n = c;
+      job->continuation = down_left_spine;
+      __builtin_prefetch(job->n);
+      MUSTTAIL return keepGoing(job, context);
+    } else {
+      job->n = nextSibling(job->n);
+      if (job->n == nullptr) {
+        job->setResult(true);
+        MUSTTAIL return complete(job, context);
+      }
+      job->continuation = down_left_spine;
+      __builtin_prefetch(job->n);
+      MUSTTAIL return keepGoing(job, context);
+    }
+  }
+  job->continuation = iterTable[taggedChild.getType()];
+  job->n = child;
+  __builtin_prefetch(child);
+  MUSTTAIL return keepGoing(job, context);
+}
+
+void down_left_spine(CheckJob *job, CheckContext *context) {
+  if (job->n->entryPresent) {
+    job->setResult(job->n->entry.rangeVersion <= job->readVersion);
+    MUSTTAIL return complete(job, context);
+  }
+  job->n = getFirstChildExists(job->n);
+  __builtin_prefetch(job->n);
+  MUSTTAIL return keepGoing(job, context);
+}
+
+} // namespace check_point_read_state_machine
+
+void CheckJob::init(const ConflictSet::ReadRange *read,
+                    ConflictSet::Result *result, Node *root,
+                    int64_t oldestVersionFullPrecision, ReadContext *tls) {
+  auto begin = std::span<const uint8_t>(read->begin.p, read->begin.len);
+  auto end = std::span<const uint8_t>(read->end.p, read->end.len);
+  if (read->readVersion < oldestVersionFullPrecision) [[unlikely]] {
+    *result = ConflictSet::TooOld;
+    continuation = complete;
+  } else if (end.size() == 0) {
+    this->begin = begin;
+    this->n = root;
+    this->readVersion = InternalVersionT(read->readVersion);
+    this->result = result;
+    continuation = check_point_read_state_machine::begin;
+  } else {
+    *result = checkRangeRead(root, begin, end,
+                             InternalVersionT(read->readVersion), tls)
+                  ? ConflictSet::Commit
+                  : ConflictSet::Conflict;
+    continuation = complete;
+  }
+}
+
 struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
 
   void check(const ReadRange *reads, Result *result, int count) {
+    assert(oldestVersionFullPrecision >=
+           newestVersionFullPrecision - kNominalVersionWindow);
+
+    if (count == 0) {
+      return;
+    }
+
     ReadContext tls;
     tls.impl = this;
     int64_t check_byte_accum = 0;
+    constexpr int kConcurrent = 16;
+    CheckJob inProgress[kConcurrent];
+    CheckContext context;
+    context.count = count;
+    context.oldestVersionFullPrecision = oldestVersionFullPrecision;
+    context.root = root;
+    context.queries = reads;
+    context.results = result;
+    context.tls = &tls;
+    int64_t started = std::min(kConcurrent, count);
+    context.started = started;
+    for (int i = 0; i < started; i++) {
+      inProgress[i].init(reads + i, result + i, root,
+                         oldestVersionFullPrecision, &tls);
+    }
+    for (int i = 0; i < started - 1; i++) {
+      inProgress[i].next = inProgress + i + 1;
+    }
+    for (int i = 1; i < started; i++) {
+      inProgress[i].prev = inProgress + i - 1;
+    }
+    inProgress[0].prev = inProgress + started - 1;
+    inProgress[started - 1].next = inProgress;
+
+#if __has_attribute(musttail)
+    // Kick off the sequence of tail calls that finally returns once all jobs
+    // are done
+    inProgress->continuation(inProgress, &context);
+#else
+    context.job = inProgress;
+    context.done = false;
+    while (!context.done) {
+      context.job->continuation(context.job, &context);
+    }
+#endif
+
     for (int i = 0; i < count; ++i) {
       assert(reads[i].readVersion >= 0);
       assert(reads[i].readVersion <= newestVersionFullPrecision);
       const auto &r = reads[i];
       check_byte_accum += r.begin.len + r.end.len;
-      auto begin = std::span<const uint8_t>(r.begin.p, r.begin.len);
-      auto end = std::span<const uint8_t>(r.end.p, r.end.len);
-      assert(oldestVersionFullPrecision >=
-             newestVersionFullPrecision - kNominalVersionWindow);
-      result[i] =
-          reads[i].readVersion < oldestVersionFullPrecision ? TooOld
-          : (end.size() > 0
-                 ? checkRangeRead(root, begin, end,
-                                  InternalVersionT(reads[i].readVersion), &tls)
-                 : checkPointRead(root, begin,
-                                  InternalVersionT(reads[i].readVersion), &tls))
-              ? Commit
-              : Conflict;
       tls.commits_accum += result[i] == Commit;
       tls.conflicts_accum += result[i] == Conflict;
       tls.too_olds_accum += result[i] == TooOld;
     }
+
     point_read_total.add(tls.point_read_accum);
     prefix_read_total.add(tls.prefix_read_accum);
     range_read_total.add(tls.range_read_accum);
@@ -3375,7 +3731,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
   std::span<const uint8_t> removalKey;
   int64_t keyUpdates;
 
-  Node *root;
+  TaggedNodePointer root;
   InternalVersionT oldestVersion;
   int64_t oldestVersionFullPrecision;
   int64_t oldestExtantVersion;
@@ -3456,7 +3812,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
   }
 };
 
-Node *&getInTree(Node *n, ConflictSet::Impl *impl) {
+TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *impl) {
   return n->parent == nullptr ? impl->root
                               : getChildExists(n->parent, n->parentsIndex);
 }
@@ -3532,7 +3888,7 @@ Node *firstGeqLogical(Node *n, const std::span<const uint8_t> key) {
       goto downLeftSpine;
     }
 
-    auto *child = getChild(n, remaining[0]);
+    Node *child = getChild(n, remaining[0]);
     if (child == nullptr) {
       auto c = getChildGeq(n, remaining[0]);
       if (c != nullptr) {

Dockerfile

@@ -8,6 +8,7 @@ RUN chmod -R 777 /tmp
 RUN apt-get update
 RUN apt-get upgrade -y
 RUN TZ=America/Los_Angeles DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    binutils-aarch64-linux-gnu \
     build-essential \
     ccache \
     clang \

aarch64-toolchain.cmake

@@ -5,3 +5,4 @@ set(CMAKE_CXX_COMPILER "/usr/bin/aarch64-linux-gnu-g++")
 set(CMAKE_FIND_ROOT_PATH /usr/aarch64-linux-gnu)
 set(CMAKE_CROSSCOMPILING_EMULATOR "qemu-aarch64;-L;/usr/aarch64-linux-gnu/")
 set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE arm64)
+set(LD_EXE "/usr/bin/aarch64-linux-gnu-ld")