Add to corpus

ConflictSet.cpp

@@ -2147,7 +2147,14 @@ bool scan16(const InternalVersionT *vs, int begin, int end,
 // path of n + [child], where child in (begin, end) is <= readVersion. Does not
 // account for the range version of firstGt(searchpath(n) + [end - 1])
 template <bool kAVX512>
-bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
+bool checkMaxBetweenExclusiveImpl(Node0 *, int, int, InternalVersionT,
+                                  ReadContext *tls) {
+  ++tls->range_read_node_scan_accum;
+  return true;
+}
+
+template <bool kAVX512>
+bool checkMaxBetweenExclusiveImpl(Node3 *n, int begin, int end,
                                   InternalVersionT readVersion,
                                   ReadContext *tls) {
   ++tls->range_read_node_scan_accum;
@@ -2156,232 +2163,282 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
   assume(-1 <= end);
   assume(end <= 256);
   assume(begin < end);
-
   assert(!(begin == -1 && end == 256));
 
-  switch (n->getType()) {
-  case Type_Node0:
-    return true;
-  case Type_Node3: {
-    auto *self = static_cast<Node3 *>(n);
+  auto *self = static_cast<Node3 *>(n);
 
-    ++begin;
+  ++begin;
 
-    const unsigned shiftUpperBound = end - begin;
-    const unsigned shiftAmount = begin;
-    auto inBounds = [&](unsigned c) {
-      return c - shiftAmount < shiftUpperBound;
-    };
+  const unsigned shiftUpperBound = end - begin;
+  const unsigned shiftAmount = begin;
+  auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
 
-    uint32_t mask = 0;
-    for (int i = 0; i < Node3::kMaxNodes; ++i) {
-      mask |= inBounds(self->index[i]) << i;
-    }
-    mask &= (1 << self->numChildren) - 1;
-    if (!mask) {
-      return true;
-    }
-    Node *child = self->children[std::countr_zero(mask)];
-    const bool firstRangeOk =
-        !child->entryPresent || child->entry.rangeVersion <= readVersion;
-    uint32_t compared = 0;
-    for (int i = 0; i < Node3::kMaxNodes; ++i) {
-      compared |= (self->childMaxVersion[i] > readVersion) << i;
-    }
-
-    return !(compared & mask) && firstRangeOk;
+  uint32_t mask = 0;
+  for (int i = 0; i < Node3::kMaxNodes; ++i) {
+    mask |= inBounds(self->index[i]) << i;
+  }
+  mask &= (1 << self->numChildren) - 1;
+  if (!mask) {
+    return true;
+  }
+  Node *child = self->children[std::countr_zero(mask)];
+  const bool firstRangeOk =
+      !child->entryPresent || child->entry.rangeVersion <= readVersion;
+  uint32_t compared = 0;
+  for (int i = 0; i < Node3::kMaxNodes; ++i) {
+    compared |= (self->childMaxVersion[i] > readVersion) << i;
   }
-  case Type_Node16: {
-    auto *self = static_cast<Node16 *>(n);
 
-    ++begin;
+  return !(compared & mask) && firstRangeOk;
+}
 
-    assert(begin <= end);
-    assert(end - begin < 256);
+template <bool kAVX512>
+bool checkMaxBetweenExclusiveImpl(Node16 *n, int begin, int end,
+                                  InternalVersionT readVersion,
+                                  ReadContext *tls) {
+  ++tls->range_read_node_scan_accum;
+  assume(-1 <= begin);
+  assume(begin <= 256);
+  assume(-1 <= end);
+  assume(end <= 256);
+  assume(begin < end);
+  assert(!(begin == -1 && end == 256));
+
+  auto *self = static_cast<Node16 *>(n);
+
+  ++begin;
+
+  assert(begin <= end);
+  assert(end - begin < 256);
 
 #ifdef HAS_ARM_NEON
 
-    uint8x16_t indices;
-    memcpy(&indices, self->index, 16);
-    // 0xff for each in bounds
-    auto results =
-        vcltq_u8(vsubq_u8(indices, vdupq_n_u8(begin)), vdupq_n_u8(end - begin));
-    // 0xf for each 0xff
-    uint64_t mask = vget_lane_u64(
-        vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(results), 4)), 0);
+  uint8x16_t indices;
+  memcpy(&indices, self->index, 16);
+  // 0xff for each in bounds
+  auto results =
+      vcltq_u8(vsubq_u8(indices, vdupq_n_u8(begin)), vdupq_n_u8(end - begin));
+  // 0xf for each 0xff
+  uint64_t mask = vget_lane_u64(
+      vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(results), 4)), 0);
 
-    mask &= self->numChildren == 16
-                ? uint64_t(-1)
-                : (uint64_t(1) << (self->numChildren << 2)) - 1;
-    if (!mask) {
-      return true;
-    }
-    Node *child = self->children[std::countr_zero(mask) >> 2];
-    const bool firstRangeOk =
-        !child->entryPresent || child->entry.rangeVersion <= readVersion;
+  mask &= self->numChildren == 16
+              ? uint64_t(-1)
+              : (uint64_t(1) << (self->numChildren << 2)) - 1;
+  if (!mask) {
+    return true;
+  }
+  Node *child = self->children[std::countr_zero(mask) >> 2];
+  const bool firstRangeOk =
+      !child->entryPresent || child->entry.rangeVersion <= readVersion;
 
-    uint32x4_t w4[4];
-    memcpy(w4, self->childMaxVersion, sizeof(w4));
-    uint32_t rv;
-    memcpy(&rv, &readVersion, sizeof(rv));
-    const auto rvVec = vdupq_n_u32(rv);
+  uint32x4_t w4[4];
+  memcpy(w4, self->childMaxVersion, sizeof(w4));
+  uint32_t rv;
+  memcpy(&rv, &readVersion, sizeof(rv));
+  const auto rvVec = vdupq_n_u32(rv);
 
-    int32x4_t z;
-    memset(&z, 0, sizeof(z));
+  int32x4_t z;
+  memset(&z, 0, sizeof(z));
 
-    uint16x4_t conflicting[4];
-    for (int i = 0; i < 4; ++i) {
-      conflicting[i] = vmovn_u32(
-          vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
-    }
-    auto combined =
-        vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
-                    vmovn_u16(vcombine_u16(conflicting[2], conflicting[3])));
+  uint16x4_t conflicting[4];
+  for (int i = 0; i < 4; ++i) {
+    conflicting[i] =
+        vmovn_u32(vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
+  }
+  auto combined =
+      vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
+                  vmovn_u16(vcombine_u16(conflicting[2], conflicting[3])));
 
-    uint64_t compared = vget_lane_u64(
-        vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(combined), 4)), 0);
+  uint64_t compared = vget_lane_u64(
+      vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(combined), 4)), 0);
 
-    return !(compared & mask) && firstRangeOk;
+  return !(compared & mask) && firstRangeOk;
 
 #elif defined(HAS_AVX)
 
-    __m128i indices;
-    memcpy(&indices, self->index, 16);
-    indices = _mm_sub_epi8(indices, _mm_set1_epi8(begin));
-    uint32_t mask =
-        0xffff &
-        ~_mm_movemask_epi8(_mm_cmpeq_epi8(
-            indices, _mm_max_epu8(indices, _mm_set1_epi8(end - begin))));
-    mask &= (1 << self->numChildren) - 1;
-    if (!mask) {
-      return true;
-    }
-    Node *child = self->children[std::countr_zero(mask)];
-    const bool firstRangeOk =
-        !child->entryPresent || child->entry.rangeVersion <= readVersion;
+  __m128i indices;
+  memcpy(&indices, self->index, 16);
+  indices = _mm_sub_epi8(indices, _mm_set1_epi8(begin));
+  uint32_t mask =
+      0xffff & ~_mm_movemask_epi8(_mm_cmpeq_epi8(
+                   indices, _mm_max_epu8(indices, _mm_set1_epi8(end - begin))));
+  mask &= (1 << self->numChildren) - 1;
+  if (!mask) {
+    return true;
+  }
+  Node *child = self->children[std::countr_zero(mask)];
+  const bool firstRangeOk =
+      !child->entryPresent || child->entry.rangeVersion <= readVersion;
 
-    uint32_t compared = 0;
-    if constexpr (kAVX512) {
-      compared = compare16_avx512(self->childMaxVersion, readVersion);
-    } else {
-      compared = compare16(self->childMaxVersion, readVersion);
-    }
-    return !(compared & mask) && firstRangeOk;
+  uint32_t compared = 0;
+  if constexpr (kAVX512) {
+    compared = compare16_avx512(self->childMaxVersion, readVersion);
+  } else {
+    compared = compare16(self->childMaxVersion, readVersion);
+  }
+  return !(compared & mask) && firstRangeOk;
 
 #else
 
-    const unsigned shiftUpperBound = end - begin;
-    const unsigned shiftAmount = begin;
-    auto inBounds = [&](unsigned c) {
-      return c - shiftAmount < shiftUpperBound;
-    };
+  const unsigned shiftUpperBound = end - begin;
+  const unsigned shiftAmount = begin;
+  auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
 
-    uint32_t mask = 0;
-    for (int i = 0; i < 16; ++i) {
-      mask |= inBounds(self->index[i]) << i;
-    }
-    mask &= (1 << self->numChildren) - 1;
-    if (!mask) {
-      return true;
-    }
-    Node *child = self->children[std::countr_zero(mask)];
-    const bool firstRangeOk =
-        !child->entryPresent || child->entry.rangeVersion <= readVersion;
-    uint32_t compared = 0;
-    for (int i = 0; i < 16; ++i) {
-      compared |= (self->childMaxVersion[i] > readVersion) << i;
-    }
-    return !(compared & mask) && firstRangeOk;
-
-#endif
+  uint32_t mask = 0;
+  for (int i = 0; i < 16; ++i) {
+    mask |= inBounds(self->index[i]) << i;
   }
-  case Type_Node48: {
-    auto *self = static_cast<Node48 *>(n);
-
-    {
-      int c = self->bitSet.firstSetGeq(begin + 1);
-      if (c >= 0 && c < end) {
-        Node *child = self->children[self->index[c]];
-        if (child->entryPresent && child->entry.rangeVersion > readVersion) {
-          return false;
-        }
-        begin = c;
-      } else {
-        return true;
-      }
-      // [begin, end) is now the half-open interval of children we're interested
-      // in.
-      assert(begin < end);
-    }
-
-    // Check all pages
-    static_assert(Node48::kMaxOfMaxPageSize == 16);
-    for (int i = 0; i < Node48::kMaxOfMaxTotalPages; ++i) {
-      if (self->maxOfMax[i] > readVersion) {
-        if (!scan16<kAVX512>(self->childMaxVersion +
-                                 (i << Node48::kMaxOfMaxShift),
-                             self->reverseIndex + (i << Node48::kMaxOfMaxShift),
-                             begin, end, readVersion)) {
-          return false;
-        }
-      }
-    }
+  mask &= (1 << self->numChildren) - 1;
+  if (!mask) {
     return true;
   }
-  case Type_Node256: {
-    static_assert(Node256::kMaxOfMaxTotalPages == 16);
-    auto *self = static_cast<Node256 *>(n);
+  Node *child = self->children[std::countr_zero(mask)];
+  const bool firstRangeOk =
+      !child->entryPresent || child->entry.rangeVersion <= readVersion;
+  uint32_t compared = 0;
+  for (int i = 0; i < 16; ++i) {
+    compared |= (self->childMaxVersion[i] > readVersion) << i;
+  }
+  return !(compared & mask) && firstRangeOk;
 
-    {
-      int c = self->bitSet.firstSetGeq(begin + 1);
-      if (c >= 0 && c < end) {
-        Node *child = self->children[c];
-        if (child->entryPresent && child->entry.rangeVersion > readVersion) {
-          return false;
-        }
-        begin = c;
-      } else {
-        return true;
-      }
-      // [begin, end) is now the half-open interval of children we're interested
-      // in.
-      assert(begin < end);
-    }
+#endif
+}
 
-    const int firstPage = begin >> Node256::kMaxOfMaxShift;
-    const int lastPage = (end - 1) >> Node256::kMaxOfMaxShift;
-    // Check the only page if there's only one
-    if (firstPage == lastPage) {
-      if (self->maxOfMax[firstPage] <= readVersion) {
-        return true;
+template <bool kAVX512>
+bool checkMaxBetweenExclusiveImpl(Node48 *n, int begin, int end,
+                                  InternalVersionT readVersion,
+                                  ReadContext *tls) {
+  ++tls->range_read_node_scan_accum;
+  assume(-1 <= begin);
+  assume(begin <= 256);
+  assume(-1 <= end);
+  assume(end <= 256);
+  assume(begin < end);
+  assert(!(begin == -1 && end == 256));
+
+  auto *self = static_cast<Node48 *>(n);
+
+  {
+    int c = self->bitSet.firstSetGeq(begin + 1);
+    if (c >= 0 && c < end) {
+      Node *child = self->children[self->index[c]];
+      if (child->entryPresent && child->entry.rangeVersion > readVersion) {
+        return false;
       }
-      const int intraPageBegin = begin & (Node256::kMaxOfMaxPageSize - 1);
-      const int intraPageEnd = end - (lastPage << Node256::kMaxOfMaxShift);
-      return scan16<kAVX512>(self->childMaxVersion +
-                                 (firstPage << Node256::kMaxOfMaxShift),
-                             intraPageBegin, intraPageEnd, readVersion);
+      begin = c;
+    } else {
+      return true;
     }
-    // Check the first page
-    if (self->maxOfMax[firstPage] > readVersion) {
-      const int intraPageBegin = begin & (Node256::kMaxOfMaxPageSize - 1);
+    // [begin, end) is now the half-open interval of children we're interested
+    // in.
+    assert(begin < end);
+  }
+
+  // Check all pages
+  static_assert(Node48::kMaxOfMaxPageSize == 16);
+  for (int i = 0; i < Node48::kMaxOfMaxTotalPages; ++i) {
+    if (self->maxOfMax[i] > readVersion) {
       if (!scan16<kAVX512>(self->childMaxVersion +
+                               (i << Node48::kMaxOfMaxShift),
+                           self->reverseIndex + (i << Node48::kMaxOfMaxShift),
+                           begin, end, readVersion)) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+template <bool kAVX512>
+bool checkMaxBetweenExclusiveImpl(Node256 *n, int begin, int end,
+                                  InternalVersionT readVersion,
+                                  ReadContext *tls) {
+  ++tls->range_read_node_scan_accum;
+  assume(-1 <= begin);
+  assume(begin <= 256);
+  assume(-1 <= end);
+  assume(end <= 256);
+  assume(begin < end);
+  assert(!(begin == -1 && end == 256));
+
+  static_assert(Node256::kMaxOfMaxTotalPages == 16);
+  auto *self = static_cast<Node256 *>(n);
+
+  {
+    int c = self->bitSet.firstSetGeq(begin + 1);
+    if (c >= 0 && c < end) {
+      Node *child = self->children[c];
+      if (child->entryPresent && child->entry.rangeVersion > readVersion) {
+        return false;
+      }
+      begin = c;
+    } else {
+      return true;
+    }
+    // [begin, end) is now the half-open interval of children we're interested
+    // in.
+    assert(begin < end);
+  }
+
+  const int firstPage = begin >> Node256::kMaxOfMaxShift;
+  const int lastPage = (end - 1) >> Node256::kMaxOfMaxShift;
+  // Check the only page if there's only one
+  if (firstPage == lastPage) {
+    if (self->maxOfMax[firstPage] <= readVersion) {
+      return true;
+    }
+    const int intraPageBegin = begin & (Node256::kMaxOfMaxPageSize - 1);
+    const int intraPageEnd = end - (lastPage << Node256::kMaxOfMaxShift);
+    return scan16<kAVX512>(self->childMaxVersion +
                                (firstPage << Node256::kMaxOfMaxShift),
-                           intraPageBegin, 16, readVersion)) {
-        return false;
-      }
+                           intraPageBegin, intraPageEnd, readVersion);
+  }
+  // Check the first page
+  if (self->maxOfMax[firstPage] > readVersion) {
+    const int intraPageBegin = begin & (Node256::kMaxOfMaxPageSize - 1);
+    if (!scan16<kAVX512>(self->childMaxVersion +
+                             (firstPage << Node256::kMaxOfMaxShift),
+                         intraPageBegin, 16, readVersion)) {
+      return false;
     }
-    // Check the last page
-    if (self->maxOfMax[lastPage] > readVersion) {
-      const int intraPageEnd = end - (lastPage << Node256::kMaxOfMaxShift);
-      if (!scan16<kAVX512>(self->childMaxVersion +
-                               (lastPage << Node256::kMaxOfMaxShift),
-                           0, intraPageEnd, readVersion)) {
-        return false;
-      }
+  }
+  // Check the last page
+  if (self->maxOfMax[lastPage] > readVersion) {
+    const int intraPageEnd = end - (lastPage << Node256::kMaxOfMaxShift);
+    if (!scan16<kAVX512>(self->childMaxVersion +
+                             (lastPage << Node256::kMaxOfMaxShift),
+                         0, intraPageEnd, readVersion)) {
+      return false;
     }
-    // Check inner pages
-    return scan16<kAVX512>(self->maxOfMax, firstPage + 1, lastPage,
-                           readVersion);
+  }
+  // Check inner pages
+  return scan16<kAVX512>(self->maxOfMax, firstPage + 1, lastPage, readVersion);
+}
+
+template <bool kAVX512>
+bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
+                                  InternalVersionT readVersion,
+                                  ReadContext *tls) {
+  switch (n->getType()) {
+  case Type_Node0:
+    return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node0 *>(n), begin,
+                                                 end, readVersion, tls);
+  case Type_Node3: {
+    return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node3 *>(n), begin,
+                                                 end, readVersion, tls);
+  }
+  case Type_Node16: {
+    return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node16 *>(n),
+                                                 begin, end, readVersion, tls);
+  }
+  case Type_Node48: {
+    return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node48 *>(n),
+                                                 begin, end, readVersion, tls);
+  }
+  case Type_Node256: {
+    return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node256 *>(n),
+                                                 begin, end, readVersion, tls);
   }
   default:                   // GCOVR_EXCL_LINE
     __builtin_unreachable(); // GCOVR_EXCL_LINE
@@ -2402,6 +2459,58 @@ bool checkMaxBetweenExclusive(Node *n, int begin, int end,
   return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
 }
 
+bool checkMaxBetweenExclusive(Node0 *n, int begin, int end,
+                              InternalVersionT readVersion, ReadContext *tls) {
+  return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
+}
+
+bool checkMaxBetweenExclusive(Node3 *n, int begin, int end,
+                              InternalVersionT readVersion, ReadContext *tls) {
+  return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
+}
+
+#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
+__attribute__((target("avx512f"))) bool
+checkMaxBetweenExclusive(Node16 *n, int begin, int end,
+                         InternalVersionT readVersion, ReadContext *tls) {
+  return checkMaxBetweenExclusiveImpl<true>(n, begin, end, readVersion, tls);
+}
+__attribute__((target("default")))
+#endif
+
+bool checkMaxBetweenExclusive(Node16 *n, int begin, int end,
+                         InternalVersionT readVersion, ReadContext *tls) {
+  return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
+}
+
+#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
+__attribute__((target("avx512f"))) bool
+checkMaxBetweenExclusive(Node48 *n, int begin, int end,
+                         InternalVersionT readVersion, ReadContext *tls) {
+  return checkMaxBetweenExclusiveImpl<true>(n, begin, end, readVersion, tls);
+}
+__attribute__((target("default")))
+#endif
+
+bool checkMaxBetweenExclusive(Node48 *n, int begin, int end,
+                         InternalVersionT readVersion, ReadContext *tls) {
+  return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
+}
+
+#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
+__attribute__((target("avx512f"))) bool
+checkMaxBetweenExclusive(Node256 *n, int begin, int end,
+                         InternalVersionT readVersion, ReadContext *tls) {
+  return checkMaxBetweenExclusiveImpl<true>(n, begin, end, readVersion, tls);
+}
+__attribute__((target("default")))
+#endif
+
+bool checkMaxBetweenExclusive(Node256 *n, int begin, int end,
+                         InternalVersionT readVersion, ReadContext *tls) {
+  return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
+}
+
 Vector<uint8_t> getSearchPath(Arena &arena, Node *n) {
   assert(n != nullptr);
   auto result = vector<uint8_t>(arena);
@@ -2424,25 +2533,27 @@ Vector<uint8_t> getSearchPath(Arena &arena, Node *n) {
 //
 // Precondition: transitively, no child of n has a search path that's a longer
 // prefix of key than n
-bool checkRangeStartsWith(Node *n, std::span<const uint8_t> key, int begin,
-                          int end, InternalVersionT readVersion,
+template <class NodeT>
+bool checkRangeStartsWith(NodeT *nTyped, std::span<const uint8_t> key,
+                          int begin, int end, InternalVersionT readVersion,
                           ReadContext *tls) {
+  Node *n;
 #if DEBUG_VERBOSE && !defined(NDEBUG)
   fprintf(stderr, "%s(%02x,%02x)*\n", printable(key).c_str(), begin, end);
 #endif
   auto remaining = key;
   if (remaining.size() == 0) {
-    return checkMaxBetweenExclusive(n, begin, end, readVersion, tls);
+    return checkMaxBetweenExclusive(nTyped, begin, end, readVersion, tls);
   }
 
-  Node *child = getChild(n, remaining[0]);
+  Node *child = getChild(nTyped, remaining[0]);
   if (child == nullptr) {
-    auto c = getChildGeq(n, remaining[0]);
+    auto c = getChildGeq(nTyped, remaining[0]);
     if (c != nullptr) {
       n = c;
       goto downLeftSpine;
     } else {
-      n = nextSibling(n);
+      n = nextSibling(nTyped);
       if (n == nullptr) {
         return true;
       }
@@ -2501,6 +2612,15 @@ scan16<true>(const InternalVersionT *vs, int begin, int end,
 template __attribute__((target("avx512f"))) bool
 checkMaxBetweenExclusiveImpl<true>(Node *n, int begin, int end,
                                    InternalVersionT readVersion, ReadContext *);
+template __attribute__((target("avx512f"))) bool
+checkMaxBetweenExclusiveImpl<true>(Node16 *n, int begin, int end,
+                                   InternalVersionT readVersion, ReadContext *);
+template __attribute__((target("avx512f"))) bool
+checkMaxBetweenExclusiveImpl<true>(Node48 *n, int begin, int end,
+                                   InternalVersionT readVersion, ReadContext *);
+template __attribute__((target("avx512f"))) bool
+checkMaxBetweenExclusiveImpl<true>(Node256 *n, int begin, int end,
+                                   InternalVersionT readVersion, ReadContext *);
 #endif
 
 // Returns a pointer the pointer to the newly inserted node in the tree.  Caller
@@ -2858,7 +2978,7 @@ struct CheckContext {
   const ConflictSet::ReadRange *queries;
   ConflictSet::Result *results;
   int64_t started;
-  ReadContext *tls;
+  ReadContext tls;
 #if !__has_attribute(musttail)
   CheckJob *job;
   bool done;
@@ -2914,7 +3034,7 @@ static Continuation iterTable[] = {iter<Node0>, iter<Node3>, iter<Node16>,
                                    iter<Node48>, iter<Node256>};
 
 void begin(CheckJob *job, CheckContext *context) {
-  ++context->tls->point_read_accum;
+  ++context->tls.point_read_accum;
 #if DEBUG_VERBOSE && !defined(NDEBUG)
   fprintf(stderr, "Check point read: %s\n", printable(key).c_str());
 #endif
@@ -2984,11 +3104,11 @@ template <class NodeT> void iter(CheckJob *job, CheckContext *context) {
     }
   }
 
-  ++context->tls->point_read_iterations_accum;
+  ++context->tls.point_read_iterations_accum;
 
   if (job->maxV <= job->readVersion) {
     job->setResult(true);
-    ++context->tls->point_read_short_circuit_accum;
+    ++context->tls.point_read_short_circuit_accum;
     MUSTTAIL return complete(job, context);
   }
 
@@ -3042,7 +3162,7 @@ static Continuation iterTable[] = {iter<Node0>, iter<Node3>, iter<Node16>,
                                    iter<Node48>, iter<Node256>};
 
 void begin(CheckJob *job, CheckContext *context) {
-  ++context->tls->prefix_read_accum;
+  ++context->tls.prefix_read_accum;
 #if DEBUG_VERBOSE && !defined(NDEBUG)
   fprintf(stderr, "Check prefix read: %s\n", printable(key).c_str());
 #endif
@@ -3113,11 +3233,11 @@ template <class NodeT> void iter(CheckJob *job, CheckContext *context) {
     }
   }
 
-  ++context->tls->prefix_read_iterations_accum;
+  ++context->tls.prefix_read_iterations_accum;
 
   if (job->maxV <= job->readVersion) {
     job->setResult(true);
-    ++context->tls->prefix_read_short_circuit_accum;
+    ++context->tls.prefix_read_short_circuit_accum;
     MUSTTAIL return complete(job, context);
   }
 
@@ -3161,6 +3281,7 @@ PRESERVE_NONE void begin(CheckJob *, CheckContext *);
 template <class NodeT>
 PRESERVE_NONE void common_prefix_iter(CheckJob *, CheckContext *);
 
+template <class NodeT>
 PRESERVE_NONE void done_common_prefix_iter(CheckJob *, CheckContext *);
 
 static Continuation commonPrefixIterTable[] = {
@@ -3168,6 +3289,11 @@ static Continuation commonPrefixIterTable[] = {
     common_prefix_iter<Node16>, common_prefix_iter<Node48>,
     common_prefix_iter<Node256>};
 
+static Continuation doneCommonPrefixIterTable[] = {
+    done_common_prefix_iter<Node0>, done_common_prefix_iter<Node3>,
+    done_common_prefix_iter<Node16>, done_common_prefix_iter<Node48>,
+    done_common_prefix_iter<Node256>};
+
 template <class NodeT>
 PRESERVE_NONE void left_side_iter(CheckJob *, CheckContext *);
 
@@ -3203,18 +3329,20 @@ PRESERVE_NONE void begin(CheckJob *job, CheckContext *context) {
     MUSTTAIL return job->continuation(job, context);
   }
 
-  ++context->tls->range_read_accum;
+  ++context->tls.range_read_accum;
   job->remaining = job->begin.subspan(0, job->lcp);
 
   if (job->remaining.size() == 0) {
-    MUSTTAIL return done_common_prefix_iter(job, context);
+    job->continuation = doneCommonPrefixIterTable[job->n->getType()];
+    MUSTTAIL return job->continuation(job, context);
   }
 
   auto [c, maxV] = getChildAndMaxVersion(job->n, job->remaining[0]);
   job->maxV = maxV;
   job->child = c;
   if (job->child == nullptr) {
-    MUSTTAIL return done_common_prefix_iter(job, context);
+    job->continuation = doneCommonPrefixIterTable[job->n->getType()];
+    MUSTTAIL return job->continuation(job, context);
   }
 
   job->continuation = commonPrefixIterTable[c.getType()];
@@ -3234,7 +3362,8 @@ void common_prefix_iter(CheckJob *job, CheckContext *context) {
     int i =
         longestCommonPrefix(child->partialKey(), job->remaining.data() + 1, cl);
     if (i != child->partialKeyLen) {
-      MUSTTAIL return done_common_prefix_iter(job, context);
+      job->continuation = doneCommonPrefixIterTable[job->n->getType()];
+      MUSTTAIL return job->continuation(job, context);
     }
   }
   job->n = child;
@@ -3242,23 +3371,25 @@ void common_prefix_iter(CheckJob *job, CheckContext *context) {
                                           job->remaining.size() -
                                               (1 + child->partialKeyLen));
 
-  ++context->tls->range_read_iterations_accum;
+  ++context->tls.range_read_iterations_accum;
 
   if (job->maxV <= job->readVersion) {
     job->setResult(true);
-    ++context->tls->range_read_short_circuit_accum;
+    ++context->tls.range_read_short_circuit_accum;
     MUSTTAIL return complete(job, context);
   }
 
   if (job->remaining.size() == 0) {
-    MUSTTAIL return done_common_prefix_iter(job, context);
+    job->continuation = done_common_prefix_iter<NodeT>;
+    MUSTTAIL return job->continuation(job, context);
   }
 
   auto [c, maxV] = getChildAndMaxVersion(child, job->remaining[0]);
   job->maxV = maxV;
   job->child = c;
   if (job->child == nullptr) {
-    MUSTTAIL return done_common_prefix_iter(job, context);
+    job->continuation = done_common_prefix_iter<NodeT>;
+    MUSTTAIL return job->continuation(job, context);
   }
 
   job->continuation = commonPrefixIterTable[c.getType()];
@@ -3266,12 +3397,15 @@ void common_prefix_iter(CheckJob *job, CheckContext *context) {
   MUSTTAIL return keepGoing(job, context);
 }
 
+template <class NodeT>
 PRESERVE_NONE void done_common_prefix_iter(CheckJob *job,
                                            CheckContext *context) {
+  assert(NodeT::kType == job->n->getType());
+  NodeT *n = static_cast<NodeT *>(job->n);
 
   {
     Arena arena;
-    assert(getSearchPath(arena, job->n) <=>
+    assert(getSearchPath(arena, n) <=>
                job->begin.subspan(0, job->lcp - job->remaining.size()) ==
            0);
   }
@@ -3282,19 +3416,18 @@ PRESERVE_NONE void done_common_prefix_iter(CheckJob *job,
   job->begin = job->begin.subspan(consumed, int(job->begin.size()) - consumed);
   job->end = job->end.subspan(consumed, int(job->end.size()) - consumed);
   job->lcp -= consumed;
-  job->commonPrefixNode = job->n;
+  job->commonPrefixNode = n;
 
   if (job->lcp == int(job->begin.size())) {
     job->remaining = job->end;
     if (job->lcp == 0) {
-      if (job->n->entryPresent &&
-          job->n->entry.pointVersion > job->readVersion) {
+      if (n->entryPresent && n->entry.pointVersion > job->readVersion) {
         job->setResult(false);
         MUSTTAIL return complete(job, context);
       }
 
-      if (!checkMaxBetweenExclusive(job->n, -1, job->remaining[0],
-                                    job->readVersion, context->tls)) {
+      if (!checkMaxBetweenExclusive(n, -1, job->remaining[0], job->readVersion,
+                                    &context->tls)) {
         job->setResult(false);
         MUSTTAIL return complete(job, context);
       }
@@ -3303,10 +3436,10 @@ PRESERVE_NONE void done_common_prefix_iter(CheckJob *job,
     // This is a hack
     --job->lcp;
 
-    auto c = getChild(job->n, job->remaining[0]);
+    auto c = getChild(n, job->remaining[0]);
     Node *child = c;
     if (child == nullptr) {
-      auto c = getChildGeq(job->n, job->remaining[0]);
+      auto c = getChildGeq(n, job->remaining[0]);
       if (c != nullptr) {
         job->n = c;
         job->continuation = down_left_spine;
@@ -3331,20 +3464,20 @@ PRESERVE_NONE void done_common_prefix_iter(CheckJob *job,
   // If this were not true we would have returned above
   assert(job->begin.size() > 0);
 
-  if (!checkRangeStartsWith(job->n, job->begin.subspan(0, job->lcp),
+  if (!checkRangeStartsWith(n, job->begin.subspan(0, job->lcp),
                             job->begin[job->lcp], job->end[job->lcp],
-                            job->readVersion, context->tls)) {
+                            job->readVersion, &context->tls)) {
     job->setResult(false);
     MUSTTAIL return complete(job, context);
   }
 
   job->remaining = job->begin;
 
-  auto [c, maxV] = getChildAndMaxVersion(job->n, job->remaining[0]);
+  auto [c, maxV] = getChildAndMaxVersion(n, job->remaining[0]);
   job->maxV = maxV;
   Node *child = c;
   if (child == nullptr) {
-    auto c = getChildGeq(job->n, job->remaining[0]);
+    auto c = getChildGeq(n, job->remaining[0]);
     if (c != nullptr) {
       job->n = c;
       job->continuation = left_side_down_left_spine;
@@ -3428,7 +3561,7 @@ PRESERVE_NONE void left_side_iter(CheckJob *job, CheckContext *context) {
     }
   }
 
-  ++context->tls->range_read_iterations_accum;
+  ++context->tls.range_read_iterations_accum;
 
   if (job->maxV <= job->readVersion) {
     job->continuation = done_left_side_iter;
@@ -3442,7 +3575,7 @@ PRESERVE_NONE void left_side_iter(CheckJob *job, CheckContext *context) {
   }
 
   if (!checkMaxBetweenExclusive(n, job->remaining[0], 256, job->readVersion,
-                                context->tls)) {
+                                &context->tls)) {
     job->setResult(false);
     MUSTTAIL return complete(job, context);
   }
@@ -3566,7 +3699,7 @@ PRESERVE_NONE void right_side_iter(CheckJob *job, CheckContext *context) {
     }
   }
 
-  ++context->tls->range_read_iterations_accum;
+  ++context->tls.range_read_iterations_accum;
 
   if (job->remaining.size() == 0) {
     job->continuation = down_left_spine;
@@ -3580,7 +3713,7 @@ PRESERVE_NONE void right_side_iter(CheckJob *job, CheckContext *context) {
   }
 
   if (!checkMaxBetweenExclusive(n, -1, job->remaining[0], job->readVersion,
-                                context->tls)) {
+                                &context->tls)) {
     job->setResult(false);
     MUSTTAIL return complete(job, context);
   }
@@ -3646,18 +3779,16 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
       return;
     }
 
-    ReadContext tls;
-    tls.impl = this;
     int64_t check_byte_accum = 0;
     constexpr int kConcurrent = 16;
     CheckJob inProgress[kConcurrent];
     CheckContext context;
+    context.tls.impl = this;
     context.count = count;
     context.oldestVersionFullPrecision = oldestVersionFullPrecision;
     context.root = root;
     context.queries = reads;
     context.results = result;
-    context.tls = &tls;
     int64_t started = std::min(kConcurrent, count);
     context.started = started;
     for (int i = 0; i < started; i++) {
@@ -3690,24 +3821,27 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
       assert(reads[i].readVersion <= newestVersionFullPrecision);
       const auto &r = reads[i];
       check_byte_accum += r.begin.len + r.end.len;
-      tls.commits_accum += result[i] == Commit;
-      tls.conflicts_accum += result[i] == Conflict;
-      tls.too_olds_accum += result[i] == TooOld;
+      context.tls.commits_accum += result[i] == Commit;
+      context.tls.conflicts_accum += result[i] == Conflict;
+      context.tls.too_olds_accum += result[i] == TooOld;
     }
 
-    point_read_total.add(tls.point_read_accum);
-    prefix_read_total.add(tls.prefix_read_accum);
-    range_read_total.add(tls.range_read_accum);
-    range_read_node_scan_total.add(tls.range_read_node_scan_accum);
-    point_read_short_circuit_total.add(tls.point_read_short_circuit_accum);
-    prefix_read_short_circuit_total.add(tls.prefix_read_short_circuit_accum);
-    range_read_short_circuit_total.add(tls.range_read_short_circuit_accum);
-    point_read_iterations_total.add(tls.point_read_iterations_accum);
-    prefix_read_iterations_total.add(tls.prefix_read_iterations_accum);
-    range_read_iterations_total.add(tls.range_read_iterations_accum);
-    commits_total.add(tls.commits_accum);
-    conflicts_total.add(tls.conflicts_accum);
-    too_olds_total.add(tls.too_olds_accum);
+    point_read_total.add(context.tls.point_read_accum);
+    prefix_read_total.add(context.tls.prefix_read_accum);
+    range_read_total.add(context.tls.range_read_accum);
+    range_read_node_scan_total.add(context.tls.range_read_node_scan_accum);
+    point_read_short_circuit_total.add(
+        context.tls.point_read_short_circuit_accum);
+    prefix_read_short_circuit_total.add(
+        context.tls.prefix_read_short_circuit_accum);
+    range_read_short_circuit_total.add(
+        context.tls.range_read_short_circuit_accum);
+    point_read_iterations_total.add(context.tls.point_read_iterations_accum);
+    prefix_read_iterations_total.add(context.tls.prefix_read_iterations_accum);
+    range_read_iterations_total.add(context.tls.range_read_iterations_accum);
+    commits_total.add(context.tls.commits_accum);
+    conflicts_total.add(context.tls.conflicts_accum);
+    too_olds_total.add(context.tls.too_olds_accum);
     check_bytes_total.add(check_byte_accum);
   }