Update corpus

This is the default now

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.18)
 project(
   conflict-set
-  VERSION 0.0.7
+  VERSION 0.0.8
   DESCRIPTION
     "A data structure for optimistic concurrency control on ranges of bitwise-lexicographically-ordered keys."
   HOMEPAGE_URL "https://git.weaselab.dev/weaselab/conflict-set"
@@ -59,10 +59,6 @@ cmake_pop_check_state()
 
 option(USE_SIMD_FALLBACK
        "Use fallback implementations of functions that use SIMD" OFF)
-option(
-  USE_32_BIT_VERSIONS
-  "Store 32 bit versions internally, and rely on versions never being different by more than 2e9"
-  OFF)
 
 # This is encouraged according to
 # https://valgrind.org/docs/manual/manual-core-adv.html#manual-core-adv.clientreq
@@ -107,10 +103,6 @@ if(NOT USE_SIMD_FALLBACK)
   endif()
 endif()
 
-if(USE_32_BIT_VERSIONS)
-  add_compile_definitions(INTERNAL_VERSION_32_BIT=1)
-endif()
-
 set(CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "")
 
 add_library(${PROJECT_NAME}-object OBJECT ConflictSet.cpp)

ConflictSet.cpp

@@ -78,18 +78,18 @@ constexpr void removeKey(struct Node *) {}
 
 // ==================== BEGIN IMPLEMENTATION ====================
 
-#ifndef INTERNAL_VERSION_32_BIT
-#define INTERNAL_VERSION_32_BIT 0
-#endif
+constexpr int64_t kNominalVersionWindow = 2e9;
+constexpr int64_t kMaxCorrectVersionWindow =
+    std::numeric_limits<int32_t>::max();
+static_assert(kNominalVersionWindow <= kMaxCorrectVersionWindow);
 
-#if INTERNAL_VERSION_32_BIT
 struct InternalVersionT {
   constexpr InternalVersionT() = default;
   constexpr explicit InternalVersionT(int64_t value) : value(value) {}
   constexpr int64_t toInt64() const { return value; } // GCOVR_EXCL_LINE
   constexpr auto operator<=>(const InternalVersionT &rhs) const {
     // Maintains ordering after overflow, as long as the full-precision versions
-    // are within ~2e9 of eachother.
+    // are within `kMaxCorrectVersionWindow` of eachother.
     return int32_t(value - rhs.value) <=> 0;
   }
   constexpr bool operator==(const InternalVersionT &) const = default;
@@ -99,20 +99,6 @@ private:
   uint32_t value;
 };
 thread_local InternalVersionT InternalVersionT::zero;
-#else
-struct InternalVersionT {
-  constexpr InternalVersionT() = default;
-  constexpr explicit InternalVersionT(int64_t value) : value(value) {}
-  constexpr int64_t toInt64() const { return value; } // GCOVR_EXCL_LINE
-  constexpr auto operator<=>(const InternalVersionT &rhs) const = default;
-  constexpr bool operator==(const InternalVersionT &) const = default;
-  static const InternalVersionT zero;
-
-private:
-  int64_t value;
-};
-const InternalVersionT InternalVersionT::zero{0};
-#endif
 
 struct Entry {
   InternalVersionT pointVersion;
@@ -527,13 +513,8 @@ std::string getSearchPath(Node *n);
 // Each node with an entry present gets a budget of kBytesPerKey. Node0 always
 // has an entry present.
 // Induction hypothesis is that each node's surplus is >= kMinNodeSurplus
-#if INTERNAL_VERSION_32_BIT
 constexpr int kBytesPerKey = 112;
 constexpr int kMinNodeSurplus = 80;
-#else
-constexpr int kBytesPerKey = 144;
-constexpr int kMinNodeSurplus = 104;
-#endif
 constexpr int kMinChildrenNode3 = 2;
 constexpr int kMinChildrenNode16 = 4;
 constexpr int kMinChildrenNode48 = 17;
@@ -625,6 +606,14 @@ template <class T> struct BoundedFreeListAllocator {
     VALGRIND_MAKE_MEM_NOACCESS(freeList, sizeof(T) + p->partialKeyCapacity);
   }
 
+  BoundedFreeListAllocator() = default;
+
+  BoundedFreeListAllocator(const BoundedFreeListAllocator &) = delete;
+  BoundedFreeListAllocator &
+  operator=(const BoundedFreeListAllocator &) = delete;
+  BoundedFreeListAllocator(BoundedFreeListAllocator &&) = delete;
+  BoundedFreeListAllocator &operator=(BoundedFreeListAllocator &&) = delete;
+
   ~BoundedFreeListAllocator() {
     for (void *iter = freeList; iter != nullptr;) {
       VALGRIND_MAKE_MEM_DEFINED(iter, sizeof(Node));
@@ -1230,6 +1219,53 @@ void maybeDecreaseCapacity(Node *&self, NodeAllocators *allocators,
   freeAndMakeCapacityAtLeast(self, maxCapacity, allocators, impl, false);
 }
 
+void rezero(Node *n, InternalVersionT z) {
+#if DEBUG_VERBOSE && !defined(NDEBUG)
+  fprintf(stderr, "rezero to %" PRId64 ": %s\n", z.toInt64(),
+          getSearchPathPrintable(n).c_str());
+#endif
+  if (n->entryPresent) {
+    n->entry.pointVersion = std::max(n->entry.pointVersion, z);
+    n->entry.rangeVersion = std::max(n->entry.rangeVersion, z);
+  }
+  switch (n->getType()) {
+  case Type_Node0: {
+  } break;
+  case Type_Node3: {
+    auto *self = static_cast<Node3 *>(n);
+    for (int i = 0; i < 3; ++i) {
+      self->childMaxVersion[i] = std::max(self->childMaxVersion[i], z);
+    }
+  } break;
+  case Type_Node16: {
+    auto *self = static_cast<Node16 *>(n);
+    for (int i = 0; i < 16; ++i) {
+      self->childMaxVersion[i] = std::max(self->childMaxVersion[i], z);
+    }
+  } break;
+  case Type_Node48: {
+    auto *self = static_cast<Node48 *>(n);
+    for (int i = 0; i < 48; ++i) {
+      self->childMaxVersion[i] = std::max(self->childMaxVersion[i], z);
+    }
+    for (auto &m : self->maxOfMax) {
+      m = std::max(m, z);
+    }
+  } break;
+  case Type_Node256: {
+    auto *self = static_cast<Node256 *>(n);
+    for (int i = 0; i < 256; ++i) {
+      self->childMaxVersion[i] = std::max(self->childMaxVersion[i], z);
+    }
+    for (auto &m : self->maxOfMax) {
+      m = std::max(m, z);
+    }
+  } break;
+  default:                   // GCOVR_EXCL_LINE
+    __builtin_unreachable(); // GCOVR_EXCL_LINE
+  }
+}
+
 void maybeDownsize(Node *self, NodeAllocators *allocators,
                    ConflictSet::Impl *impl, Node *&dontInvalidate) {
 
@@ -1281,6 +1317,9 @@ void maybeDownsize(Node *self, NodeAllocators *allocators,
       // Max versions are stored in the parent, so we need to update it now
       // that we have a new parent.
       setMaxVersion(child, impl, childMaxVersion);
+      if (child->parent) {
+        rezero(child->parent, InternalVersionT::zero);
+      }
 
       getInTree(self, impl) = child;
       allocators->node3.release(self3);
@@ -1323,7 +1362,7 @@ void maybeDownsize(Node *self, NodeAllocators *allocators,
 // the node after self. If erase invalidates the pointee of `dontInvalidate`, it
 // will update it to its new pointee as well.
 Node *erase(Node *self, NodeAllocators *allocators, ConflictSet::Impl *impl,
-            Node *&dontInvalidate) {
+            bool logical, Node *&dontInvalidate) {
   assert(self->parent != nullptr);
 
 #if DEBUG_VERBOSE && !defined(NDEBUG)
@@ -1333,7 +1372,7 @@ Node *erase(Node *self, NodeAllocators *allocators, ConflictSet::Impl *impl,
   Node *parent = self->parent;
   uint8_t parentsIndex = self->parentsIndex;
 
-  auto *result = nextLogical(self);
+  auto *result = logical ? nextLogical(self) : nextPhysical(self);
 
   removeKey(self);
   self->entryPresent = false;
@@ -1833,7 +1872,6 @@ bool scan16(const InternalVersionT *vs, const uint8_t *is, int begin, int end,
   uint64_t mask = vget_lane_u64(
       vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(results), 4)), 0);
 
-#if INTERNAL_VERSION_32_BIT
   uint32x4_t w4[4];
   memcpy(w4, vs, sizeof(w4));
   uint32_t rv;
@@ -1845,7 +1883,8 @@ bool scan16(const InternalVersionT *vs, const uint8_t *is, int begin, int end,
 
   uint16x4_t conflicting[4];
   for (int i = 0; i < 4; ++i) {
-    conflicting[i] = vmovn_u32(vcgtq_s32(vsubq_u32(w4[i], rvVec), z));
+    conflicting[i] =
+        vmovn_u32(vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
   }
   auto combined =
       vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
@@ -1853,12 +1892,6 @@ bool scan16(const InternalVersionT *vs, const uint8_t *is, int begin, int end,
 
   uint64_t compared = vget_lane_u64(
       vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(combined), 4)), 0);
-#else
-  uint64_t compared = 0;
-  for (int i = 0; i < 16; ++i) {
-    compared |= uint64_t(vs[i] > readVersion) << (i << 2);
-  }
-#endif
 
   return !(compared & mask);
 
@@ -1871,17 +1904,11 @@ bool scan16(const InternalVersionT *vs, const uint8_t *is, int begin, int end,
       indices, _mm_max_epu8(indices, _mm_set1_epi8(end - begin))));
 
   uint32_t compared = 0;
-#if INTERNAL_VERSION_32_BIT
   if constexpr (kAVX512) {
     compared = compare16_32bit_avx512(vs, readVersion);
   } else {
     compared = compare16_32bit(vs, readVersion);
   }
-#else
-  for (int i = 0; i < 16; ++i) {
-    compared |= (vs[i] > readVersion) << i;
-  }
-#endif
   return !(compared & mask);
 
 #else
@@ -1914,7 +1941,7 @@ scan16(const InternalVersionT *vs, int begin, int end,
   assert(0 <= end && end <= 16);
   assert(begin <= end);
 
-#if INTERNAL_VERSION_32_BIT && defined(HAS_ARM_NEON)
+#if defined(HAS_ARM_NEON)
   uint32x4_t w4[4];
   memcpy(w4, vs, sizeof(w4));
   uint32_t rv;
@@ -1926,7 +1953,8 @@ scan16(const InternalVersionT *vs, int begin, int end,
 
   uint16x4_t conflicting[4];
   for (int i = 0; i < 4; ++i) {
-    conflicting[i] = vmovn_u32(vcgtq_s32(vsubq_u32(w4[i], rvVec), z));
+    conflicting[i] =
+        vmovn_u32(vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
   }
   auto combined =
       vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
@@ -1938,7 +1966,7 @@ scan16(const InternalVersionT *vs, int begin, int end,
   conflict &= end == 16 ? -1 : (uint64_t(1) << (end << 2)) - 1;
   conflict >>= begin << 2;
   return !conflict;
-#elif INTERNAL_VERSION_32_BIT && defined(HAS_AVX)
+#elif defined(HAS_AVX)
   uint32_t conflict;
   if constexpr (kAVX512) {
     conflict = compare16_32bit_avx512(vs, readVersion);
@@ -1973,55 +2001,163 @@ bool checkMaxBetweenExclusive(Node *n, int begin, int end,
 
   assert(!(begin == -1 && end == 256));
 
-  {
-    int c = getChildGeq(n, begin + 1);
-    if (c >= 0 && c < end) {
-      auto *child = getChildExists(n, c);
-      if (child->entryPresent) {
-        if (!(child->entry.rangeVersion <= readVersion)) {
-          return false;
-        };
-      }
-      begin = c;
-    } else {
-      return true;
-    }
-  }
-
-  // [begin, end) is now the half-open interval of children we're interested in.
-  assert(begin < end);
-
   switch (n->getType()) {
-  case Type_Node0: // GCOVR_EXCL_LINE
-    // We would have returned above, after not finding a child
-    __builtin_unreachable(); // GCOVR_EXCL_LINE
+  case Type_Node0:
+    return true;
   case Type_Node3: {
     auto *self = static_cast<Node3 *>(n);
 
+    ++begin;
+
     const unsigned shiftUpperBound = end - begin;
     const unsigned shiftAmount = begin;
     auto inBounds = [&](unsigned c) {
       return c - shiftAmount < shiftUpperBound;
     };
 
-    uint32_t compared = 0;
-    for (int i = 0; i < Node3::kMaxNodes; ++i) {
-      compared |= (self->childMaxVersion[i] > readVersion) << i;
-    }
     uint32_t mask = 0;
     for (int i = 0; i < Node3::kMaxNodes; ++i) {
       mask |= inBounds(self->index[i]) << i;
     }
-    return !(compared & mask);
+    mask &= (1 << self->numChildren) - 1;
+    if (!mask) {
+      return true;
+    }
+    auto *child = self->children[__builtin_ctz(mask)];
+    const bool firstRangeOk =
+        !child->entryPresent || child->entry.rangeVersion <= readVersion;
+    uint32_t compared = 0;
+    for (int i = 0; i < Node3::kMaxNodes; ++i) {
+      compared |= (self->childMaxVersion[i] > readVersion) << i;
+    }
+
+    return !(compared & mask) && firstRangeOk;
   }
   case Type_Node16: {
     auto *self = static_cast<Node16 *>(n);
 
-    return scan16<kAVX512>(self->childMaxVersion, self->index, begin, end,
-                           readVersion);
+    ++begin;
+
+    assert(begin <= end);
+    assert(end - begin < 256);
+
+#ifdef HAS_ARM_NEON
+
+    uint8x16_t indices;
+    memcpy(&indices, self->index, 16);
+    // 0xff for each in bounds
+    auto results =
+        vcltq_u8(vsubq_u8(indices, vdupq_n_u8(begin)), vdupq_n_u8(end - begin));
+    // 0xf for each 0xff
+    uint64_t mask = vget_lane_u64(
+        vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(results), 4)), 0);
+
+    mask &= self->numChildren == 16
+                ? uint64_t(-1)
+                : (uint64_t(1) << (self->numChildren << 2)) - 1;
+    if (!mask) {
+      return true;
+    }
+    auto *child = self->children[__builtin_ctzll(mask) >> 2];
+    const bool firstRangeOk =
+        !child->entryPresent || child->entry.rangeVersion <= readVersion;
+
+    uint32x4_t w4[4];
+    memcpy(w4, self->childMaxVersion, sizeof(w4));
+    uint32_t rv;
+    memcpy(&rv, &readVersion, sizeof(rv));
+    const auto rvVec = vdupq_n_u32(rv);
+
+    int32x4_t z;
+    memset(&z, 0, sizeof(z));
+
+    uint16x4_t conflicting[4];
+    for (int i = 0; i < 4; ++i) {
+      conflicting[i] = vmovn_u32(
+          vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
+    }
+    auto combined =
+        vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
+                    vmovn_u16(vcombine_u16(conflicting[2], conflicting[3])));
+
+    uint64_t compared = vget_lane_u64(
+        vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(combined), 4)), 0);
+
+    return !(compared & mask) && firstRangeOk;
+
+#elif defined(HAS_AVX)
+
+    __m128i indices;
+    memcpy(&indices, self->index, 16);
+    indices = _mm_sub_epi8(indices, _mm_set1_epi8(begin));
+    uint32_t mask =
+        0xffff &
+        ~_mm_movemask_epi8(_mm_cmpeq_epi8(
+            indices, _mm_max_epu8(indices, _mm_set1_epi8(end - begin))));
+    mask &= (1 << self->numChildren) - 1;
+    if (!mask) {
+      return true;
+    }
+    auto *child = self->children[__builtin_ctz(mask)];
+    const bool firstRangeOk =
+        !child->entryPresent || child->entry.rangeVersion <= readVersion;
+
+    uint32_t compared = 0;
+    if constexpr (kAVX512) {
+      compared = compare16_32bit_avx512(self->childMaxVersion, readVersion);
+    } else {
+      compared = compare16_32bit(self->childMaxVersion, readVersion);
+    }
+    return !(compared & mask) && firstRangeOk;
+
+#else
+
+    const unsigned shiftUpperBound = end - begin;
+    const unsigned shiftAmount = begin;
+    auto inBounds = [&](unsigned c) {
+      return c - shiftAmount < shiftUpperBound;
+    };
+
+    uint32_t mask = 0;
+    for (int i = 0; i < 16; ++i) {
+      mask |= inBounds(self->index[i]) << i;
+    }
+    mask &= (1 << self->numChildren) - 1;
+    if (!mask) {
+      return true;
+    }
+    auto *child = self->children[__builtin_ctz(mask)];
+    const bool firstRangeOk =
+        !child->entryPresent || child->entry.rangeVersion <= readVersion;
+    uint32_t compared = 0;
+    for (int i = 0; i < 16; ++i) {
+      compared |= (self->childMaxVersion[i] > readVersion) << i;
+    }
+    return !(compared & mask) && firstRangeOk;
+
+#endif
   }
   case Type_Node48: {
     auto *self = static_cast<Node48 *>(n);
+
+    {
+      int c = self->bitSet.firstSetGeq(begin + 1);
+      if (c >= 0 && c < end) {
+        auto *child = self->children[self->index[c]];
+        if (child->entryPresent) {
+          if (!(child->entry.rangeVersion <= readVersion)) {
+            return false;
+          };
+        }
+        begin = c;
+      } else {
+        return true;
+      }
+      // [begin, end) is now the half-open interval of children we're interested
+      // in.
+      assert(begin < end);
+    }
+
     // Check all pages
     static_assert(Node48::kMaxOfMaxPageSize == 16);
     for (int i = 0; i < Node48::kMaxOfMaxTotalPages; ++i) {
@@ -2039,6 +2175,25 @@ bool checkMaxBetweenExclusive(Node *n, int begin, int end,
   case Type_Node256: {
     static_assert(Node256::kMaxOfMaxTotalPages == 16);
     auto *self = static_cast<Node256 *>(n);
+
+    {
+      int c = self->bitSet.firstSetGeq(begin + 1);
+      if (c >= 0 && c < end) {
+        auto *child = self->children[c];
+        if (child->entryPresent) {
+          if (!(child->entry.rangeVersion <= readVersion)) {
+            return false;
+          };
+        }
+        begin = c;
+      } else {
+        return true;
+      }
+      // [begin, end) is now the half-open interval of children we're interested
+      // in.
+      assert(begin < end);
+    }
+
     const int firstPage = begin >> Node256::kMaxOfMaxShift;
     const int lastPage = (end - 1) >> Node256::kMaxOfMaxShift;
     // Check the only page if there's only one
@@ -2678,9 +2833,9 @@ void destroyTree(Node *root) {
   }
 }
 
-void addPointWrite(Node *&root, InternalVersionT oldestVersion,
-                   std::span<const uint8_t> key, InternalVersionT writeVersion,
-                   NodeAllocators *allocators, ConflictSet::Impl *impl) {
+void addPointWrite(Node *&root, std::span<const uint8_t> key,
+                   InternalVersionT writeVersion, NodeAllocators *allocators,
+                   ConflictSet::Impl *impl) {
   auto *n = insert<true>(&root, key, writeVersion, allocators, impl);
   if (!n->entryPresent) {
     auto *p = nextLogical(n);
@@ -2691,24 +2846,23 @@ void addPointWrite(Node *&root, InternalVersionT oldestVersion,
     n->entry.pointVersion = writeVersion;
     setMaxVersion(n, impl, writeVersion);
     n->entry.rangeVersion =
-        p != nullptr ? p->entry.rangeVersion : oldestVersion;
+        p == nullptr ? InternalVersionT::zero
+                     : std::max(p->entry.rangeVersion, InternalVersionT::zero);
   } else {
     assert(writeVersion >= n->entry.pointVersion);
     n->entry.pointVersion = writeVersion;
   }
 }
 
-void addWriteRange(Node *&root, InternalVersionT oldestVersion,
-                   std::span<const uint8_t> begin, std::span<const uint8_t> end,
-                   InternalVersionT writeVersion, NodeAllocators *allocators,
-                   ConflictSet::Impl *impl) {
+void addWriteRange(Node *&root, std::span<const uint8_t> begin,
+                   std::span<const uint8_t> end, InternalVersionT writeVersion,
+                   NodeAllocators *allocators, ConflictSet::Impl *impl) {
 
   int lcp = longestCommonPrefix(begin.data(), end.data(),
                                 std::min(begin.size(), end.size()));
   if (lcp == int(begin.size()) && end.size() == begin.size() + 1 &&
       end.back() == 0) {
-    return addPointWrite(root, oldestVersion, begin, writeVersion, allocators,
-                         impl);
+    return addPointWrite(root, begin, writeVersion, allocators, impl);
   }
   const bool beginIsPrefix = lcp == int(begin.size());
   auto remaining = begin.subspan(0, lcp);
@@ -2756,7 +2910,8 @@ void addWriteRange(Node *&root, InternalVersionT oldestVersion,
   if (insertedBegin) {
     auto *p = nextLogical(beginNode);
     beginNode->entry.rangeVersion =
-        p != nullptr ? p->entry.rangeVersion : oldestVersion;
+        p == nullptr ? InternalVersionT::zero
+                     : std::max(p->entry.rangeVersion, InternalVersionT::zero);
     beginNode->entry.pointVersion = writeVersion;
     assert(maxVersion(beginNode, impl) <= writeVersion);
     setMaxVersion(beginNode, impl, writeVersion);
@@ -2775,7 +2930,8 @@ void addWriteRange(Node *&root, InternalVersionT oldestVersion,
   if (insertedEnd) {
     auto *p = nextLogical(endNode);
     endNode->entry.pointVersion =
-        p != nullptr ? p->entry.rangeVersion : oldestVersion;
+        p == nullptr ? InternalVersionT::zero
+                     : std::max(p->entry.rangeVersion, InternalVersionT::zero);
     auto m = maxVersion(endNode, impl);
     setMaxVersion(endNode, impl,
                   std::max<InternalVersionT>(m, endNode->entry.pointVersion));
@@ -2790,21 +2946,16 @@ void addWriteRange(Node *&root, InternalVersionT oldestVersion,
   }
 
   for (beginNode = nextLogical(beginNode); beginNode != endNode;
-       beginNode = erase(beginNode, allocators, impl, endNode)) {
+       beginNode =
+           erase(beginNode, allocators, impl, /*logical*/ true, endNode)) {
   }
 }
 
-Iterator firstGeq(Node *n, const std::span<const uint8_t> key) {
+Node *firstGeqPhysical(Node *n, const std::span<const uint8_t> key) {
   auto remaining = key;
   for (;;) {
     if (remaining.size() == 0) {
-      if (n->entryPresent) {
-        return {n, 0};
-      }
-      int c = getChildGeq(n, 0);
-      assert(c >= 0);
-      n = getChildExists(n, c);
-      goto downLeftSpine;
+      return n;
     }
 
     auto *child = getChild(n, remaining[0]);
@@ -2812,7 +2963,7 @@ Iterator firstGeq(Node *n, const std::span<const uint8_t> key) {
       int c = getChildGeq(n, remaining[0]);
       if (c >= 0) {
         n = getChildExists(n, c);
-        goto downLeftSpine;
+        return n;
       } else {
         n = nextSibling(n);
         if (n == nullptr) {
@@ -2820,9 +2971,9 @@ Iterator firstGeq(Node *n, const std::span<const uint8_t> key) {
           //  final library, since we can't remove a key without introducing a
           //  key after it, and the only production caller of firstGeq is for
           //  resuming the setOldestVersion scan.
-          return {nullptr, 1}; // GCOVR_EXCL_LINE
+          return nullptr; // GCOVR_EXCL_LINE
         }
-        goto downLeftSpine;
+        return n;
       }
     }
 
@@ -2835,10 +2986,10 @@ Iterator firstGeq(Node *n, const std::span<const uint8_t> key) {
       if (i < commonLen) {
         auto c = n->partialKey()[i] <=> remaining[i];
         if (c > 0) {
-          goto downLeftSpine;
+          return n;
         } else {
           n = nextSibling(n);
-          goto downLeftSpine;
+          return n;
         }
       }
       if (commonLen == n->partialKeyLen) {
@@ -2847,19 +2998,10 @@ Iterator firstGeq(Node *n, const std::span<const uint8_t> key) {
       } else if (n->partialKeyLen > int(remaining.size())) {
         // n is the first physical node greater than remaining, and there's no
         // eq node
-        goto downLeftSpine;
+        return n;
       }
     }
   }
-downLeftSpine:
-  for (;;) {
-    if (n->entryPresent) {
-      return {n, 1};
-    }
-    int c = getChildGeq(n, 0);
-    assert(c >= 0);
-    n = getChildExists(n, c);
-  }
 }
 
 struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
@@ -2869,10 +3011,10 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
       const auto &r = reads[i];
       auto begin = std::span<const uint8_t>(r.begin.p, r.begin.len);
       auto end = std::span<const uint8_t>(r.end.p, r.end.len);
+      assert(oldestVersionFullPrecision >=
+             newestVersionFullPrecision - kNominalVersionWindow);
       result[i] =
-          InternalVersionT(reads[i].readVersion) < oldestVersion ||
-                  reads[i].readVersion < newestVersionFullPrecision - 2e9
-              ? TooOld
+          reads[i].readVersion < oldestVersionFullPrecision ? TooOld
           : (end.size() > 0
                  ? checkRangeRead(root, begin, end,
                                   InternalVersionT(reads[i].readVersion), this)
@@ -2885,50 +3027,51 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
 
   void addWrites(const WriteRange *writes, int count, int64_t writeVersion) {
     assert(writeVersion >= newestVersionFullPrecision);
+
+    if (writeVersion > newestVersionFullPrecision + kNominalVersionWindow) {
+      destroyTree(root);
+      init(writeVersion - kNominalVersionWindow);
+    }
+
     newestVersionFullPrecision = writeVersion;
-#if INTERNAL_VERSION_32_BIT
-    InternalVersionT::zero = oldestVersion;
-#endif
+    setOldestVersion(
+        std::max(oldestVersionFullPrecision,
+                 newestVersionFullPrecision - kNominalVersionWindow));
+    while (oldestExtantVersion <
+           newestVersionFullPrecision - kMaxCorrectVersionWindow) {
+      gcScanStep(1000);
+    }
+
     for (int i = 0; i < count; ++i) {
       const auto &w = writes[i];
       auto begin = std::span<const uint8_t>(w.begin.p, w.begin.len);
       auto end = std::span<const uint8_t>(w.end.p, w.end.len);
       if (w.end.len > 0) {
         keyUpdates += 3;
-        addWriteRange(root, oldestVersion, begin, end,
-                      InternalVersionT(writeVersion), &allocators, this);
+        addWriteRange(root, begin, end, InternalVersionT(writeVersion),
+                      &allocators, this);
       } else {
         keyUpdates += 2;
-        addPointWrite(root, oldestVersion, begin,
-                      InternalVersionT(writeVersion), &allocators, this);
+        addPointWrite(root, begin, InternalVersionT(writeVersion), &allocators,
+                      this);
       }
     }
   }
 
-  void setOldestVersion(int64_t o) {
-    InternalVersionT oldestVersion{o};
-    assert(o >= oldestVersionFullPrecision);
-    this->oldestVersionFullPrecision = o;
-    this->oldestVersion = oldestVersion;
-#if INTERNAL_VERSION_32_BIT
-    InternalVersionT::zero = oldestVersion;
-#endif
-#ifdef NDEBUG
-    // This is here for performance reasons, since we want to amortize the cost
-    // of storing the search path as a string. In tests, we want to exercise the
-    // rest of the code often.
-    if (keyUpdates < 100) {
-      return;
-    }
-#endif
-    Node *n = firstGeq(root, removalKey).n;
+  // Spends up to `fuel` gc'ing, and returns its unused fuel. Reclaims memory
+  // and updates oldestExtantVersion after spending enough fuel.
+  int64_t gcScanStep(int64_t fuel) {
+    Node *n = firstGeqPhysical(root, removalKey);
     // There's no way to erase removalKey without introducing a key after it
     assert(n != nullptr);
     // Don't erase the root
     if (n == root) {
+      rezero(n, oldestVersion);
+      rootMaxVersion = std::max(rootMaxVersion, oldestVersion);
       n = nextPhysical(n);
     }
-    for (; keyUpdates > 0 && n != nullptr; --keyUpdates) {
+    for (; fuel > 0 && n != nullptr; --fuel) {
+      rezero(n, oldestVersion);
       if (n->entryPresent && std::max(n->entry.pointVersion,
                                       n->entry.rangeVersion) <= oldestVersion) {
         // Any transaction n would have prevented from committing is
@@ -2938,7 +3081,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
         // node is greater than the point version of the left node
         assert(n->entry.rangeVersion <= oldestVersion);
         Node *dummy = nullptr;
-        n = erase(n, &allocators, this, dummy);
+        n = erase(n, &allocators, this, /*logical*/ false, dummy);
       } else {
         maybeDecreaseCapacity(n, &allocators, this);
         n = nextPhysical(n);
@@ -2946,18 +3089,53 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
     }
     if (n == nullptr) {
       removalKey = {};
+      oldestExtantVersion = oldestVersionAtGcBegin;
+      oldestVersionAtGcBegin = oldestVersionFullPrecision;
+#if DEBUG_VERBOSE && !defined(NDEBUG)
+      fprintf(stderr,
+              "new oldestExtantVersion: %" PRId64
+              ", new oldestVersionAtGcBegin: %" PRId64 "\n",
+              oldestExtantVersion, oldestVersionAtGcBegin);
+#endif
     } else {
       removalKeyArena = Arena();
       removalKey = getSearchPath(removalKeyArena, n);
     }
+    return fuel;
   }
 
-  explicit Impl(int64_t oldestVersion)
-      : oldestVersion(oldestVersion), oldestVersionFullPrecision(oldestVersion),
-        newestVersionFullPrecision(oldestVersion) {
-#if DEBUG_VERBOSE
-    fprintf(stderr, "radix_tree: create\n");
+  void setOldestVersion(int64_t o) {
+    if (o <= oldestVersionFullPrecision) {
+      return;
+    }
+    InternalVersionT oldestVersion{o};
+    this->oldestVersionFullPrecision = o;
+    this->oldestVersion = oldestVersion;
+    InternalVersionT::zero = oldestVersion;
+#ifdef NDEBUG
+    // This is here for performance reasons, since we want to amortize the cost
+    // of storing the search path as a string. In tests, we want to exercise the
+    // rest of the code often.
+    if (keyUpdates < 100) {
+      return;
+    }
 #endif
+    keyUpdates = gcScanStep(keyUpdates);
+  }
+
+  int64_t getBytes() const { return totalBytes; }
+
+  void init(int64_t oldestVersion) {
+    this->oldestVersion = InternalVersionT(oldestVersion);
+    oldestVersionFullPrecision = oldestExtantVersion = oldestVersionAtGcBegin =
+        newestVersionFullPrecision = oldestVersion;
+
+    allocators.~NodeAllocators();
+    new (&allocators) NodeAllocators();
+
+    removalKeyArena = Arena{};
+    removalKey = {};
+    keyUpdates = 10;
 
     // Insert ""
     root = allocators.node0.allocate(0);
@@ -2972,27 +3150,27 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
     root->entryPresent = true;
     root->entry.pointVersion = this->oldestVersion;
     root->entry.rangeVersion = this->oldestVersion;
+
+    InternalVersionT::zero = this->oldestVersion;
+
+    // Intentionally not resetting totalBytes
   }
-  ~Impl() {
-#if DEBUG_VERBOSE
-    fprintf(stderr, "radix_tree: destroy\n");
-#endif
-    destroyTree(root);
-  }
+
+  explicit Impl(int64_t oldestVersion) { init(oldestVersion); }
+  ~Impl() { destroyTree(root); }
 
   NodeAllocators allocators;
 
   Arena removalKeyArena;
   std::span<const uint8_t> removalKey;
-  int64_t keyUpdates = 10;
+  int64_t keyUpdates;
 
   Node *root;
   InternalVersionT rootMaxVersion;
   InternalVersionT oldestVersion;
-  // TODO this doesn't fully mitigate the 32-bit precision issue, since we still
-  // need to make sure we clean up versions in the tree before they fall out of
-  // the 2e9 window.
   int64_t oldestVersionFullPrecision;
+  int64_t oldestExtantVersion;
+  int64_t oldestVersionAtGcBegin;
   int64_t newestVersionFullPrecision;
   int64_t totalBytes = 0;
 };
@@ -3124,12 +3302,76 @@ void internal_destroy(ConflictSet::Impl *impl) {
   safe_free(impl, sizeof(ConflictSet::Impl));
 }
 
-int64_t internal_getBytes(ConflictSet::Impl *impl) { return impl->totalBytes; }
+int64_t internal_getBytes(ConflictSet::Impl *impl) { return impl->getBytes(); }
 
 // ==================== END IMPLEMENTATION ====================
 
 // GCOVR_EXCL_START
 
+Iterator firstGeqLogical(Node *n, const std::span<const uint8_t> key) {
+  auto remaining = key;
+  for (;;) {
+    if (remaining.size() == 0) {
+      if (n->entryPresent) {
+        return {n, 0};
+      }
+      int c = getChildGeq(n, 0);
+      assert(c >= 0);
+      n = getChildExists(n, c);
+      goto downLeftSpine;
+    }
+
+    auto *child = getChild(n, remaining[0]);
+    if (child == nullptr) {
+      int c = getChildGeq(n, remaining[0]);
+      if (c >= 0) {
+        n = getChildExists(n, c);
+        goto downLeftSpine;
+      } else {
+        n = nextSibling(n);
+        if (n == nullptr) {
+          return {nullptr, 1};
+        }
+        goto downLeftSpine;
+      }
+    }
+
+    n = child;
+    remaining = remaining.subspan(1, remaining.size() - 1);
+
+    if (n->partialKeyLen > 0) {
+      int commonLen = std::min<int>(n->partialKeyLen, remaining.size());
+      int i = longestCommonPrefix(n->partialKey(), remaining.data(), commonLen);
+      if (i < commonLen) {
+        auto c = n->partialKey()[i] <=> remaining[i];
+        if (c > 0) {
+          goto downLeftSpine;
+        } else {
+          n = nextSibling(n);
+          goto downLeftSpine;
+        }
+      }
+      if (commonLen == n->partialKeyLen) {
+        // partial key matches
+        remaining = remaining.subspan(commonLen, remaining.size() - commonLen);
+      } else if (n->partialKeyLen > int(remaining.size())) {
+        // n is the first physical node greater than remaining, and there's no
+        // eq node
+        goto downLeftSpine;
+      }
+    }
+  }
+downLeftSpine:
+  for (;;) {
+    if (n->entryPresent) {
+      return {n, 1};
+    }
+    int c = getChildGeq(n, 0);
+    assert(c >= 0);
+    n = getChildExists(n, c);
+  }
+}
+
 void ConflictSet::check(const ReadRange *reads, Result *results,
                         int count) const {
   internal_check(impl, reads, results, count);
@@ -3326,14 +3568,60 @@ void checkParentPointers(Node *node, bool &success) {
 }
 
 Iterator firstGeq(Node *n, std::string_view key) {
-  return firstGeq(
+  return firstGeqLogical(
       n, std::span<const uint8_t>((const uint8_t *)key.data(), key.size()));
 }
 
+void checkVersionsGeqOldestExtant(Node *n,
+                                  InternalVersionT oldestExtantVersion) {
+  if (n->entryPresent) {
+    assert(n->entry.pointVersion >= oldestExtantVersion);
+    assert(n->entry.rangeVersion >= oldestExtantVersion);
+  }
+  switch (n->getType()) {
+  case Type_Node0: {
+  } break;
+  case Type_Node3: {
+    auto *self = static_cast<Node3 *>(n);
+    for (int i = 0; i < 3; ++i) {
+      assert(self->childMaxVersion[i] >= oldestExtantVersion);
+    }
+  } break;
+  case Type_Node16: {
+    auto *self = static_cast<Node16 *>(n);
+    for (int i = 0; i < 16; ++i) {
+      assert(self->childMaxVersion[i] >= oldestExtantVersion);
+    }
+  } break;
+  case Type_Node48: {
+    auto *self = static_cast<Node48 *>(n);
+    for (int i = 0; i < 48; ++i) {
+      assert(self->childMaxVersion[i] >= oldestExtantVersion);
+    }
+    for (auto m : self->maxOfMax) {
+      assert(m >= oldestExtantVersion);
+    }
+  } break;
+  case Type_Node256: {
+    auto *self = static_cast<Node256 *>(n);
+    for (int i = 0; i < 256; ++i) {
+      assert(self->childMaxVersion[i] >= oldestExtantVersion);
+    }
+    for (auto m : self->maxOfMax) {
+      assert(m >= oldestExtantVersion);
+    }
+  } break;
+  default:
+    abort();
+  }
+}
+
 [[maybe_unused]] InternalVersionT
 checkMaxVersion(Node *root, Node *node, InternalVersionT oldestVersion,
                 bool &success, ConflictSet::Impl *impl) {
-  InternalVersionT expected{0};
+  checkVersionsGeqOldestExtant(node,
+                               InternalVersionT(impl->oldestExtantVersion));
+  auto expected = InternalVersionT::zero;
   if (node->entryPresent) {
     expected = std::max(expected, node->entry.pointVersion);
   }

Internal.h

@@ -467,13 +467,15 @@ inline uint32_t Arbitrary::bounded(uint32_t s) {
 // ==================== END ARBITRARY IMPL ====================
 
 struct ReferenceImpl {
-  explicit ReferenceImpl(int64_t oldestVersion) : oldestVersion(oldestVersion) {
+  explicit ReferenceImpl(int64_t oldestVersion)
+      : oldestVersion(oldestVersion), newestVersion(oldestVersion) {
     writeVersionMap[""] = oldestVersion;
   }
   void check(const ConflictSet::ReadRange *reads, ConflictSet::Result *results,
              int count) const {
     for (int i = 0; i < count; ++i) {
-      if (reads[i].readVersion < oldestVersion) {
+      if (reads[i].readVersion < oldestVersion ||
+          reads[i].readVersion < newestVersion - 2e9) {
         results[i] = ConflictSet::TooOld;
         continue;
       }
@@ -495,6 +497,8 @@ struct ReferenceImpl {
   }
   void addWrites(const ConflictSet::WriteRange *writes, int count,
                  int64_t writeVersion) {
+    assert(writeVersion >= newestVersion);
+    newestVersion = writeVersion;
     for (int i = 0; i < count; ++i) {
       auto begin =
           std::string((const char *)writes[i].begin.p, writes[i].begin.len);
@@ -514,11 +518,12 @@ struct ReferenceImpl {
   }
 
   void setOldestVersion(int64_t oldestVersion) {
-    assert(oldestVersion >= oldestVersion);
+    assert(oldestVersion >= this->oldestVersion);
     this->oldestVersion = oldestVersion;
   }
 
   int64_t oldestVersion;
+  int64_t newestVersion;
   std::map<std::string, int64_t> writeVersionMap;
 };
 
@@ -578,8 +583,8 @@ template <class ConflictSetImpl> struct TestDriver {
   explicit TestDriver(const uint8_t *data, size_t size)
       : arbitrary({data, size}) {}
 
-  int64_t oldestVersion = arbitrary.bounded(2) ? 0 : 0xfffffff0;
-  int64_t writeVersion = oldestVersion + 100;
+  int64_t oldestVersion = arbitrary.next();
+  int64_t writeVersion = oldestVersion;
   ConflictSetImpl cs{oldestVersion};
   ReferenceImpl refImpl{oldestVersion};
 
@@ -593,6 +598,7 @@ template <class ConflictSetImpl> struct TestDriver {
   // Call until it returns true, for "done". Check internal invariants etc
   // between calls to next.
   bool next() {
+    assert(cs.getBytes() >= 0);
     if (!arbitrary.hasEntropy()) {
       return true;
     }
@@ -600,7 +606,8 @@ template <class ConflictSetImpl> struct TestDriver {
     {
       int numPointWrites = arbitrary.bounded(100);
       int numRangeWrites = arbitrary.bounded(100);
-      int64_t v = (writeVersion += arbitrary.bounded(10));
+      int64_t v = (writeVersion += arbitrary.bounded(10) ? arbitrary.bounded(10)
+                                                         : arbitrary.next());
       auto *writes =
           new (arena) ConflictSet::WriteRange[numPointWrites + numRangeWrites];
       auto keys = set<std::string_view>(arena);
@@ -642,33 +649,79 @@ template <class ConflictSetImpl> struct TestDriver {
         }
 #if DEBUG_VERBOSE && !defined(NDEBUG)
         if (writes[i].end.len == 0) {
-          fprintf(stderr, "Write: {%s} -> %" PRId64 "\n",
-                  printable(writes[i].begin).c_str(), writeVersion);
+          fprintf(stderr, "Write: {%s}\n", printable(writes[i].begin).c_str());
         } else {
-          fprintf(stderr, "Write: [%s, %s) -> %" PRId64 "\n",
+          fprintf(stderr, "Write: [%s, %s)\n",
                   printable(writes[i].begin).c_str(),
-                  printable(writes[i].end).c_str(), writeVersion);
+                  printable(writes[i].end).c_str());
         }
 #endif
       }
       assert(iter == keys.end());
       assert(i == numPointWrites + numRangeWrites);
 
+#if DEBUG_VERBOSE && !defined(NDEBUG)
+      fprintf(stderr, "Write @ %" PRId64 "\n", v);
+#endif
+
+      // Test non-canonical writes
+      if (numPointWrites > 0) {
+        int overlaps = arbitrary.bounded(numPointWrites);
+        for (int i = 0; i < numPointWrites + numRangeWrites && overlaps > 0;
+             ++i) {
+          if (writes[i].end.len == 0) {
+            int keyLen = prefixLen + arbitrary.bounded(kMaxKeySuffixLen);
+            auto *begin = new (arena) uint8_t[keyLen];
+            memset(begin, prefixByte, prefixLen);
+            arbitrary.randomBytes(begin + prefixLen, keyLen - prefixLen);
+            writes[i].end.len = keyLen;
+            writes[i].end.p = begin;
+            auto c =
+                std::span<const uint8_t>(writes[i].begin.p,
+                                         writes[i].begin.len) <=>
+                std::span<const uint8_t>(writes[i].end.p, writes[i].end.len);
+            if (c > 0) {
+              using std::swap;
+              swap(writes[i].begin, writes[i].end);
+            } else if (c == 0) {
+              // It's a point write after all, I guess
+              writes[i].end.len = 0;
+            }
+            --overlaps;
+          }
+        }
+      }
+      if (arbitrary.bounded(2)) {
+        // Shuffle writes
+        for (int i = numPointWrites + numRangeWrites - 1; i > 0; --i) {
+          int j = arbitrary.bounded(i + 1);
+          if (i != j) {
+            using std::swap;
+            swap(writes[i], writes[j]);
+          }
+        }
+      }
+
       CALLGRIND_START_INSTRUMENTATION;
       cs.addWrites(writes, numPointWrites + numRangeWrites, v);
       CALLGRIND_STOP_INSTRUMENTATION;
 
       refImpl.addWrites(writes, numPointWrites + numRangeWrites, v);
 
-      oldestVersion =
-          std::min(writeVersion - 10, oldestVersion + arbitrary.bounded(10));
+      oldestVersion +=
+          arbitrary.bounded(10) ? arbitrary.bounded(10) : arbitrary.next();
+      oldestVersion = std::min(oldestVersion, writeVersion);
       cs.setOldestVersion(oldestVersion);
       refImpl.setOldestVersion(oldestVersion);
     }
     {
       int numPointReads = arbitrary.bounded(100);
       int numRangeReads = arbitrary.bounded(100);
-      int64_t v = std::max<int64_t>(writeVersion - arbitrary.bounded(10), 0);
+
+      int64_t v = std::max<int64_t>(writeVersion - (arbitrary.bounded(10)
+                                                        ? arbitrary.bounded(10)
+                                                        : arbitrary.next()),
+                                    0);
       auto *reads =
           new (arena) ConflictSet::ReadRange[numPointReads + numRangeReads];
       auto keys = set<std::string_view>(arena);
@@ -710,12 +763,12 @@ template <class ConflictSetImpl> struct TestDriver {
         reads[i].readVersion = v;
 #if DEBUG_VERBOSE && !defined(NDEBUG)
         if (reads[i].end.len == 0) {
-          fprintf(stderr, "Read: {%s} @ %d\n",
-                  printable(reads[i].begin).c_str(), int(reads[i].readVersion));
+          fprintf(stderr, "Read: {%s} @ %" PRId64 "\n",
+                  printable(reads[i].begin).c_str(), reads[i].readVersion);
         } else {
-          fprintf(stderr, "Read: [%s, %s) @ %d\n",
+          fprintf(stderr, "Read: [%s, %s) @ %" PRId64 "\n",
                   printable(reads[i].begin).c_str(),
-                  printable(reads[i].end).c_str(), int(reads[i].readVersion));
+                  printable(reads[i].end).c_str(), reads[i].readVersion);
         }
 #endif
       }

Jenkinsfile

@@ -59,17 +59,6 @@ pipeline {
                 CleanBuildAndTest("-DUSE_SIMD_FALLBACK=ON")
             }
         }
-        stage('32-bit versions') {
-            agent {
-                dockerfile {
-                    args '-v /home/jenkins/ccache:/ccache'
-                    reuseNode true
-                }
-            }
-            steps {
-                CleanBuildAndTest("-DUSE_32_BIT_VERSIONS=ON")
-            }
-        }
         stage('Release [gcc]') {
             agent {
                 dockerfile {

README.md

@@ -60,27 +60,27 @@ Performance counters:
 
 |               ns/op |                op/s |    err% |     total | benchmark
 |--------------------:|--------------------:|--------:|----------:|:----------
-|              256.89 |        3,892,784.92 |    0.3% |      0.01 | `point reads`
-|              272.90 |        3,664,395.04 |    0.2% |      0.01 | `prefix reads`
-|              507.22 |        1,971,549.50 |    0.7% |      0.01 | `range reads`
-|              452.66 |        2,209,181.91 |    0.5% |      0.01 | `point writes`
-|              438.09 |        2,282,619.96 |    0.4% |      0.01 | `prefix writes`
-|              253.33 |        3,947,420.36 |    2.5% |      0.02 | `range writes`
-|              574.07 |        1,741,936.71 |    0.3% |      0.01 | `monotonic increasing point writes`
-|          151,562.50 |            6,597.94 |    1.5% |      0.01 | `worst case for radix tree`
+|              245.99 |        4,065,232.81 |    0.3% |      0.01 | `point reads`
+|              265.93 |        3,760,430.49 |    0.2% |      0.01 | `prefix reads`
+|              485.30 |        2,060,569.50 |    0.2% |      0.01 | `range reads`
+|              449.60 |        2,224,195.17 |    0.4% |      0.01 | `point writes`
+|              441.76 |        2,263,688.18 |    1.1% |      0.01 | `prefix writes`
+|              245.42 |        4,074,647.54 |    2.4% |      0.02 | `range writes`
+|              572.80 |        1,745,810.06 |    1.3% |      0.01 | `monotonic increasing point writes`
+|          154,819.33 |            6,459.14 |    0.9% |      0.01 | `worst case for radix tree`
 
 ## Radix tree (this implementation)
 
 |               ns/op |                op/s |    err% |     total | benchmark
 |--------------------:|--------------------:|--------:|----------:|:----------
-|               19.83 |       50,420,955.28 |    0.1% |      0.01 | `point reads`
-|               55.95 |       17,872,542.40 |    0.5% |      0.01 | `prefix reads`
-|               88.28 |       11,327,709.50 |    0.4% |      0.01 | `range reads`
-|               29.15 |       34,309,531.64 |    0.5% |      0.01 | `point writes`
-|               42.36 |       23,607,424.27 |    1.1% |      0.01 | `prefix writes`
-|               50.00 |       20,000,000.00 |    0.0% |      0.01 | `range writes`
-|               93.52 |       10,692,413.79 |    3.3% |      0.01 | `monotonic increasing point writes`
-|        2,388,417.00 |              418.69 |    0.4% |      0.03 | `worst case for radix tree`
+|               19.17 |       52,163,930.66 |    0.1% |      0.01 | `point reads`
+|               23.68 |       42,224,388.21 |    0.7% |      0.01 | `prefix reads`
+|               63.30 |       15,797,506.06 |    0.9% |      0.01 | `range reads`
+|               29.66 |       33,720,994.74 |    0.3% |      0.01 | `point writes`
+|               43.50 |       22,987,781.25 |    1.0% |      0.01 | `prefix writes`
+|               50.00 |       20,000,000.00 |    0.8% |      0.01 | `range writes`
+|              103.25 |        9,684,786.47 |    2.9% |      0.01 | `monotonic increasing point writes`
+|        1,181,500.00 |              846.38 |    2.3% |      0.01 | `worst case for radix tree`
 
 # "Real data" test
 

SkipList.cpp

@@ -22,6 +22,8 @@
 
 #include "ConflictSet.h"
 #include "Internal.h"
+
+#include <algorithm>
 #include <span>
 
 std::span<const uint8_t> keyAfter(Arena &arena, std::span<const uint8_t> key) {
@@ -52,6 +54,135 @@ struct KeyRangeRef {
       : begin(begin), end(keyAfter(arena, begin)) {}
 };
 
+struct KeyInfo {
+  StringRef key;
+  bool begin;
+  bool write;
+
+  KeyInfo() = default;
+  KeyInfo(StringRef key, bool begin, bool write)
+      : key(key), begin(begin), write(write) {}
+};
+
+force_inline int extra_ordering(const KeyInfo &ki) {
+  return ki.begin * 2 + (ki.write ^ ki.begin);
+}
+
+// returns true if done with string
+force_inline bool getCharacter(const KeyInfo &ki, int character,
+                               int &outputCharacter) {
+  // normal case
+  if (character < ki.key.size()) {
+    outputCharacter = 5 + ki.key.begin()[character];
+    return false;
+  }
+
+  // termination
+  if (character == ki.key.size()) {
+    outputCharacter = 0;
+    return false;
+  }
+
+  if (character == ki.key.size() + 1) {
+    // end/begin+read/write relative sorting
+    outputCharacter = extra_ordering(ki);
+    return false;
+  }
+
+  outputCharacter = 0;
+  return true;
+}
+
+bool operator<(const KeyInfo &lhs, const KeyInfo &rhs) {
+  int i = std::min(lhs.key.size(), rhs.key.size());
+  int c = memcmp(lhs.key.data(), rhs.key.data(), i);
+  if (c != 0)
+    return c < 0;
+
+  // Always sort shorter keys before longer keys.
+  if (lhs.key.size() < rhs.key.size()) {
+    return true;
+  }
+  if (lhs.key.size() > rhs.key.size()) {
+    return false;
+  }
+
+  // When the keys are the same length, use the extra ordering constraint.
+  return extra_ordering(lhs) < extra_ordering(rhs);
+}
+
+bool operator==(const KeyInfo &lhs, const KeyInfo &rhs) {
+  return !(lhs < rhs || rhs < lhs);
+}
+
+void swapSort(std::vector<KeyInfo> &points, int a, int b) {
+  if (points[b] < points[a]) {
+    KeyInfo temp;
+    temp = points[a];
+    points[a] = points[b];
+    points[b] = temp;
+  }
+}
+
+struct SortTask {
+  int begin;
+  int size;
+  int character;
+  SortTask(int begin, int size, int character)
+      : begin(begin), size(size), character(character) {}
+};
+
+void sortPoints(std::vector<KeyInfo> &points) {
+  std::vector<SortTask> tasks;
+  std::vector<KeyInfo> newPoints;
+  std::vector<int> counts;
+
+  tasks.emplace_back(0, points.size(), 0);
+
+  while (tasks.size()) {
+    SortTask st = tasks.back();
+    tasks.pop_back();
+
+    if (st.size < 10) {
+      std::sort(points.begin() + st.begin, points.begin() + st.begin + st.size);
+      continue;
+    }
+
+    newPoints.resize(st.size);
+    counts.assign(256 + 5, 0);
+
+    // get counts
+    int c;
+    bool allDone = true;
+    for (int i = st.begin; i < st.begin + st.size; i++) {
+      allDone &= getCharacter(points[i], st.character, c);
+      counts[c]++;
+    }
+    if (allDone)
+      continue;
+
+    // calculate offsets from counts and build next level of tasks
+    int total = 0;
+    for (int i = 0; i < counts.size(); i++) {
+      int temp = counts[i];
+      if (temp > 1)
+        tasks.emplace_back(st.begin + total, temp, st.character + 1);
+      counts[i] = total;
+      total += temp;
+    }
+
+    // put in their places
+    for (int i = st.begin; i < st.begin + st.size; i++) {
+      getCharacter(points[i], st.character, c);
+      newPoints[counts[c]++] = points[i];
+    }
+
+    // copy back into original points array
+    for (int i = 0; i < st.size; i++)
+      points[st.begin + i] = newPoints[i];
+  }
+}
+
 static thread_local uint32_t g_seed = 0;
 
 static inline int skfastrand() {
@@ -602,10 +733,40 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
 
   void addWrites(const ConflictSet::WriteRange *writes, int count,
                  int64_t writeVersion) {
+    auto points = std::vector<KeyInfo>(count * 2);
+    Arena arena;
+
+    for (int r = 0; r < count; r++) {
+      points.emplace_back(StringRef(writes[r].begin.p, writes[r].begin.len),
+                          true, true);
+      points.emplace_back(
+          writes[r].end.len > 0
+              ? StringRef{writes[r].end.p, size_t(writes[r].end.len)}
+              : keyAfter(arena, points.back().key),
+          false, true);
+    }
+
+    sortPoints(points);
+
+    int activeWriteCount = 0;
+    std::vector<std::pair<StringRef, StringRef>> combinedWriteConflictRanges;
+    for (const KeyInfo &point : points) {
+      if (point.write) {
+        if (point.begin) {
+          activeWriteCount++;
+          if (activeWriteCount == 1)
+            combinedWriteConflictRanges.emplace_back(point.key, StringRef());
+        } else /*if (point.end)*/ {
+          activeWriteCount--;
+          if (activeWriteCount == 0)
+            combinedWriteConflictRanges.back().second = point.key;
+        }
+      }
+    }
+
     assert(writeVersion >= newestVersion);
     newestVersion = writeVersion;
-    Arena arena;
-    const int stringCount = count * 2;
+    const int stringCount = combinedWriteConflictRanges.size() * 2;
 
     const int stripeSize = 16;
     SkipList::Finger fingers[stripeSize];
@@ -616,15 +777,13 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
     int ss = stringCount - (stripes - 1) * stripeSize;
     for (int s = stripes - 1; s >= 0; s--) {
       for (int i = 0; i * 2 < ss; ++i) {
-        const auto &w = writes[s * stripeSize / 2 + i];
+        const auto &w = combinedWriteConflictRanges[s * stripeSize / 2 + i];
 #if DEBUG_VERBOSE
         printf("Write begin: %s\n", printable(w.begin).c_str());
         fflush(stdout);
 #endif
-        values[i * 2] = {w.begin.p, size_t(w.begin.len)};
-        values[i * 2 + 1] = w.end.len > 0
-                                ? StringRef{w.end.p, size_t(w.end.len)}
-                                : keyAfter(arena, values[i * 2]);
+        values[i * 2] = w.first;
+        values[i * 2 + 1] = w.second;
         keyUpdates += 3;
       }
       skipList.find(values, fingers, temp, ss);