Declare dependency on version.txt in Makefile

It's bogus since we mess with oldestVersionFullPrecision in addWrites

ConflictSet.cpp

@@ -606,6 +606,14 @@ template <class T> struct BoundedFreeListAllocator {
     VALGRIND_MAKE_MEM_NOACCESS(freeList, sizeof(T) + p->partialKeyCapacity);
   }
 
+  BoundedFreeListAllocator() = default;
+
+  BoundedFreeListAllocator(const BoundedFreeListAllocator &) = delete;
+  BoundedFreeListAllocator &
+  operator=(const BoundedFreeListAllocator &) = delete;
+  BoundedFreeListAllocator(BoundedFreeListAllocator &&) = delete;
+  BoundedFreeListAllocator &operator=(BoundedFreeListAllocator &&) = delete;
+
   ~BoundedFreeListAllocator() {
     for (void *iter = freeList; iter != nullptr;) {
       VALGRIND_MAKE_MEM_DEFINED(iter, sizeof(Node));
@@ -1875,7 +1883,8 @@ bool scan16(const InternalVersionT *vs, const uint8_t *is, int begin, int end,
 
   uint16x4_t conflicting[4];
   for (int i = 0; i < 4; ++i) {
-    conflicting[i] = vmovn_u32(vcgtq_s32(vsubq_u32(w4[i], rvVec), z));
+    conflicting[i] =
+        vmovn_u32(vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
   }
   auto combined =
       vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
@@ -1944,7 +1953,8 @@ scan16(const InternalVersionT *vs, int begin, int end,
 
   uint16x4_t conflicting[4];
   for (int i = 0; i < 4; ++i) {
-    conflicting[i] = vmovn_u32(vcgtq_s32(vsubq_u32(w4[i], rvVec), z));
+    conflicting[i] =
+        vmovn_u32(vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
   }
   auto combined =
       vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
@@ -1991,55 +2001,163 @@ bool checkMaxBetweenExclusive(Node *n, int begin, int end,
 
   assert(!(begin == -1 && end == 256));
 
-  {
-    int c = getChildGeq(n, begin + 1);
-    if (c >= 0 && c < end) {
-      auto *child = getChildExists(n, c);
-      if (child->entryPresent) {
-        if (!(child->entry.rangeVersion <= readVersion)) {
-          return false;
-        };
-      }
-      begin = c;
-    } else {
-      return true;
-    }
-  }
-
-  // [begin, end) is now the half-open interval of children we're interested in.
-  assert(begin < end);
-
   switch (n->getType()) {
-  case Type_Node0: // GCOVR_EXCL_LINE
-    // We would have returned above, after not finding a child
-    __builtin_unreachable(); // GCOVR_EXCL_LINE
+  case Type_Node0:
+    return true;
   case Type_Node3: {
     auto *self = static_cast<Node3 *>(n);
 
+    ++begin;
+
     const unsigned shiftUpperBound = end - begin;
     const unsigned shiftAmount = begin;
     auto inBounds = [&](unsigned c) {
       return c - shiftAmount < shiftUpperBound;
     };
 
-    uint32_t compared = 0;
-    for (int i = 0; i < Node3::kMaxNodes; ++i) {
-      compared |= (self->childMaxVersion[i] > readVersion) << i;
-    }
     uint32_t mask = 0;
     for (int i = 0; i < Node3::kMaxNodes; ++i) {
       mask |= inBounds(self->index[i]) << i;
     }
-    return !(compared & mask);
+    mask &= (1 << self->numChildren) - 1;
+    if (!mask) {
+      return true;
+    }
+    auto *child = self->children[__builtin_ctz(mask)];
+    const bool firstRangeOk =
+        !child->entryPresent || child->entry.rangeVersion <= readVersion;
+    uint32_t compared = 0;
+    for (int i = 0; i < Node3::kMaxNodes; ++i) {
+      compared |= (self->childMaxVersion[i] > readVersion) << i;
+    }
+
+    return !(compared & mask) && firstRangeOk;
   }
   case Type_Node16: {
     auto *self = static_cast<Node16 *>(n);
 
-    return scan16<kAVX512>(self->childMaxVersion, self->index, begin, end,
-                           readVersion);
+    ++begin;
+
+    assert(begin <= end);
+    assert(end - begin < 256);
+
+#ifdef HAS_ARM_NEON
+
+    uint8x16_t indices;
+    memcpy(&indices, self->index, 16);
+    // 0xff for each in bounds
+    auto results =
+        vcltq_u8(vsubq_u8(indices, vdupq_n_u8(begin)), vdupq_n_u8(end - begin));
+    // 0xf for each 0xff
+    uint64_t mask = vget_lane_u64(
+        vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(results), 4)), 0);
+
+    mask &= self->numChildren == 16
+                ? uint64_t(-1)
+                : (uint64_t(1) << (self->numChildren << 2)) - 1;
+    if (!mask) {
+      return true;
+    }
+    auto *child = self->children[__builtin_ctzll(mask) >> 2];
+    const bool firstRangeOk =
+        !child->entryPresent || child->entry.rangeVersion <= readVersion;
+
+    uint32x4_t w4[4];
+    memcpy(w4, self->childMaxVersion, sizeof(w4));
+    uint32_t rv;
+    memcpy(&rv, &readVersion, sizeof(rv));
+    const auto rvVec = vdupq_n_u32(rv);
+
+    int32x4_t z;
+    memset(&z, 0, sizeof(z));
+
+    uint16x4_t conflicting[4];
+    for (int i = 0; i < 4; ++i) {
+      conflicting[i] = vmovn_u32(
+          vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
+    }
+    auto combined =
+        vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
+                    vmovn_u16(vcombine_u16(conflicting[2], conflicting[3])));
+
+    uint64_t compared = vget_lane_u64(
+        vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(combined), 4)), 0);
+
+    return !(compared & mask) && firstRangeOk;
+
+#elif defined(HAS_AVX)
+
+    __m128i indices;
+    memcpy(&indices, self->index, 16);
+    indices = _mm_sub_epi8(indices, _mm_set1_epi8(begin));
+    uint32_t mask =
+        0xffff &
+        ~_mm_movemask_epi8(_mm_cmpeq_epi8(
+            indices, _mm_max_epu8(indices, _mm_set1_epi8(end - begin))));
+    mask &= (1 << self->numChildren) - 1;
+    if (!mask) {
+      return true;
+    }
+    auto *child = self->children[__builtin_ctz(mask)];
+    const bool firstRangeOk =
+        !child->entryPresent || child->entry.rangeVersion <= readVersion;
+
+    uint32_t compared = 0;
+    if constexpr (kAVX512) {
+      compared = compare16_32bit_avx512(self->childMaxVersion, readVersion);
+    } else {
+      compared = compare16_32bit(self->childMaxVersion, readVersion);
+    }
+    return !(compared & mask) && firstRangeOk;
+
+#else
+
+    const unsigned shiftUpperBound = end - begin;
+    const unsigned shiftAmount = begin;
+    auto inBounds = [&](unsigned c) {
+      return c - shiftAmount < shiftUpperBound;
+    };
+
+    uint32_t mask = 0;
+    for (int i = 0; i < 16; ++i) {
+      mask |= inBounds(self->index[i]) << i;
+    }
+    mask &= (1 << self->numChildren) - 1;
+    if (!mask) {
+      return true;
+    }
+    auto *child = self->children[__builtin_ctz(mask)];
+    const bool firstRangeOk =
+        !child->entryPresent || child->entry.rangeVersion <= readVersion;
+    uint32_t compared = 0;
+    for (int i = 0; i < 16; ++i) {
+      compared |= (self->childMaxVersion[i] > readVersion) << i;
+    }
+    return !(compared & mask) && firstRangeOk;
+
+#endif
   }
   case Type_Node48: {
     auto *self = static_cast<Node48 *>(n);
+
+    {
+      int c = self->bitSet.firstSetGeq(begin + 1);
+      if (c >= 0 && c < end) {
+        auto *child = self->children[self->index[c]];
+        if (child->entryPresent) {
+          if (!(child->entry.rangeVersion <= readVersion)) {
+            return false;
+          };
+        }
+        begin = c;
+      } else {
+        return true;
+      }
+      // [begin, end) is now the half-open interval of children we're interested
+      // in.
+      assert(begin < end);
+    }
+
     // Check all pages
     static_assert(Node48::kMaxOfMaxPageSize == 16);
     for (int i = 0; i < Node48::kMaxOfMaxTotalPages; ++i) {
@@ -2057,6 +2175,25 @@ bool checkMaxBetweenExclusive(Node *n, int begin, int end,
   case Type_Node256: {
     static_assert(Node256::kMaxOfMaxTotalPages == 16);
     auto *self = static_cast<Node256 *>(n);
+
+    {
+      int c = self->bitSet.firstSetGeq(begin + 1);
+      if (c >= 0 && c < end) {
+        auto *child = self->children[c];
+        if (child->entryPresent) {
+          if (!(child->entry.rangeVersion <= readVersion)) {
+            return false;
+          };
+        }
+        begin = c;
+      } else {
+        return true;
+      }
+      // [begin, end) is now the half-open interval of children we're interested
+      // in.
+      assert(begin < end);
+    }
+
     const int firstPage = begin >> Node256::kMaxOfMaxShift;
     const int lastPage = (end - 1) >> Node256::kMaxOfMaxShift;
     // Check the only page if there's only one
@@ -2874,11 +3011,10 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
       const auto &r = reads[i];
       auto begin = std::span<const uint8_t>(r.begin.p, r.begin.len);
       auto end = std::span<const uint8_t>(r.end.p, r.end.len);
+      assert(oldestVersionFullPrecision >=
+             newestVersionFullPrecision - kNominalVersionWindow);
       result[i] =
-          reads[i].readVersion < oldestVersionFullPrecision ||
-                  reads[i].readVersion <
-                      newestVersionFullPrecision - kNominalVersionWindow
-              ? TooOld
+          reads[i].readVersion < oldestVersionFullPrecision ? TooOld
           : (end.size() > 0
                  ? checkRangeRead(root, begin, end,
                                   InternalVersionT(reads[i].readVersion), this)
@@ -2892,8 +3028,10 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
   void addWrites(const WriteRange *writes, int count, int64_t writeVersion) {
     assert(writeVersion >= newestVersionFullPrecision);
 
-    // TODO allow this condition
-    assert(writeVersion < newestVersionFullPrecision + kNominalVersionWindow);
+    if (writeVersion > newestVersionFullPrecision + kNominalVersionWindow) {
+      destroyTree(root);
+      init(writeVersion - kNominalVersionWindow);
+    }
 
     newestVersionFullPrecision = writeVersion;
     setOldestVersion(
@@ -2985,14 +3123,19 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
     keyUpdates = gcScanStep(keyUpdates);
   }
 
-  explicit Impl(int64_t oldestVersion)
-      : oldestVersion(oldestVersion), oldestVersionFullPrecision(oldestVersion),
-        oldestExtantVersion(oldestVersion),
-        oldestVersionAtGcBegin(oldestVersion),
-        newestVersionFullPrecision(oldestVersion) {
-#if DEBUG_VERBOSE
-    fprintf(stderr, "radix_tree: create\n");
-#endif
+  int64_t getBytes() const { return totalBytes; }
+
+  void init(int64_t oldestVersion) {
+    this->oldestVersion = InternalVersionT(oldestVersion);
+    oldestVersionFullPrecision = oldestExtantVersion = oldestVersionAtGcBegin =
+        newestVersionFullPrecision = oldestVersion;
+
+    allocators.~NodeAllocators();
+    new (&allocators) NodeAllocators();
+
+    removalKeyArena = Arena{};
+    removalKey = {};
+    keyUpdates = 10;
 
     // Insert ""
     root = allocators.node0.allocate(0);
@@ -3009,26 +3152,22 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
     root->entry.rangeVersion = this->oldestVersion;
 
     InternalVersionT::zero = this->oldestVersion;
+
+    // Intentionally not resetting totalBytes
   }
-  ~Impl() {
-#if DEBUG_VERBOSE
-    fprintf(stderr, "radix_tree: destroy\n");
-#endif
-    destroyTree(root);
-  }
+
+  explicit Impl(int64_t oldestVersion) { init(oldestVersion); }
+  ~Impl() { destroyTree(root); }
 
   NodeAllocators allocators;
 
   Arena removalKeyArena;
   std::span<const uint8_t> removalKey;
-  int64_t keyUpdates = 10;
+  int64_t keyUpdates;
 
   Node *root;
   InternalVersionT rootMaxVersion;
   InternalVersionT oldestVersion;
-  // TODO this doesn't fully mitigate the 32-bit precision issue, since we still
-  // need to make sure we clean up versions in the tree before they fall out of
-  // the `kMaxCorrectVersionWindow` window.
   int64_t oldestVersionFullPrecision;
   int64_t oldestExtantVersion;
   int64_t oldestVersionAtGcBegin;
@@ -3163,7 +3302,7 @@ void internal_destroy(ConflictSet::Impl *impl) {
   safe_free(impl, sizeof(ConflictSet::Impl));
 }
 
-int64_t internal_getBytes(ConflictSet::Impl *impl) { return impl->totalBytes; }
+int64_t internal_getBytes(ConflictSet::Impl *impl) { return impl->getBytes(); }
 
 // ==================== END IMPLEMENTATION ====================
 
@@ -3191,11 +3330,7 @@ Iterator firstGeqLogical(Node *n, const std::span<const uint8_t> key) {
       } else {
         n = nextSibling(n);
         if (n == nullptr) {
-          //  This line is genuinely unreachable from any entry point of the
-          //  final library, since we can't remove a key without introducing a
-          //  key after it, and the only production caller of firstGeq is for
-          //  resuming the setOldestVersion scan.
-          return {nullptr, 1}; // GCOVR_EXCL_LINE
+          return {nullptr, 1};
         }
         goto downLeftSpine;
       }
@@ -3476,8 +3611,8 @@ void checkVersionsGeqOldestExtant(Node *n,
       assert(m >= oldestExtantVersion);
     }
   } break;
-  default:                   // GCOVR_EXCL_LINE
-    __builtin_unreachable(); // GCOVR_EXCL_LINE
+  default:
+    abort();
   }
 }
 

Internal.h

@@ -518,7 +518,7 @@ struct ReferenceImpl {
   }
 
   void setOldestVersion(int64_t oldestVersion) {
-    assert(oldestVersion >= oldestVersion);
+    assert(oldestVersion >= this->oldestVersion);
     this->oldestVersion = oldestVersion;
   }
 
@@ -583,8 +583,8 @@ template <class ConflictSetImpl> struct TestDriver {
   explicit TestDriver(const uint8_t *data, size_t size)
       : arbitrary({data, size}) {}
 
-  int64_t oldestVersion = 0;
-  int64_t writeVersion = 0;
+  int64_t oldestVersion = arbitrary.next();
+  int64_t writeVersion = oldestVersion;
   ConflictSetImpl cs{oldestVersion};
   ReferenceImpl refImpl{oldestVersion};
 
@@ -598,6 +598,7 @@ template <class ConflictSetImpl> struct TestDriver {
   // Call until it returns true, for "done". Check internal invariants etc
   // between calls to next.
   bool next() {
+    assert(cs.getBytes() >= 0);
     if (!arbitrary.hasEntropy()) {
       return true;
     }
@@ -605,7 +606,8 @@ template <class ConflictSetImpl> struct TestDriver {
     {
       int numPointWrites = arbitrary.bounded(100);
       int numRangeWrites = arbitrary.bounded(100);
-      int64_t v = (writeVersion += arbitrary.bounded(2e9));
+      int64_t v = (writeVersion += arbitrary.bounded(10) ? arbitrary.bounded(10)
+                                                         : arbitrary.next());
       auto *writes =
           new (arena) ConflictSet::WriteRange[numPointWrites + numRangeWrites];
       auto keys = set<std::string_view>(arena);
@@ -668,15 +670,20 @@ template <class ConflictSetImpl> struct TestDriver {
 
       refImpl.addWrites(writes, numPointWrites + numRangeWrites, v);
 
-      oldestVersion =
-          std::min(writeVersion - 10, oldestVersion + arbitrary.bounded(10));
+      oldestVersion +=
+          arbitrary.bounded(10) ? arbitrary.bounded(10) : arbitrary.next();
+      oldestVersion = std::min(oldestVersion, writeVersion);
       cs.setOldestVersion(oldestVersion);
       refImpl.setOldestVersion(oldestVersion);
     }
     {
       int numPointReads = arbitrary.bounded(100);
       int numRangeReads = arbitrary.bounded(100);
-      int64_t v = std::max<int64_t>(writeVersion - arbitrary.bounded(10), 0);
+
+      int64_t v = std::max<int64_t>(writeVersion - (arbitrary.bounded(10)
+                                                        ? arbitrary.bounded(10)
+                                                        : arbitrary.next()),
+                                    0);
       auto *reads =
           new (arena) ConflictSet::ReadRange[numPointReads + numRangeReads];
       auto keys = set<std::string_view>(arena);

corpus/0e410d4488defd877d2c9690a26970f57ab78b0c

@@ -0,0 +1,4 @@
+*
+
+
+

corpus/22c23c2418773edaaff055ed387f82c8ad078f9f

@@ -0,0 +1,2 @@
+*
+