Track "max of max" versions in Node256

ConflictSet.cpp

@@ -308,6 +308,8 @@ struct Node256 : Node {
   constexpr static auto kType = Type_Node256;
   BitSet bitSet;
   Child children[256];
+  // Max of each "page" of 16 children
+  int64_t maxOfMax[16];
   uint8_t *partialKey() { return (uint8_t *)(this + 1); }
   void copyChildrenAndKeyFrom(const Node48 &other);
   void copyChildrenAndKeyFrom(const Node256 &other);
@@ -457,13 +459,16 @@ inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) {
 inline void Node256::copyChildrenAndKeyFrom(const Node48 &other) {
   memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
          kNodeCopySize);
-  memset(children, 0, sizeof(children));
   bitSet = other.bitSet;
+  memset(children, 0, sizeof(children));
+  memset(maxOfMax, 0, sizeof(maxOfMax));
   bitSet.forEachInRange(
       [&](int c) {
         children[c] = other.children[other.index[c]];
         assert(children[c].child->parent == &other);
         children[c].child->parent = this;
+        maxOfMax[c >> 4] =
+            std::max(maxOfMax[c >> 4], children[c].childMaxVersion);
       },
       0, 256);
   memcpy(partialKey(), &other + 1, partialKeyLen);
@@ -481,6 +486,7 @@ inline void Node256::copyChildrenAndKeyFrom(const Node256 &other) {
         children[c].child->parent = this;
       },
       0, 256);
+  memcpy(maxOfMax, other.maxOfMax, sizeof(maxOfMax));
   memcpy(partialKey(), &other + 1, partialKeyLen);
 }
 
@@ -562,9 +568,12 @@ template <class T> struct BoundedFreeListAllocator {
 
   T *allocate(int partialKeyCapacity) {
     T *result = allocate_helper(partialKeyCapacity);
-    if constexpr (std::is_same_v<T, Node3> || std::is_same_v<T, Node16>) {
+    if constexpr (!std::is_same_v<T, Node0>) {
       memset(result->children, 0, sizeof(result->children));
     }
+    if constexpr (std::is_same_v<T, Node256>) {
+      memset(result->maxOfMax, 0, sizeof(result->maxOfMax));
+    }
     return result;
   }
 
@@ -1689,17 +1698,35 @@ bool checkMaxBetweenExclusive(Node *n, int begin, int end,
     return result;
   }
   case Type_Node256: {
-    bool conflict[256] = {};
     auto *self = static_cast<Node256 *>(n);
-    self->bitSet.forEachInRange(
-        [&](int i) {
-          conflict[i] = self->children[i].childMaxVersion > readVersion;
-        },
-        begin, end);
-    // Should be vectorized
+    // Check the first page
+    if (self->maxOfMax[begin >> 4] > readVersion) {
+      bool result = true;
+      for (int i = 0; i < 16; ++i) {
+        int j = (begin & ~15) + i;
+        result &= !((self->children[j].childMaxVersion > readVersion) &
+                    (begin <= j) & (j < end));
+      }
+      if (!result) {
+        return result;
+      }
+    }
+    // Check the last page
+    if (end >= 1 && self->maxOfMax[(end - 1) >> 4] > readVersion) {
+      bool result = true;
+      for (int i = 0; i < 16; ++i) {
+        int j = ((end - 1) & ~15) + i;
+        result &= !((self->children[j].childMaxVersion > readVersion) &
+                    (begin <= j) & (j < end));
+      }
+      if (!result) {
+        return result;
+      }
+    }
+    // Check inner pages
     bool result = true;
-    for (auto c : conflict) {
-      result &= !c;
+    for (int i = (begin >> 4) + 1; i < (end - 1) >> 4; ++i) {
+      result &= self->maxOfMax[i] <= readVersion;
     }
     return result;
   }
@@ -2635,6 +2662,7 @@ void setMaxVersion(Node *n, ConflictSet::Impl *impl, int64_t newMax) {
     auto *n256 = static_cast<Node256 *>(n);
     assert(n256->bitSet.test(index));
     n256->children[index].childMaxVersion = newMax;
+    n256->maxOfMax[index >> 4] = std::max(n256->maxOfMax[index >> 4], newMax);
     return;
   }
   default:                   // GCOVR_EXCL_LINE