Add forwarding pointers + deferred release

ConflictSet.cpp

@@ -262,7 +262,11 @@ private:
 struct Node {
 
   /* begin section that's copied to the next node */
-  Entry entry;
+  union {
+    Entry entry;
+    /* Set to the forwarding point for this node if deferredReleased is set */
+    Node *forwardTo;
+  };
   Node *parent;
   int32_t partialKeyLen;
   int16_t numChildren;
@@ -272,9 +276,19 @@ struct Node {
   uint8_t parentsIndex;
   /* end section that's copied to the next node */
 
+  /* If set, this node has been replaced and the next node in the forwarding
+   * chain is `forwardTo`*/
+  bool deferredReleased;
+
   uint8_t *partialKey();
-  Type getType() const { return type; }
-  int32_t getCapacity() const { return partialKeyCapacity; }
+  Type getType() const {
+    assert(!deferredReleased);
+    return type;
+  }
+  int32_t getCapacity() const {
+    assert(!deferredReleased);
+    return partialKeyCapacity;
+  }
 
 private:
   template <class T> friend struct BoundedFreeListAllocator;
@@ -305,7 +319,10 @@ constexpr int kNodeCopySize =
 struct Node0 : Node {
   constexpr static auto kType = Type_Node0;
 
-  uint8_t *partialKey() { return (uint8_t *)(this + 1); }
+  uint8_t *partialKey() {
+    assert(!deferredReleased);
+    return (uint8_t *)(this + 1);
+  }
   void copyChildrenAndKeyFrom(const Node0 &other);
   void copyChildrenAndKeyFrom(const struct Node3 &other);
   size_t size() const { return sizeof(Node0) + getCapacity(); }
@@ -320,7 +337,10 @@ struct Node3 : Node {
   // Sorted
   uint8_t index[kMaxNodes];
 
-  uint8_t *partialKey() { return (uint8_t *)(this + 1); }
+  uint8_t *partialKey() {
+    assert(!deferredReleased);
+    return (uint8_t *)(this + 1);
+  }
   void copyChildrenAndKeyFrom(const Node0 &other);
   void copyChildrenAndKeyFrom(const Node3 &other);
   void copyChildrenAndKeyFrom(const struct Node16 &other);
@@ -336,7 +356,10 @@ struct Node16 : Node {
   // Sorted
   uint8_t index[kMaxNodes];
 
-  uint8_t *partialKey() { return (uint8_t *)(this + 1); }
+  uint8_t *partialKey() {
+    assert(!deferredReleased);
+    return (uint8_t *)(this + 1);
+  }
   void copyChildrenAndKeyFrom(const Node3 &other);
   void copyChildrenAndKeyFrom(const Node16 &other);
   void copyChildrenAndKeyFrom(const struct Node48 &other);
@@ -358,7 +381,10 @@ struct Node48 : Node {
   uint8_t reverseIndex[kMaxNodes];
   int8_t index[256];
 
-  uint8_t *partialKey() { return (uint8_t *)(this + 1); }
+  uint8_t *partialKey() {
+    assert(!deferredReleased);
+    return (uint8_t *)(this + 1);
+  }
   void copyChildrenAndKeyFrom(const Node16 &other);
   void copyChildrenAndKeyFrom(const Node48 &other);
   void copyChildrenAndKeyFrom(const struct Node256 &other);
@@ -378,7 +404,10 @@ struct Node256 : Node {
   InternalVersionT childMaxVersion[kMaxNodes];
   InternalVersionT maxOfMax[kMaxOfMaxTotalPages];
 
-  uint8_t *partialKey() { return (uint8_t *)(this + 1); }
+  uint8_t *partialKey() {
+    assert(!deferredReleased);
+    return (uint8_t *)(this + 1);
+  }
   void copyChildrenAndKeyFrom(const Node48 &other);
   void copyChildrenAndKeyFrom(const Node256 &other);
   size_t size() const { return sizeof(Node256) + getCapacity(); }
@@ -679,6 +708,7 @@ template <class T> struct BoundedFreeListAllocator {
   T *allocate(int partialKeyCapacity) {
     T *result = allocate_helper(partialKeyCapacity);
     result->endOfRange = false;
+    result->deferredReleased = false;
     if constexpr (!std::is_same_v<T, Node0>) {
       memset(result->children, 0, sizeof(result->children));
       const auto z = InternalVersionT::zero;
@@ -821,7 +851,42 @@ struct WriteContext {
     }
   }
 
+  // Place in a list to be released in the next call to releaseDeferred.
+  void deferRelease(Node *n) {
+    n->parent = deferredList;
+    deferredList = n;
+  }
+  // Release all nodes passed to deferRelease since the last call to
+  // releaseDeferred.
+  void releaseDeferred() {
+    for (Node *n = std::exchange(deferredList, nullptr); n != nullptr;) {
+      auto *tmp = n;
+      n = n->parent;
+      switch (tmp->getType()) {
+      case Type_Node0:
+        release(static_cast<Node0 *>(tmp));
+        break;
+      case Type_Node3:
+        release(static_cast<Node3 *>(tmp));
+        break;
+      case Type_Node16:
+        release(static_cast<Node16 *>(tmp));
+        break;
+      case Type_Node48:
+        release(static_cast<Node48 *>(tmp));
+        break;
+      case Type_Node256:
+        release(static_cast<Node256 *>(tmp));
+        break;
+      default:                   // GCOVR_EXCL_LINE
+        __builtin_unreachable(); // GCOVR_EXCL_LINE
+      }
+    }
+  }
+
 private:
+  Node *deferredList = nullptr;
+
   BoundedFreeListAllocator<Node0> node0;
   BoundedFreeListAllocator<Node3> node3;
   BoundedFreeListAllocator<Node16> node16;