Copy Node members and set children pointers in copyChildrenAndKeyFrom

ConflictSet.cpp

@@ -237,6 +237,12 @@ struct Child {
   Node *child;
 };
 
+// copyChildrenAndKeyFrom is responsible for copying all
+// public members of Node, copying the partial key, logically copying the
+// children (converting representation if necessary), and updating all the
+// children's parent pointers. The caller must then insert the new node into the
+// tree.
+
 struct Node0 : Node {
   constexpr static auto kType = Type_Node0;
   uint8_t *partialKey() { return (uint8_t *)(this + 1); }
@@ -296,59 +302,73 @@ struct Node256 : Node {
 };
 
 inline void Node0::copyChildrenAndKeyFrom(const Node0 &other) {
-  assert(numChildren == other.numChildren);
+  memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
-  assert(partialKeyLen == other.partialKeyLen);
+         kNodeCopySize);
   memcpy(partialKey(), &other + 1, partialKeyLen);
 }
 
 inline void Node0::copyChildrenAndKeyFrom(const Node3 &other) {
-  assert(numChildren == other.numChildren);
+  memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
-  assert(partialKeyLen == other.partialKeyLen);
+         kNodeCopySize);
   memcpy(partialKey(), &other + 1, partialKeyLen);
 }
 
 inline void Node3::copyChildrenAndKeyFrom(const Node0 &other) {
-  assert(numChildren == other.numChildren);
+  memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
-  assert(partialKeyLen == other.partialKeyLen);
+         kNodeCopySize);
   memcpy(partialKey(), &other + 1, partialKeyLen);
 }
 
 inline void Node3::copyChildrenAndKeyFrom(const Node3 &other) {
-  assert(numChildren == other.numChildren);
+  memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
-  assert(partialKeyLen == other.partialKeyLen);
+         kNodeCopySize);
   memcpy(index, other.index,
          sizeof(*this) - offsetof(Node3, index) + partialKeyLen);
+  for (int i = 0; i < numChildren; ++i) {
+    assert(children[i].child->parent == &other);
+    children[i].child->parent = this;
+  }
 }
 
 inline void Node3::copyChildrenAndKeyFrom(const Node16 &other) {
-  assert(numChildren == other.numChildren);
+  memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
-  assert(partialKeyLen == other.partialKeyLen);
+         kNodeCopySize);
   memcpy(index, other.index, kMaxNodes);
   memcpy(children, other.children, kMaxNodes * sizeof(Child));
   memcpy(partialKey(), &other + 1, partialKeyLen);
+  for (int i = 0; i < numChildren; ++i) {
+    assert(children[i].child->parent == &other);
+    children[i].child->parent = this;
+  }
 }
 
 inline void Node16::copyChildrenAndKeyFrom(const Node3 &other) {
-  assert(numChildren == other.numChildren);
+  memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
-  assert(partialKeyLen == other.partialKeyLen);
+         kNodeCopySize);
   memcpy(index, other.index, Node3::kMaxNodes);
   memcpy(children, other.children, Node3::kMaxNodes * sizeof(Child));
   memcpy(partialKey(), &other + 1, partialKeyLen);
+  for (int i = 0; i < numChildren; ++i) {
+    assert(children[i].child->parent == &other);
+    children[i].child->parent = this;
+  }
 }
 
 inline void Node16::copyChildrenAndKeyFrom(const Node16 &other) {
-  assert(numChildren == other.numChildren);
+  memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
-  assert(partialKeyLen == other.partialKeyLen);
+         kNodeCopySize);
   memcpy(index, other.index, sizeof(index));
   for (int i = 0; i < numChildren; ++i) {
     children[i] = other.children[i];
+    assert(children[i].child->parent == &other);
+    children[i].child->parent = this;
   }
   memcpy(partialKey(), &other + 1, partialKeyLen);
 }
 
 inline void Node16::copyChildrenAndKeyFrom(const Node48 &other) {
-  assert(numChildren == other.numChildren);
+  memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
-  assert(partialKeyLen == other.partialKeyLen);
+         kNodeCopySize);
   int i = 0;
   other.bitSet.forEachInRange(
       [&](int c) {
@@ -359,6 +379,8 @@ inline void Node16::copyChildrenAndKeyFrom(const Node48 &other) {
         }
         index[i] = c;
         children[i] = other.children[other.index[c]];
+        assert(children[i].child->parent == &other);
+        children[i].child->parent = this;
         ++i;
       },
       0, 256);
@@ -366,8 +388,9 @@ inline void Node16::copyChildrenAndKeyFrom(const Node48 &other) {
 }
 
 inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) {
-  assert(numChildren == other.numChildren);
+  memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
-  assert(partialKeyLen == other.partialKeyLen);
+         kNodeCopySize);
+  assert(numChildren == Node16::kMaxNodes);
   memset(index, -1, sizeof(index));
   memcpy(partialKey(), &other + 1, partialKeyLen);
   bitSet.init();
@@ -377,22 +400,28 @@ inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) {
     bitSet.set(x);
     index[x] = i;
     children[i] = other.children[i];
+    assert(children[i].child->parent == &other);
+    children[i].child->parent = this;
     ++i;
   }
 }
 
 inline void Node48::copyChildrenAndKeyFrom(const Node48 &other) {
-  assert(numChildren == other.numChildren);
+  memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
-  assert(partialKeyLen == other.partialKeyLen);
+         kNodeCopySize);
   memcpy(&bitSet, &other.bitSet,
          offsetof(Node48, children) - offsetof(Node48, bitSet));
   for (int i = 0; i < numChildren; ++i) {
     children[i] = other.children[i];
+    assert(children[i].child->parent == &other);
+    children[i].child->parent = this;
   }
   memcpy(partialKey(), &other + 1, partialKeyLen);
 }
 
 inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) {
+  memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
+         kNodeCopySize);
   memset(index, -1, sizeof(index));
   nextFree = other.numChildren;
   bitSet = other.bitSet;
@@ -406,6 +435,8 @@ inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) {
         }
         index[c] = i;
         children[i] = other.children[c];
+        assert(children[i].child->parent == &other);
+        children[i].child->parent = this;
         ++i;
       },
       0, 256);
@@ -413,20 +444,33 @@ inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) {
 }
 
 inline void Node256::copyChildrenAndKeyFrom(const Node48 &other) {
+  memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
+         kNodeCopySize);
   for (int i = 0; i < 256; ++i) {
     children[i].child = nullptr;
   }
-
   bitSet = other.bitSet;
   bitSet.forEachInRange(
-      [&](int c) { children[c] = other.children[other.index[c]]; }, 0, 256);
+      [&](int c) {
+        children[c] = other.children[other.index[c]];
+        assert(children[c].child->parent == &other);
+        children[c].child->parent = this;
+      },
+      0, 256);
   memcpy(partialKey(), &other + 1, partialKeyLen);
 }
 
 inline void Node256::copyChildrenAndKeyFrom(const Node256 &other) {
+  memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
+         kNodeCopySize);
   bitSet = other.bitSet;
-  bitSet.forEachInRange([&](int c) { children[c] = other.children[c]; }, 0,
+  bitSet.forEachInRange(
-                        256);
+      [&](int c) {
+        children[c] = other.children[c];
+        assert(children[c].child->parent == &other);
+        children[c].child->parent = this;
+      },
+      0, 256);
   memcpy(partialKey(), &other + 1, partialKeyLen);
 }
 
@@ -760,28 +804,6 @@ int getChildGeq(Node *self, int child) {
   }
 }
 
-void setChildrenParents(Node3 *n) {
-  for (int i = 0; i < n->numChildren; ++i) {
-    n->children[i].child->parent = n;
-  }
-}
-
-void setChildrenParents(Node16 *n) {
-  for (int i = 0; i < n->numChildren; ++i) {
-    n->children[i].child->parent = n;
-  }
-}
-
-void setChildrenParents(Node48 *n) {
-  n->bitSet.forEachInRange(
-      [&](int i) { n->children[n->index[i]].child->parent = n; }, 0, 256);
-}
-
-void setChildrenParents(Node256 *n) {
-  n->bitSet.forEachInRange([&](int i) { n->children[i].child->parent = n; }, 0,
-                           256);
-}
-
 // Caller is responsible for assigning a non-null pointer to the returned
 // reference if null
 Node *&getOrCreateChild(Node *&self, uint8_t index,
@@ -827,8 +849,6 @@ Node *&getOrCreateChild(Node *&self, uint8_t index,
     auto *self0 = static_cast<Node0 *>(self);
 
     auto *newSelf = allocators->node3.allocate(self->partialKeyLen);
-    memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
-           kNodeCopySize);
     newSelf->copyChildrenAndKeyFrom(*self0);
     allocators->node0.release(self0);
     self = newSelf;
@@ -843,7 +863,6 @@ Node *&getOrCreateChild(Node *&self, uint8_t index,
              kNodeCopySize);
       newSelf->copyChildrenAndKeyFrom(*self3);
       allocators->node3.release(self3);
-      setChildrenParents(newSelf);
       self = newSelf;
       goto insert16;
     }
@@ -874,7 +893,6 @@ Node *&getOrCreateChild(Node *&self, uint8_t index,
              kNodeCopySize);
       newSelf->copyChildrenAndKeyFrom(*self16);
       allocators->node16.release(self16);
-      setChildrenParents(newSelf);
       self = newSelf;
       goto insert48;
     }
@@ -904,11 +922,8 @@ Node *&getOrCreateChild(Node *&self, uint8_t index,
     if (self->numChildren == 48) {
       auto *self48 = static_cast<Node48 *>(self);
       auto *newSelf = allocators->node256.allocate(self->partialKeyLen);
-      memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
-             kNodeCopySize);
       newSelf->copyChildrenAndKeyFrom(*self48);
       allocators->node48.release(self48);
-      setChildrenParents(newSelf);
       self = newSelf;
       goto insert256;
     }
@@ -968,8 +983,6 @@ void freeAndMakeCapacityAtLeast(Node *&self, int capacity,
   case Type_Node0: {
     auto *self0 = (Node0 *)self;
     auto *newSelf = allocators->node0.allocate(capacity);
-    memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
-           kNodeCopySize);
     newSelf->copyChildrenAndKeyFrom(*self0);
     getInTree(self, impl) = newSelf;
     if constexpr (kUseFreeList) {
@@ -983,11 +996,8 @@ void freeAndMakeCapacityAtLeast(Node *&self, int capacity,
   case Type_Node3: {
     auto *self3 = (Node3 *)self;
     auto *newSelf = allocators->node3.allocate(capacity);
-    memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
-           kNodeCopySize);
     newSelf->copyChildrenAndKeyFrom(*self3);
     getInTree(self, impl) = newSelf;
-    setChildrenParents(newSelf);
     if constexpr (kUseFreeList) {
       allocators->node3.release(self3);
     } else {
@@ -999,11 +1009,8 @@ void freeAndMakeCapacityAtLeast(Node *&self, int capacity,
   case Type_Node16: {
     auto *self16 = (Node16 *)self;
     auto *newSelf = allocators->node16.allocate(capacity);
-    memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
-           kNodeCopySize);
     newSelf->copyChildrenAndKeyFrom(*self16);
     getInTree(self, impl) = newSelf;
-    setChildrenParents(newSelf);
     if constexpr (kUseFreeList) {
       allocators->node16.release(self16);
     } else {
@@ -1015,11 +1022,8 @@ void freeAndMakeCapacityAtLeast(Node *&self, int capacity,
   case Type_Node48: {
     auto *self48 = (Node48 *)self;
     auto *newSelf = allocators->node48.allocate(capacity);
-    memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
-           kNodeCopySize);
     newSelf->copyChildrenAndKeyFrom(*self48);
     getInTree(self, impl) = newSelf;
-    setChildrenParents(newSelf);
     if constexpr (kUseFreeList) {
       allocators->node48.release(self48);
     } else {
@@ -1031,11 +1035,8 @@ void freeAndMakeCapacityAtLeast(Node *&self, int capacity,
   case Type_Node256: {
     auto *self256 = (Node256 *)self;
     auto *newSelf = allocators->node256.allocate(capacity);
-    memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
-           kNodeCopySize);
     newSelf->copyChildrenAndKeyFrom(*self256);
     getInTree(self, impl) = newSelf;
-    setChildrenParents(newSelf);
     if constexpr (kUseFreeList) {
       allocators->node256.release(self256);
     } else {
@@ -1078,8 +1079,6 @@ void maybeDownsize(Node *self, NodeAllocators *allocators,
     auto *self3 = (Node3 *)self;
     if (self->numChildren == 0) {
       auto *newSelf = allocators->node0.allocate(self->partialKeyLen);
-      memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
-             kNodeCopySize);
       newSelf->copyChildrenAndKeyFrom(*self3);
       getInTree(self, impl) = newSelf;
       allocators->node3.release(self3);
@@ -1127,10 +1126,7 @@ void maybeDownsize(Node *self, NodeAllocators *allocators,
     if (self->numChildren + int(self->entryPresent) < kMinChildrenNode16) {
       auto *self16 = (Node16 *)self;
       auto *newSelf = allocators->node3.allocate(self->partialKeyLen);
-      memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
-             kNodeCopySize);
       newSelf->copyChildrenAndKeyFrom(*self16);
-      setChildrenParents(newSelf);
       getInTree(self, impl) = newSelf;
       allocators->node16.release(self16);
     }
@@ -1139,26 +1135,7 @@ void maybeDownsize(Node *self, NodeAllocators *allocators,
     if (self->numChildren + int(self->entryPresent) < kMinChildrenNode48) {
       auto *self48 = (Node48 *)self;
       auto *newSelf = allocators->node16.allocate(self->partialKeyLen);
-      memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
+      newSelf->copyChildrenAndKeyFrom(*self48);
-             kNodeCopySize);
-      memcpy(newSelf->partialKey(), self48->partialKey(), self->partialKeyLen);
-
-      static_assert(Node16::kMaxNodes == kMinChildrenNode48 - 1);
-      int i = 0;
-      self48->bitSet.forEachInRange(
-          [&](int c) {
-            // Suppress a false positive -Waggressive-loop-optimizations warning
-            // in gcc. `assume` doesn't work for some reason.
-            if (!(i < Node16::kMaxNodes)) {
-              __builtin_unreachable(); // GCOVR_EXCL_LINE
-            }
-            newSelf->index[i] = c;
-            newSelf->children[i] = self48->children[self48->index[c]];
-            ++i;
-          },
-          0, 256);
-
-      setChildrenParents(newSelf);
       getInTree(self, impl) = newSelf;
       allocators->node48.release(self48);
     }
@@ -1167,10 +1144,7 @@ void maybeDownsize(Node *self, NodeAllocators *allocators,
     if (self->numChildren + int(self->entryPresent) < kMinChildrenNode256) {
       auto *self256 = (Node256 *)self;
       auto *newSelf = allocators->node48.allocate(self->partialKeyLen);
-      memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
-             kNodeCopySize);
       newSelf->copyChildrenAndKeyFrom(*self256);
-      setChildrenParents(newSelf);
       getInTree(self, impl) = newSelf;
       allocators->node256.release(self256);
     }