Consolidate copyChildrenAndKeyFrom implementations

ConflictSet.cpp

@@ -81,7 +81,7 @@ struct BitSet {
   int firstSetGeq(int i) const;
 
   // Calls `f` with the index of each bit set in [begin, end)
-  template <class F> void forEachInRange(F f, int begin, int end) {
+  template <class F> void forEachInRange(F f, int begin, int end) const {
     // See section 3.1 in https://arxiv.org/pdf/1709.07821.pdf for details about
     // this approach
 
@@ -240,6 +240,9 @@ struct Child {
 struct Node0 : Node {
   constexpr static auto kType = Type_Node0;
   uint8_t *partialKey() { return (uint8_t *)(this + 1); }
+
+  void copyChildrenAndKeyFrom(const Node0 &other);
+  void copyChildrenAndKeyFrom(const struct Node3 &other);
 };
 
 struct Node3 : Node {
@@ -250,16 +253,11 @@ struct Node3 : Node {
   Child children[kMaxNodes];
   uint8_t *partialKey() { return (uint8_t *)(this + 1); }
 
+  void copyChildrenAndKeyFrom(const Node0 &other);
   void copyChildrenAndKeyFrom(const Node3 &other);
+  void copyChildrenAndKeyFrom(const struct Node16 &other);
 };
 
-inline void Node3::copyChildrenAndKeyFrom(const Node3 &other) {
-  assert(numChildren == other.numChildren);
-  assert(partialKeyLen == other.partialKeyLen);
-  memcpy(index, other.index,
-         sizeof(*this) - offsetof(Node3, index) + partialKeyLen);
-}
-
 struct Node16 : Node {
   constexpr static auto kType = Type_Node16;
   constexpr static auto kMaxNodes = 16;
@@ -268,9 +266,76 @@ struct Node16 : Node {
   Child children[kMaxNodes];
   uint8_t *partialKey() { return (uint8_t *)(this + 1); }
 
+  void copyChildrenAndKeyFrom(const Node3 &other);
   void copyChildrenAndKeyFrom(const Node16 &other);
+  void copyChildrenAndKeyFrom(const struct Node48 &other);
 };
 
+struct Node48 : Node {
+  constexpr static auto kType = Type_Node48;
+  constexpr static auto kMaxNodes = 48;
+  BitSet bitSet;
+  int8_t nextFree;
+  int8_t index[256];
+  Child children[kMaxNodes];
+
+  uint8_t *partialKey() { return (uint8_t *)(this + 1); }
+
+  void copyChildrenAndKeyFrom(const Node16 &other);
+  void copyChildrenAndKeyFrom(const Node48 &other);
+  void copyChildrenAndKeyFrom(const struct Node256 &other);
+};
+
+struct Node256 : Node {
+  constexpr static auto kType = Type_Node256;
+  BitSet bitSet;
+  Child children[256];
+  uint8_t *partialKey() { return (uint8_t *)(this + 1); }
+  void copyChildrenAndKeyFrom(const Node48 &other);
+  void copyChildrenAndKeyFrom(const Node256 &other);
+};
+
+inline void Node0::copyChildrenAndKeyFrom(const Node0 &other) {
+  assert(numChildren == other.numChildren);
+  assert(partialKeyLen == other.partialKeyLen);
+  memcpy(partialKey(), &other + 1, partialKeyLen);
+}
+
+inline void Node0::copyChildrenAndKeyFrom(const Node3 &other) {
+  assert(numChildren == other.numChildren);
+  assert(partialKeyLen == other.partialKeyLen);
+  memcpy(partialKey(), &other + 1, partialKeyLen);
+}
+
+inline void Node3::copyChildrenAndKeyFrom(const Node0 &other) {
+  assert(numChildren == other.numChildren);
+  assert(partialKeyLen == other.partialKeyLen);
+  memcpy(partialKey(), &other + 1, partialKeyLen);
+}
+
+inline void Node3::copyChildrenAndKeyFrom(const Node3 &other) {
+  assert(numChildren == other.numChildren);
+  assert(partialKeyLen == other.partialKeyLen);
+  memcpy(index, other.index,
+         sizeof(*this) - offsetof(Node3, index) + partialKeyLen);
+}
+
+inline void Node3::copyChildrenAndKeyFrom(const Node16 &other) {
+  assert(numChildren == other.numChildren);
+  assert(partialKeyLen == other.partialKeyLen);
+  memcpy(index, other.index, kMaxNodes);
+  memcpy(children, other.children, kMaxNodes * sizeof(Child));
+  memcpy(partialKey(), &other + 1, partialKeyLen);
+}
+
+inline void Node16::copyChildrenAndKeyFrom(const Node3 &other) {
+  assert(numChildren == other.numChildren);
+  assert(partialKeyLen == other.partialKeyLen);
+  memcpy(index, other.index, Node3::kMaxNodes);
+  memcpy(children, other.children, Node3::kMaxNodes * sizeof(Child));
+  memcpy(partialKey(), &other + 1, partialKeyLen);
+}
+
 inline void Node16::copyChildrenAndKeyFrom(const Node16 &other) {
   assert(numChildren == other.numChildren);
   assert(partialKeyLen == other.partialKeyLen);
@@ -281,18 +346,24 @@ inline void Node16::copyChildrenAndKeyFrom(const Node16 &other) {
   memcpy(partialKey(), &other + 1, partialKeyLen);
 }
 
-struct Node48 : Node {
+inline void Node16::copyChildrenAndKeyFrom(const Node48 &other) {
-  constexpr static auto kType = Type_Node48;
+  assert(numChildren == other.numChildren);
-  constexpr static auto kMaxNodes = 48;
+  assert(partialKeyLen == other.partialKeyLen);
-  BitSet bitSet;
+  int i = 0;
-  Child children[kMaxNodes];
+  other.bitSet.forEachInRange(
-  int8_t nextFree;
+      [&](int c) {
-  int8_t index[256];
+        // Suppress a false positive -Waggressive-loop-optimizations warning
-  uint8_t *partialKey() { return (uint8_t *)(this + 1); }
+        // in gcc. `assume` doesn't work for some reason.
-
+        if (!(i < Node16::kMaxNodes)) {
-  void copyChildrenAndKeyFrom(const Node16 &other);
+          __builtin_unreachable(); // GCOVR_EXCL_LINE
-  void copyChildrenAndKeyFrom(const struct Node256 &other);
+        }
-};
+        index[i] = c;
+        children[i] = other.children[other.index[c]];
+        ++i;
+      },
+      0, 256);
+  memcpy(partialKey(), &other + 1, partialKeyLen);
+}
 
 inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) {
   assert(numChildren == other.numChildren);
@@ -310,16 +381,19 @@ inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) {
   }
 }
 
-struct Node256 : Node {
+inline void Node48::copyChildrenAndKeyFrom(const Node48 &other) {
-  constexpr static auto kType = Type_Node256;
+  assert(numChildren == other.numChildren);
-  BitSet bitSet;
+  assert(partialKeyLen == other.partialKeyLen);
-  Child children[256];
+  memcpy(&bitSet, &other.bitSet,
-  uint8_t *partialKey() { return (uint8_t *)(this + 1); }
+         offsetof(Node48, children) - offsetof(Node48, bitSet));
-};
+  for (int i = 0; i < numChildren; ++i) {
+    children[i] = other.children[i];
+  }
+  memcpy(partialKey(), &other + 1, partialKeyLen);
+}
 
 inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) {
-  memcpy(partialKey(), &other + 1, partialKeyLen);
+  memset(index, -1, sizeof(index));
-
   nextFree = other.numChildren;
   bitSet = other.bitSet;
   int i = 0;
@@ -335,6 +409,25 @@ inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) {
         ++i;
       },
       0, 256);
+  memcpy(partialKey(), &other + 1, partialKeyLen);
+}
+
+inline void Node256::copyChildrenAndKeyFrom(const Node48 &other) {
+  for (int i = 0; i < 256; ++i) {
+    children[i].child = nullptr;
+  }
+
+  bitSet = other.bitSet;
+  bitSet.forEachInRange(
+      [&](int c) { children[c] = other.children[other.index[c]]; }, 0, 256);
+  memcpy(partialKey(), &other + 1, partialKeyLen);
+}
+
+inline void Node256::copyChildrenAndKeyFrom(const Node256 &other) {
+  bitSet = other.bitSet;
+  bitSet.forEachInRange([&](int c) { children[c] = other.children[c]; }, 0,
+                        256);
+  memcpy(partialKey(), &other + 1, partialKeyLen);
 }
 
 namespace {
@@ -736,7 +829,7 @@ Node *&getOrCreateChild(Node *&self, uint8_t index,
     auto *newSelf = allocators->node3.allocate(self->partialKeyLen);
     memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
            kNodeCopySize);
-    memcpy(newSelf->partialKey(), self0->partialKey(), self->partialKeyLen);
+    newSelf->copyChildrenAndKeyFrom(*self0);
     allocators->node0.release(self0);
     self = newSelf;
 
@@ -748,12 +841,7 @@ Node *&getOrCreateChild(Node *&self, uint8_t index,
       auto *newSelf = allocators->node16.allocate(self->partialKeyLen);
       memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
              kNodeCopySize);
-      memcpy(newSelf->partialKey(), self3->partialKey(), self->partialKeyLen);
+      newSelf->copyChildrenAndKeyFrom(*self3);
-      // TODO replace with memcpy?
-      for (int i = 0; i < Node3::kMaxNodes; ++i) {
-        newSelf->index[i] = self3->index[i];
-        newSelf->children[i] = self3->children[i];
-      }
       allocators->node3.release(self3);
       setChildrenParents(newSelf);
       self = newSelf;
@@ -816,18 +904,9 @@ Node *&getOrCreateChild(Node *&self, uint8_t index,
     if (self->numChildren == 48) {
       auto *self48 = static_cast<Node48 *>(self);
       auto *newSelf = allocators->node256.allocate(self->partialKeyLen);
-      for (int i = 0; i < 256; ++i) {
-        newSelf->children[i].child = nullptr;
-      }
       memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
              kNodeCopySize);
-      memcpy(newSelf->partialKey(), self48->partialKey(), self->partialKeyLen);
+      newSelf->copyChildrenAndKeyFrom(*self48);
-      newSelf->bitSet = self48->bitSet;
-      newSelf->bitSet.forEachInRange(
-          [&](int i) {
-            newSelf->children[i] = self48->children[self48->index[i]];
-          },
-          0, 256);
       allocators->node48.release(self48);
       setChildrenParents(newSelf);
       self = newSelf;
@@ -891,7 +970,7 @@ void freeAndMakeCapacityAtLeast(Node *&self, int capacity,
     auto *newSelf = allocators->node0.allocate(capacity);
     memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
            kNodeCopySize);
-    memcpy(newSelf->partialKey(), self0->partialKey(), self->partialKeyLen);
+    newSelf->copyChildrenAndKeyFrom(*self0);
     getInTree(self, impl) = newSelf;
     if constexpr (kUseFreeList) {
       allocators->node0.release(self0);
@@ -936,21 +1015,9 @@ void freeAndMakeCapacityAtLeast(Node *&self, int capacity,
   case Type_Node48: {
     auto *self48 = (Node48 *)self;
     auto *newSelf = allocators->node48.allocate(capacity);
-    memset(newSelf->index, -1, sizeof(newSelf->index));
     memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
            kNodeCopySize);
-    memcpy(newSelf->partialKey(), self48->partialKey(), self->partialKeyLen);
+    newSelf->copyChildrenAndKeyFrom(*self48);
-    newSelf->bitSet = self48->bitSet;
-    // TODO check codegen here?
-    newSelf->nextFree = 0;
-    newSelf->bitSet.forEachInRange(
-        [&](int c) {
-          int index = newSelf->nextFree;
-          newSelf->index[c] = index;
-          newSelf->children[index] = self48->children[self48->index[c]];
-          ++newSelf->nextFree;
-        },
-        0, 256);
     getInTree(self, impl) = newSelf;
     setChildrenParents(newSelf);
     if constexpr (kUseFreeList) {
@@ -964,15 +1031,9 @@ void freeAndMakeCapacityAtLeast(Node *&self, int capacity,
   case Type_Node256: {
     auto *self256 = (Node256 *)self;
     auto *newSelf = allocators->node256.allocate(capacity);
-    for (int i = 0; i < 256; ++i) {
-      newSelf->children[i].child = nullptr;
-    }
     memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
            kNodeCopySize);
-    memcpy(newSelf->partialKey(), self256->partialKey(), self->partialKeyLen);
+    newSelf->copyChildrenAndKeyFrom(*self256);
-    newSelf->bitSet = self256->bitSet;
-    newSelf->bitSet.forEachInRange(
-        [&](int c) { newSelf->children[c] = self256->children[c]; }, 0, 256);
     getInTree(self, impl) = newSelf;
     setChildrenParents(newSelf);
     if constexpr (kUseFreeList) {
@@ -1019,12 +1080,10 @@ void maybeDownsize(Node *self, NodeAllocators *allocators,
       auto *newSelf = allocators->node0.allocate(self->partialKeyLen);
       memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
              kNodeCopySize);
-      memcpy(newSelf->partialKey(), self3->partialKey(), self->partialKeyLen);
+      newSelf->copyChildrenAndKeyFrom(*self3);
-
       getInTree(self, impl) = newSelf;
       allocators->node3.release(self3);
-    } else if (self->numChildren == 1) {
+    } else if (self->numChildren == 1 && !self->entryPresent) {
-      if (!self->entryPresent) {
       auto *child = self3->children[0].child;
       int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen;
 
@@ -1063,7 +1122,6 @@ void maybeDownsize(Node *self, NodeAllocators *allocators,
       getInTree(self, impl) = child;
       allocators->node3.release(self3);
     }
-    }
   } break;
   case Type_Node16:
     if (self->numChildren + int(self->entryPresent) < kMinChildrenNode16) {
@@ -1071,12 +1129,7 @@ void maybeDownsize(Node *self, NodeAllocators *allocators,
       auto *newSelf = allocators->node3.allocate(self->partialKeyLen);
       memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
              kNodeCopySize);
-      memcpy(newSelf->partialKey(), self16->partialKey(), self->partialKeyLen);
+      newSelf->copyChildrenAndKeyFrom(*self16);
-      static_assert(Node3::kMaxNodes == kMinChildrenNode16 - 1);
-      for (int i = 0; i < Node3::kMaxNodes; ++i) {
-        newSelf->index[i] = self16->index[i];
-        newSelf->children[i] = self16->children[i];
-      }
       setChildrenParents(newSelf);
       getInTree(self, impl) = newSelf;
       allocators->node16.release(self16);
@@ -1114,7 +1167,6 @@ void maybeDownsize(Node *self, NodeAllocators *allocators,
     if (self->numChildren + int(self->entryPresent) < kMinChildrenNode256) {
       auto *self256 = (Node256 *)self;
       auto *newSelf = allocators->node48.allocate(self->partialKeyLen);
-      memset(newSelf->index, -1, sizeof(newSelf->index));
       memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
              kNodeCopySize);
       newSelf->copyChildrenAndKeyFrom(*self256);