Improve copyChildrenAndKeyFrom-related codegen

ConflictSet.cpp

@@ -253,7 +253,7 @@ struct Node3 : Node {
   void copyChildrenAndKeyFrom(const Node3 &other);
 };
 
-void Node3::copyChildrenAndKeyFrom(const Node3 &other) {
+inline void Node3::copyChildrenAndKeyFrom(const Node3 &other) {
   assert(numChildren == other.numChildren);
   assert(partialKeyLen == other.partialKeyLen);
   memcpy(index, other.index,
@@ -271,7 +271,7 @@ struct Node16 : Node {
   void copyChildrenAndKeyFrom(const Node16 &other);
 };
 
-void Node16::copyChildrenAndKeyFrom(const Node16 &other) {
+inline void Node16::copyChildrenAndKeyFrom(const Node16 &other) {
   assert(numChildren == other.numChildren);
   assert(partialKeyLen == other.partialKeyLen);
   memcpy(index, other.index, sizeof(index));
@@ -283,13 +283,33 @@ void Node16::copyChildrenAndKeyFrom(const Node16 &other) {
 
 struct Node48 : Node {
   constexpr static auto kType = Type_Node48;
+  constexpr static auto kMaxNodes = 48;
   BitSet bitSet;
-  Child children[48];
+  Child children[kMaxNodes];
   int8_t nextFree;
   int8_t index[256];
   uint8_t *partialKey() { return (uint8_t *)(this + 1); }
+
+  void copyChildrenAndKeyFrom(const Node16 &other);
+  void copyChildrenAndKeyFrom(const struct Node256 &other);
 };
 
+inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) {
+  assert(numChildren == other.numChildren);
+  assert(partialKeyLen == other.partialKeyLen);
+  memset(index, -1, sizeof(index));
+  memcpy(partialKey(), &other + 1, partialKeyLen);
+  bitSet.init();
+  nextFree = Node16::kMaxNodes;
+  int i = 0;
+  for (auto x : other.index) {
+    bitSet.set(x);
+    index[x] = i;
+    children[i] = other.children[i];
+    ++i;
+  }
+}
+
 struct Node256 : Node {
   constexpr static auto kType = Type_Node256;
   BitSet bitSet;
@@ -297,6 +317,26 @@ struct Node256 : Node {
   uint8_t *partialKey() { return (uint8_t *)(this + 1); }
 };
 
+inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) {
+  memcpy(partialKey(), &other + 1, partialKeyLen);
+
+  nextFree = other.numChildren;
+  bitSet = other.bitSet;
+  int i = 0;
+  bitSet.forEachInRange(
+      [&](int c) {
+        // Suppress a false positive -Waggressive-loop-optimizations warning
+        // in gcc. `assume` doesn't work for some reason.
+        if (!(i < Node48::kMaxNodes)) {
+          __builtin_unreachable(); // GCOVR_EXCL_LINE
+        }
+        index[c] = i;
+        children[i] = other.children[c];
+        ++i;
+      },
+      0, 256);
+}
+
 namespace {
 std::string getSearchPathPrintable(Node *n);
 }
@@ -742,20 +782,9 @@ Node *&getOrCreateChild(Node *&self, uint8_t index,
     if (self->numChildren == Node16::kMaxNodes) {
       auto *self16 = static_cast<Node16 *>(self);
       auto *newSelf = allocators->node48.allocate(self->partialKeyLen);
-      memset(newSelf->index, -1, sizeof(newSelf->index));
       memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
              kNodeCopySize);
-      memcpy(newSelf->partialKey(), self16->partialKey(), self->partialKeyLen);
+      newSelf->copyChildrenAndKeyFrom(*self16);
-      newSelf->bitSet.init();
-      newSelf->nextFree = Node16::kMaxNodes;
-      int i = 0;
-      for (auto x : self16->index) {
-        newSelf->bitSet.set(x);
-        newSelf->children[i] = self16->children[i];
-        newSelf->index[x] = i;
-        ++i;
-      }
-      assert(i == Node16::kMaxNodes);
       allocators->node16.release(self16);
       setChildrenParents(newSelf);
       self = newSelf;
@@ -1043,7 +1072,6 @@ void maybeDownsize(Node *self, NodeAllocators *allocators,
       memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
              kNodeCopySize);
       memcpy(newSelf->partialKey(), self16->partialKey(), self->partialKeyLen);
-      // TODO replace with memcpy?
       static_assert(Node3::kMaxNodes == kMinChildrenNode16 - 1);
       for (int i = 0; i < Node3::kMaxNodes; ++i) {
         newSelf->index[i] = self16->index[i];
@@ -1089,18 +1117,7 @@ void maybeDownsize(Node *self, NodeAllocators *allocators,
       memset(newSelf->index, -1, sizeof(newSelf->index));
       memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
              kNodeCopySize);
-      memcpy(newSelf->partialKey(), self256->partialKey(), self->partialKeyLen);
+      newSelf->copyChildrenAndKeyFrom(*self256);
-
-      newSelf->nextFree = 0;
-      newSelf->bitSet = self256->bitSet;
-      newSelf->bitSet.forEachInRange(
-          [&](int c) {
-            newSelf->index[c] = newSelf->nextFree;
-            newSelf->children[newSelf->nextFree] = self256->children[c];
-            ++newSelf->nextFree;
-          },
-          0, 256);
-
       setChildrenParents(newSelf);
       getInTree(self, impl) = newSelf;
       allocators->node256.release(self256);