Downsize nodes in erase

ConflictSet.cpp

@@ -261,8 +261,8 @@ struct Node256 : Node {
 
 // Bound memory usage following the analysis in the ART paper
 
-constexpr int kBytesPerKey = 86;
+constexpr int kBytesPerKey = 121;
-constexpr int kMinChildrenNode4 = 2;
+constexpr int kMinChildrenNode4 = 1;
 constexpr int kMinChildrenNode16 = 5;
 constexpr int kMinChildrenNode48 = 17;
 constexpr int kMinChildrenNode256 = 49;
@@ -551,6 +551,12 @@ int getChildGeq(Node *self, int child) {
   return -1;
 }
 
+void setChildrenParents(Node4 *n) {
+  for (int i = 0; i < n->numChildren; ++i) {
+    n->children[i].child->parent = n;
+  }
+}
+
 void setChildrenParents(Node16 *n) {
   for (int i = 0; i < n->numChildren; ++i) {
     n->children[i].child->parent = n;
@@ -727,6 +733,132 @@ Node *nextLogical(Node *node) {
   return node;
 }
 
+// TODO fuse into erase child so we don't need to repeat branches on type
+void maybeDownsize(Node *self, NodeAllocators *allocators,
+                   ConflictSet::Impl *impl) {
+  switch (self->type) {
+  case Type::Node0:
+    __builtin_unreachable(); // GCOVR_EXCL_LINE
+  case Type::Node4: {
+    auto *self4 = (Node4 *)self;
+    if (self->numChildren == 0) {
+      auto *newSelf = allocators->node0.allocate(self->partialKeyLen);
+      memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
+             kNodeCopySize);
+      memcpy(newSelf->partialKey(), self4->partialKey(), self->partialKeyLen);
+
+      getInTree(self, impl) = newSelf;
+      allocators->node4.release(self4);
+    } else if (self->numChildren == 1) {
+      if (!self->entryPresent) {
+        auto *child = self4->children[0].child;
+        int minCapacity = self4->partialKeyLen + 1 + child->partialKeyLen;
+
+        if (minCapacity > child->partialKeyCapacity) {
+          // TODO resize child? It seems to be quite challenging to implement,
+          // since callers would now have to account for erase invalidating
+          // not on the search path. We could lower kBytesPerKey by doing this
+          // though.
+          return;
+        }
+
+        // Merge partial key with child
+#if DEBUG_VERBOSE && !defined(NDEBUG)
+        fprintf(stderr, "Merge %s into %s\n",
+                getSearchPathPrintable(self).c_str(),
+                getSearchPathPrintable(child).c_str());
+#endif
+
+        int64_t childMaxVersion = maxVersion(child, impl);
+
+        // Construct new partial key for child
+        memmove(child->partialKey() + self4->partialKeyLen + 1,
+                child->partialKey(), child->partialKeyLen);
+        memcpy(child->partialKey(), self4->partialKey(), self->partialKeyLen);
+        child->partialKey()[self4->partialKeyLen] = self4->index[0];
+        child->partialKeyLen += 1 + self4->partialKeyLen;
+
+        child->parent = self->parent;
+        child->parentsIndex = self->parentsIndex;
+
+        // Max versions are stored in the parent, so we need to update it now
+        // that we have a new parent.
+        maxVersion(child, impl) = childMaxVersion;
+
+        getInTree(self, impl) = child;
+        allocators->node4.release(self4);
+      }
+    }
+  } break;
+  case Type::Node16:
+    if (self->numChildren < kMinChildrenNode16) {
+      auto *self16 = (Node16 *)self;
+      auto *newSelf = allocators->node4.allocate(self->partialKeyLen);
+      memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
+             kNodeCopySize);
+      memcpy(newSelf->partialKey(), self16->partialKey(), self->partialKeyLen);
+      // TODO replace with memcpy?
+      for (int i = 0; i < 4; ++i) {
+        newSelf->index[i] = self16->index[i];
+        newSelf->children[i] = self16->children[i];
+      }
+      setChildrenParents(newSelf);
+      getInTree(self, impl) = newSelf;
+      allocators->node16.release(self16);
+    }
+    break;
+  case Type::Node48:
+    if (self->numChildren < kMinChildrenNode48) {
+      auto *self48 = (Node48 *)self;
+      auto *newSelf = allocators->node16.allocate(self->partialKeyLen);
+      memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
+             kNodeCopySize);
+      memcpy(newSelf->partialKey(), self48->partialKey(), self->partialKeyLen);
+
+      int i = 0;
+      self48->bitSet.forEachInRange(
+          [&](int c) {
+            // Suppress a false positive -Waggressive-loop-optimizations warning
+            // in gcc. `assume` doesn't work for some reason.
+            if (!(i < 16)) {
+              __builtin_unreachable();
+            }
+            newSelf->index[i] = c;
+            newSelf->children[i] = self48->children[self48->index[c]];
+            ++i;
+          },
+          0, 256);
+
+      setChildrenParents(newSelf);
+      getInTree(self, impl) = newSelf;
+      allocators->node48.release(self48);
+    }
+    break;
+  case Type::Node256:
+    if (self->numChildren < kMinChildrenNode256) {
+      auto *self256 = (Node256 *)self;
+      auto *newSelf = allocators->node48.allocate(self->partialKeyLen);
+      memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
+             kNodeCopySize);
+      memcpy(newSelf->partialKey(), self256->partialKey(), self->partialKeyLen);
+
+      newSelf->bitSet = self256->bitSet;
+      newSelf->bitSet.forEachInRange(
+          [&](int c) {
+            newSelf->index[c] = newSelf->nextFree;
+            newSelf->children[newSelf->nextFree] = self256->children[c];
+            ++newSelf->nextFree;
+          },
+          0, 256);
+
+      setChildrenParents(newSelf);
+      getInTree(self, impl) = newSelf;
+      allocators->node256.release(self256);
+    }
+    break;
+  }
+}
+
 // Precondition: self is not the root. May invalidate nodes along the search
 // path to self.
 Node *erase(Node *self, NodeAllocators *allocators, ConflictSet::Impl *impl) {
@@ -792,6 +924,8 @@ Node *erase(Node *self, NodeAllocators *allocators, ConflictSet::Impl *impl) {
   if (parent->numChildren == 0 && !parent->entryPresent &&
       parent->parent != nullptr) {
     erase(parent, allocators, impl);
+  } else {
+    maybeDownsize(parent, allocators, impl);
   }
   return result;
 }
@@ -2241,6 +2375,48 @@ Iterator firstGeq(Node *n, std::string_view key) {
   return total;
 }
 
+[[maybe_unused]] void checkMemoryBoundInvariants(Node *node, bool &success) {
+  int minNumChildren;
+  switch (node->type) {
+  case Type::Node0:
+    minNumChildren = 0;
+    break;
+  case Type::Node4:
+    minNumChildren = kMinChildrenNode4;
+    break;
+  case Type::Node16:
+    minNumChildren = kMinChildrenNode16;
+    break;
+  case Type::Node48:
+    minNumChildren = kMinChildrenNode48;
+    break;
+  case Type::Node256:
+    minNumChildren = kMinChildrenNode256;
+    break;
+  }
+  if (node->numChildren < minNumChildren) {
+    fprintf(stderr,
+            "%s has %d children, which is less than the minimum required %d\n",
+            getSearchPathPrintable(node).c_str(), node->numChildren,
+            minNumChildren);
+    success = false;
+  }
+  // if (node->numChildren > 0 &&
+  //     node->numChildren * node->partialKeyLen < node->partialKeyCapacity) {
+  //   fprintf(stderr,
+  //           "%s has %d children, partial key length %d, and partial key "
+  //           "capacity %d. It's required that nodes with children have
+  //           children "
+  //           "* length >= capacity\n",
+  //           getSearchPathPrintable(node).c_str(), node->numChildren,
+  //           node->partialKeyLen, node->partialKeyCapacity);
+  //   success = false;
+  // }
+  for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) {
+    auto *child = getChildExists(node, i);
+    checkMemoryBoundInvariants(child, success);
+  }
+}
 bool checkCorrectness(Node *node, int64_t oldestVersion,
                       ConflictSet::Impl *impl) {
   bool success = true;
@@ -2248,6 +2424,7 @@ bool checkCorrectness(Node *node, int64_t oldestVersion,
   checkParentPointers(node, success);
   checkMaxVersion(node, node, oldestVersion, success, impl);
   checkEntriesExist(node, success);
+  checkMemoryBoundInvariants(node, success);
 
   return success;
 }