Relax capacity property so that it _eventually_ needs to hold

Doing otherwise turned out to be prohibitively inconvenient

closes #9

ConflictSet.cpp

@@ -273,8 +273,12 @@ static_assert(sizeof(Node16) < kMinChildrenNode16 * kBytesPerKey);
 static_assert(sizeof(Node4) < kMinChildrenNode4 * kBytesPerKey);
 static_assert(sizeof(Node0) < kBytesPerKey);
 
-template <class T, int64_t kMemoryBound = (1 << 20),
-          int64_t kMaxIndividual = (1 << 10)>
+// setOldestVersion will additionally try to maintain this property:
+//     `max(children, 1) * length >= capacity`
+//
+// Which should give us the budget to pay for the key bytes
+
+template <class T, int64_t kMemoryBound = (1 << 20)>
 struct BoundedFreeListAllocator {
   static_assert(sizeof(T) >= sizeof(void *));
   static_assert(std::derived_from<T, Node>);
@@ -310,8 +314,7 @@ struct BoundedFreeListAllocator {
     --liveAllocations;
 #endif
     static_assert(std::is_trivially_destructible_v<T>);
-    if (sizeof(T) + p->partialKeyCapacity > kMaxIndividual ||
-        freeListBytes >= kMemoryBound) {
+    if (freeListBytes >= kMemoryBound) {
       return free(p);
     }
     memcpy((void *)p, &freeList, sizeof(freeList));
@@ -733,6 +736,97 @@ Node *nextLogical(Node *node) {
   return node;
 }
 
+// Fix larger-than-desired capacities. Does not return nodes to freelists,
+// since that wouldn't actually reclaim the memory used for partial key
+// capacity.
+void maybeDecreaseCapacity(Node *&self, NodeAllocators *allocators,
+                           ConflictSet::Impl *impl) {
+  const int maxCapacity =
+      std::max<int>(self->numChildren, 1) * self->partialKeyLen;
+  if (self->partialKeyCapacity <= maxCapacity) {
+    return;
+  }
+
+  switch (self->type) {
+  case Type::Node0: {
+    auto *self0 = (Node0 *)self;
+    auto *newSelf = allocators->node0.allocate(maxCapacity);
+    memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
+           kNodeCopySize);
+    memcpy(newSelf->partialKey(), self0->partialKey(), self->partialKeyLen);
+    getInTree(self, impl) = newSelf;
+    free(self0);
+    self = newSelf;
+  } break;
+  case Type::Node4: {
+    auto *self4 = (Node4 *)self;
+    auto *newSelf = allocators->node4.allocate(maxCapacity);
+    memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
+           kNodeCopySize);
+    memcpy(newSelf->partialKey(), self4->partialKey(), self->partialKeyLen);
+    // TODO replace with memcpy?
+    for (int i = 0; i < 4; ++i) {
+      newSelf->index[i] = self4->index[i];
+      newSelf->children[i] = self4->children[i];
+    }
+    getInTree(self, impl) = newSelf;
+    setChildrenParents(newSelf);
+    free(self4);
+    self = newSelf;
+  } break;
+  case Type::Node16: {
+    auto *self16 = (Node16 *)self;
+    auto *newSelf = allocators->node16.allocate(maxCapacity);
+    memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
+           kNodeCopySize);
+    memcpy(newSelf->partialKey(), self16->partialKey(), self->partialKeyLen);
+    // TODO replace with memcpy?
+    for (int i = 0; i < 16; ++i) {
+      newSelf->index[i] = self16->index[i];
+      newSelf->children[i] = self16->children[i];
+    }
+    getInTree(self, impl) = newSelf;
+    setChildrenParents(newSelf);
+    free(self16);
+    self = newSelf;
+  } break;
+  case Type::Node48: {
+    auto *self48 = (Node48 *)self;
+    auto *newSelf = allocators->node48.allocate(maxCapacity);
+    memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
+           kNodeCopySize);
+    memcpy(newSelf->partialKey(), self48->partialKey(), self->partialKeyLen);
+    newSelf->bitSet = self48->bitSet;
+    newSelf->bitSet.forEachInRange(
+        [&](int c) {
+          int index = newSelf->nextFree;
+          newSelf->index[c] = index;
+          newSelf->children[index] = self48->children[self48->index[c]];
+          ++newSelf->nextFree;
+        },
+        0, 256);
+    getInTree(self, impl) = newSelf;
+    setChildrenParents(newSelf);
+    free(self48);
+    self = newSelf;
+  } break;
+  case Type::Node256: {
+    auto *self256 = (Node256 *)self;
+    auto *newSelf = allocators->node256.allocate(maxCapacity);
+    memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin,
+           kNodeCopySize);
+    memcpy(newSelf->partialKey(), self256->partialKey(), self->partialKeyLen);
+    newSelf->bitSet = self256->bitSet;
+    newSelf->bitSet.forEachInRange(
+        [&](int c) { newSelf->children[c] = self256->children[c]; }, 0, 256);
+    getInTree(self, impl) = newSelf;
+    setChildrenParents(newSelf);
+    free(self256);
+    self = newSelf;
+  } break;
+  }
+}
+
 // TODO fuse into erase child so we don't need to repeat branches on type
 void maybeDownsize(Node *self, NodeAllocators *allocators,
                    ConflictSet::Impl *impl) {
@@ -1749,6 +1843,10 @@ template <bool kBegin>
         memmove(old->partialKey(), old->partialKey() + partialKeyIndex + 1,
                 old->partialKeyLen - (partialKeyIndex + 1));
         old->partialKeyLen -= partialKeyIndex + 1;
+
+        // We would consider decreasing capacity here, but we can't invalidate
+        // old since it's not on the search path. setOldestVersion will clean it
+        // up.
       }
       key = key.subspan(partialKeyIndex, key.size() - partialKeyIndex);
 
@@ -2045,6 +2143,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
         assert(n->entry.rangeVersion <= oldestVersion);
         n = erase(n, &allocators, this);
       } else {
+        maybeDecreaseCapacity(n, &allocators, this);
         n = nextLogical(n);
       }
     }
@@ -2401,22 +2500,13 @@ Iterator firstGeq(Node *n, std::string_view key) {
             minNumChildren);
     success = false;
   }
-  // if (node->numChildren > 0 &&
-  //     node->numChildren * node->partialKeyLen < node->partialKeyCapacity) {
-  //   fprintf(stderr,
-  //           "%s has %d children, partial key length %d, and partial key "
-  //           "capacity %d. It's required that nodes with children have
-  //           children "
-  //           "* length >= capacity\n",
-  //           getSearchPathPrintable(node).c_str(), node->numChildren,
-  //           node->partialKeyLen, node->partialKeyCapacity);
-  //   success = false;
-  // }
+  // TODO check that the max capacity property eventually holds
   for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) {
     auto *child = getChildExists(node, i);
     checkMemoryBoundInvariants(child, success);
   }
 }
+
 bool checkCorrectness(Node *node, int64_t oldestVersion,
                       ConflictSet::Impl *impl) {
   bool success = true;