Use a free list

malloc and free were showing up in profiles of fdbserver

ConflictSet.cpp

@@ -43,6 +43,46 @@ struct Entry {
   int64_t rangeVersion;
 };
 
+template <class T, size_t kMemoryBound = (1 << 20)>
+struct BoundedFreeListAllocator {
+  static_assert(sizeof(T) >= sizeof(void *));
+
+  T *allocate() {
+    if (freeListSize == 0) {
+      assert(freeList == nullptr);
+      return new (safe_malloc(sizeof(T))) T;
+    }
+    assert(freeList != nullptr);
+    void *result = freeList;
+    memcpy(&freeList, freeList, sizeof(freeList));
+    --freeListSize;
+    return new (result) T;
+  }
+
+  void release(T *p) {
+    p->~T();
+    if (freeListSize == kMaxFreeListSize) {
+      return free(p);
+    }
+    memcpy(p, &freeList, sizeof(freeList));
+    freeList = p;
+    ++freeListSize;
+  }
+
+  ~BoundedFreeListAllocator() {
+    for (void *iter = freeList; iter != nullptr;) {
+      auto *tmp = iter;
+      memcpy(&iter, iter, sizeof(void *));
+      free(tmp);
+    }
+  }
+
+private:
+  static constexpr int kMaxFreeListSize = kMemoryBound / sizeof(T);
+  int freeListSize = 0;
+  void *freeList = nullptr;
+};
+
 enum class Type : int8_t {
   Node4,
   Node16,
@@ -75,8 +115,6 @@ struct Node4 : Node {
   Node4() { this->type = Type::Node4; }
 };
 
-Node *newNode() { return new (safe_malloc(sizeof(Node4))) Node4; }
-
 struct Node16 : Node {
   // Sorted
   uint8_t index[16] = {};
@@ -158,6 +196,13 @@ struct Node256 : Node {
   Node256() { this->type = Type::Node256; }
 };
 
+struct NodeAllocators {
+  BoundedFreeListAllocator<Node4> node4;
+  BoundedFreeListAllocator<Node16> node16;
+  BoundedFreeListAllocator<Node48> node48;
+  BoundedFreeListAllocator<Node256> node256;
+};
+
 int getNodeIndex(Node4 *self, uint8_t index) {
   for (int i = 0; i < self->numChildren; ++i) {
     if (self->index[i] == index) {
@@ -330,7 +375,8 @@ void setChildrenParents(Node *node) {
 
 // Caller is responsible for assigning a non-null pointer to the returned
 // reference if null
-Node *&getOrCreateChild(Node *&self, uint8_t index) {
+Node *&getOrCreateChild(Node *&self, uint8_t index,
+                        NodeAllocators *allocators) {
   if (self->type == Type::Node4) {
     auto *self4 = static_cast<Node4 *>(self);
     {
@@ -340,11 +386,12 @@ Node *&getOrCreateChild(Node *&self, uint8_t index) {
       }
     }
     if (self->numChildren == 4) {
-      auto *newSelf = new (safe_malloc(sizeof(Node16))) Node16;
+      auto *newSelf = allocators->node16.allocate();
       memcpy((void *)newSelf, self, offsetof(Node, type));
       memcpy(newSelf->index, self4->index, 4);
       memcpy(newSelf->children, self4->children, 4 * sizeof(void *));
-      free(std::exchange(self, newSelf));
+      allocators->node4.release(self4);
+      self = newSelf;
       setChildrenParents(self);
       goto insert16;
     } else {
@@ -374,7 +421,7 @@ Node *&getOrCreateChild(Node *&self, uint8_t index) {
       }
     }
     if (self->numChildren == 16) {
-      auto *newSelf = new (safe_malloc(sizeof(Node48))) Node48;
+      auto *newSelf = allocators->node48.allocate();
       memcpy((void *)newSelf, self, offsetof(Node, type));
       newSelf->nextFree = 16;
       int i = 0;
@@ -385,7 +432,8 @@ Node *&getOrCreateChild(Node *&self, uint8_t index) {
         ++i;
       }
       assert(i == 16);
-      free(std::exchange(self, newSelf));
+      allocators->node16.release(self16);
+      self = newSelf;
       setChildrenParents(self);
       goto insert48;
     } else {
@@ -412,7 +460,7 @@ Node *&getOrCreateChild(Node *&self, uint8_t index) {
       return self48->children[self48->index[index]];
     }
     if (self->numChildren == 48) {
-      auto *newSelf = new (safe_malloc(sizeof(Node256))) Node256;
+      auto *newSelf = allocators->node256.allocate();
       memcpy((void *)newSelf, self, offsetof(Node, type));
       for (int i = 0; i < 256; ++i) {
         if (self48->bitSet.test(i)) {
@@ -420,7 +468,7 @@ Node *&getOrCreateChild(Node *&self, uint8_t index) {
           newSelf->children[i] = self48->children[self48->index[i]];
         }
       }
-      free(std::exchange(self, newSelf));
+      allocators->node48.release(self48);
       self = newSelf;
       setChildrenParents(self);
       goto insert256;
@@ -444,8 +492,24 @@ Node *&getOrCreateChild(Node *&self, uint8_t index) {
 }
 
 // Precondition - an entry for index must exist in the node
-void eraseChild(Node *self, uint8_t index) {
-  free(getChildExists(self, index));
+void eraseChild(Node *self, uint8_t index, NodeAllocators *allocators) {
+  auto *child = getChildExists(self, index);
+  switch (child->type) {
+  case Type::Node4:
+    allocators->node4.release((Node4 *)child);
+    break;
+  case Type::Node16:
+    allocators->node16.release((Node16 *)child);
+    break;
+  case Type::Node48:
+    allocators->node48.release((Node48 *)child);
+    break;
+  case Type::Node256:
+    allocators->node256.release((Node256 *)child);
+    break;
+  case Type::Invalid:
+    __builtin_unreachable(); // GCOVR_EXCL_LINE
+  }
 
   if (self->type == Type::Node4) {
     auto *self4 = static_cast<Node4 *>(self);
@@ -484,7 +548,7 @@ void eraseChild(Node *self, uint8_t index) {
   --self->numChildren;
   if (self->numChildren == 0 && !self->entryPresent &&
       self->parent != nullptr) {
-    eraseChild(self->parent, self->parentsIndex);
+    eraseChild(self->parent, self->parentsIndex, allocators);
   }
 }
 
@@ -1315,7 +1379,8 @@ bool checkRangeRead(Node *n, std::span<const uint8_t> begin,
 // !entryPresent. The search path of the result's parent will have
 // `maxVersion` at least `writeVersion` as a postcondition.
 [[nodiscard]] Node *insert(Node **self_, std::span<const uint8_t> key,
-                           int64_t writeVersion, bool begin) {
+                           int64_t writeVersion, bool begin,
+                           NodeAllocators *allocators) {
   for (;;) {
     auto &self = *self_;
     // Handle an existing partial key
@@ -1324,14 +1389,15 @@ bool checkRangeRead(Node *n, std::span<const uint8_t> begin,
       if (partialKeyIndex == int(key.size()) ||
           self->partialKey[partialKeyIndex] != key[partialKeyIndex]) {
         auto *old = self;
-        self = newNode();
+        self = allocators->node4.allocate();
         self->maxVersion = old->maxVersion;
         self->partialKeyLen = partialKeyIndex;
         self->parent = old->parent;
         self->parentsIndex = old->parentsIndex;
         memcpy(self->partialKey, old->partialKey, partialKeyIndex);
 
-        getOrCreateChild(self, old->partialKey[partialKeyIndex]) = old;
+        getOrCreateChild(self, old->partialKey[partialKeyIndex], allocators) =
+            old;
         old->parent = self;
         old->parentsIndex = old->partialKey[partialKeyIndex];
 
@@ -1362,9 +1428,9 @@ bool checkRangeRead(Node *n, std::span<const uint8_t> begin,
       self->maxVersion = std::max(self->maxVersion, writeVersion);
     }
 
-    auto &child = getOrCreateChild(self, key.front());
+    auto &child = getOrCreateChild(self, key.front(), allocators);
     if (!child) {
-      child = newNode();
+      child = allocators->node4.allocate();
       child->parent = self;
       child->parentsIndex = key.front();
       child->maxVersion =
@@ -1395,8 +1461,9 @@ void destroyTree(Node *root) {
 }
 
 void addPointWrite(Node *&root, int64_t oldestVersion,
-                   std::span<const uint8_t> key, int64_t writeVersion) {
-  auto *n = insert(&root, key, writeVersion, true);
+                   std::span<const uint8_t> key, int64_t writeVersion,
+                   NodeAllocators *allocators) {
+  auto *n = insert(&root, key, writeVersion, true, allocators);
   if (!n->entryPresent) {
     auto *p = nextLogical(n);
     n->entryPresent = true;
@@ -1411,7 +1478,8 @@ void addPointWrite(Node *&root, int64_t oldestVersion,
 }
 
 void addWriteRange(Node *&root, int64_t oldestVersion,
-                   const ConflictSet::WriteRange &w) {
+                   const ConflictSet::WriteRange &w,
+                   NodeAllocators *allocators) {
 
   auto begin = std::span<const uint8_t>(w.begin.p, w.begin.len);
   auto end = std::span<const uint8_t>(w.end.p, w.end.len);
@@ -1420,7 +1488,8 @@ void addWriteRange(Node *&root, int64_t oldestVersion,
                                 std::min(begin.size(), end.size()));
   if (lcp == int(begin.size()) && end.size() == begin.size() + 1 &&
       end.back() == 0) {
-    return addPointWrite(root, oldestVersion, begin, w.writeVersion);
+    return addPointWrite(root, oldestVersion, begin, w.writeVersion,
+                         allocators);
   }
   auto remaining = begin.subspan(0, lcp);
 
@@ -1461,7 +1530,7 @@ void addWriteRange(Node *&root, int64_t oldestVersion,
   begin = begin.subspan(consumed, begin.size() - consumed);
   end = end.subspan(consumed, end.size() - consumed);
 
-  auto *beginNode = insert(useAsRoot, begin, w.writeVersion, true);
+  auto *beginNode = insert(useAsRoot, begin, w.writeVersion, true, allocators);
 
   const bool insertedBegin = !std::exchange(beginNode->entryPresent, true);
 
@@ -1476,7 +1545,7 @@ void addWriteRange(Node *&root, int64_t oldestVersion,
   beginNode->entry.pointVersion =
       std::max(beginNode->entry.pointVersion, w.writeVersion);
 
-  auto *endNode = insert(useAsRoot, end, w.writeVersion, false);
+  auto *endNode = insert(useAsRoot, end, w.writeVersion, false, allocators);
 
   const bool insertedEnd = !std::exchange(endNode->entryPresent, true);
 
@@ -1491,7 +1560,7 @@ void addWriteRange(Node *&root, int64_t oldestVersion,
 
   if (insertedEnd) {
     // beginNode may have been invalidated
-    beginNode = insert(useAsRoot, begin, w.writeVersion, true);
+    beginNode = insert(useAsRoot, begin, w.writeVersion, true, allocators);
   }
 
   for (beginNode = nextLogical(beginNode); beginNode != endNode;) {
@@ -1499,7 +1568,7 @@ void addWriteRange(Node *&root, int64_t oldestVersion,
     beginNode = nextLogical(beginNode);
     old->entryPresent = false;
     if (old->numChildren == 0 && old->parent != nullptr) {
-      eraseChild(old->parent, old->parentsIndex);
+      eraseChild(old->parent, old->parentsIndex, allocators);
     }
   }
 }
@@ -1639,12 +1708,12 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
       const auto &w = writes[i];
       if (w.end.len > 0) {
         keyUpdates += 2;
-        addWriteRange(root, oldestVersion, w);
+        addWriteRange(root, oldestVersion, w, &allocators);
       } else {
         keyUpdates += 1;
         addPointWrite(root, oldestVersion,
                       std::span<const uint8_t>(w.begin.p, w.begin.len),
-                      w.writeVersion);
+                      w.writeVersion, &allocators);
       }
     }
   }
@@ -1671,7 +1740,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
         assert(n->entry.rangeVersion <= oldestVersion);
         prev->entryPresent = false;
         if (prev->numChildren == 0 && prev->parent != nullptr) {
-          eraseChild(prev->parent, prev->parentsIndex);
+          eraseChild(prev->parent, prev->parentsIndex, &allocators);
         }
       }
 
@@ -1683,7 +1752,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
 
   explicit Impl(int64_t oldestVersion) : oldestVersion(oldestVersion) {
     // Insert ""
-    root = newNode();
+    root = allocators.node4.allocate();
     root->maxVersion = oldestVersion;
     root->entry.pointVersion = oldestVersion;
     root->entry.rangeVersion = oldestVersion;
@@ -1691,6 +1760,8 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
   }
   ~Impl() { destroyTree(root); }
 
+  NodeAllocators allocators;
+
   Arena removalKeyArena;
   std::span<const uint8_t> removalKey;
   int64_t keyUpdates = 0;