diff --git a/ConflictSet.cpp b/ConflictSet.cpp index 02fbe6d..ffe9971 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -43,6 +43,46 @@ struct Entry { int64_t rangeVersion; }; +template +struct BoundedFreeListAllocator { + static_assert(sizeof(T) >= sizeof(void *)); + + T *allocate() { + if (freeListSize == 0) { + assert(freeList == nullptr); + return new (safe_malloc(sizeof(T))) T; + } + assert(freeList != nullptr); + void *result = freeList; + memcpy(&freeList, freeList, sizeof(freeList)); + --freeListSize; + return new (result) T; + } + + void release(T *p) { + p->~T(); + if (freeListSize == kMaxFreeListSize) { + return free(p); + } + memcpy(p, &freeList, sizeof(freeList)); + freeList = p; + ++freeListSize; + } + + ~BoundedFreeListAllocator() { + for (void *iter = freeList; iter != nullptr;) { + auto *tmp = iter; + memcpy(&iter, iter, sizeof(void *)); + free(tmp); + } + } + +private: + static constexpr int kMaxFreeListSize = kMemoryBound / sizeof(T); + int freeListSize = 0; + void *freeList = nullptr; +}; + enum class Type : int8_t { Node4, Node16, @@ -75,8 +115,6 @@ struct Node4 : Node { Node4() { this->type = Type::Node4; } }; -Node *newNode() { return new (safe_malloc(sizeof(Node4))) Node4; } - struct Node16 : Node { // Sorted uint8_t index[16] = {}; @@ -158,6 +196,13 @@ struct Node256 : Node { Node256() { this->type = Type::Node256; } }; +struct NodeAllocators { + BoundedFreeListAllocator node4; + BoundedFreeListAllocator node16; + BoundedFreeListAllocator node48; + BoundedFreeListAllocator node256; +}; + int getNodeIndex(Node4 *self, uint8_t index) { for (int i = 0; i < self->numChildren; ++i) { if (self->index[i] == index) { @@ -330,7 +375,8 @@ void setChildrenParents(Node *node) { // Caller is responsible for assigning a non-null pointer to the returned // reference if null -Node *&getOrCreateChild(Node *&self, uint8_t index) { +Node *&getOrCreateChild(Node *&self, uint8_t index, + NodeAllocators *allocators) { if (self->type == Type::Node4) { auto *self4 = static_cast(self); { @@ -340,11 +386,12 @@ Node *&getOrCreateChild(Node *&self, uint8_t index) { } } if (self->numChildren == 4) { - auto *newSelf = new (safe_malloc(sizeof(Node16))) Node16; + auto *newSelf = allocators->node16.allocate(); memcpy((void *)newSelf, self, offsetof(Node, type)); memcpy(newSelf->index, self4->index, 4); memcpy(newSelf->children, self4->children, 4 * sizeof(void *)); - free(std::exchange(self, newSelf)); + allocators->node4.release(self4); + self = newSelf; setChildrenParents(self); goto insert16; } else { @@ -374,7 +421,7 @@ Node *&getOrCreateChild(Node *&self, uint8_t index) { } } if (self->numChildren == 16) { - auto *newSelf = new (safe_malloc(sizeof(Node48))) Node48; + auto *newSelf = allocators->node48.allocate(); memcpy((void *)newSelf, self, offsetof(Node, type)); newSelf->nextFree = 16; int i = 0; @@ -385,7 +432,8 @@ Node *&getOrCreateChild(Node *&self, uint8_t index) { ++i; } assert(i == 16); - free(std::exchange(self, newSelf)); + allocators->node16.release(self16); + self = newSelf; setChildrenParents(self); goto insert48; } else { @@ -412,7 +460,7 @@ Node *&getOrCreateChild(Node *&self, uint8_t index) { return self48->children[self48->index[index]]; } if (self->numChildren == 48) { - auto *newSelf = new (safe_malloc(sizeof(Node256))) Node256; + auto *newSelf = allocators->node256.allocate(); memcpy((void *)newSelf, self, offsetof(Node, type)); for (int i = 0; i < 256; ++i) { if (self48->bitSet.test(i)) { @@ -420,7 +468,7 @@ Node *&getOrCreateChild(Node *&self, uint8_t index) { newSelf->children[i] = self48->children[self48->index[i]]; } } - free(std::exchange(self, newSelf)); + allocators->node48.release(self48); self = newSelf; setChildrenParents(self); goto insert256; @@ -444,8 +492,24 @@ Node *&getOrCreateChild(Node *&self, uint8_t index) { } // Precondition - an entry for index must exist in the node -void eraseChild(Node *self, uint8_t index) { - free(getChildExists(self, index)); +void eraseChild(Node *self, uint8_t index, NodeAllocators *allocators) { + auto *child = getChildExists(self, index); + switch (child->type) { + case Type::Node4: + allocators->node4.release((Node4 *)child); + break; + case Type::Node16: + allocators->node16.release((Node16 *)child); + break; + case Type::Node48: + allocators->node48.release((Node48 *)child); + break; + case Type::Node256: + allocators->node256.release((Node256 *)child); + break; + case Type::Invalid: + __builtin_unreachable(); // GCOVR_EXCL_LINE + } if (self->type == Type::Node4) { auto *self4 = static_cast(self); @@ -484,7 +548,7 @@ void eraseChild(Node *self, uint8_t index) { --self->numChildren; if (self->numChildren == 0 && !self->entryPresent && self->parent != nullptr) { - eraseChild(self->parent, self->parentsIndex); + eraseChild(self->parent, self->parentsIndex, allocators); } } @@ -1315,7 +1379,8 @@ bool checkRangeRead(Node *n, std::span begin, // !entryPresent. The search path of the result's parent will have // `maxVersion` at least `writeVersion` as a postcondition. [[nodiscard]] Node *insert(Node **self_, std::span key, - int64_t writeVersion, bool begin) { + int64_t writeVersion, bool begin, + NodeAllocators *allocators) { for (;;) { auto &self = *self_; // Handle an existing partial key @@ -1324,14 +1389,15 @@ bool checkRangeRead(Node *n, std::span begin, if (partialKeyIndex == int(key.size()) || self->partialKey[partialKeyIndex] != key[partialKeyIndex]) { auto *old = self; - self = newNode(); + self = allocators->node4.allocate(); self->maxVersion = old->maxVersion; self->partialKeyLen = partialKeyIndex; self->parent = old->parent; self->parentsIndex = old->parentsIndex; memcpy(self->partialKey, old->partialKey, partialKeyIndex); - getOrCreateChild(self, old->partialKey[partialKeyIndex]) = old; + getOrCreateChild(self, old->partialKey[partialKeyIndex], allocators) = + old; old->parent = self; old->parentsIndex = old->partialKey[partialKeyIndex]; @@ -1362,9 +1428,9 @@ bool checkRangeRead(Node *n, std::span begin, self->maxVersion = std::max(self->maxVersion, writeVersion); } - auto &child = getOrCreateChild(self, key.front()); + auto &child = getOrCreateChild(self, key.front(), allocators); if (!child) { - child = newNode(); + child = allocators->node4.allocate(); child->parent = self; child->parentsIndex = key.front(); child->maxVersion = @@ -1395,8 +1461,9 @@ void destroyTree(Node *root) { } void addPointWrite(Node *&root, int64_t oldestVersion, - std::span key, int64_t writeVersion) { - auto *n = insert(&root, key, writeVersion, true); + std::span key, int64_t writeVersion, + NodeAllocators *allocators) { + auto *n = insert(&root, key, writeVersion, true, allocators); if (!n->entryPresent) { auto *p = nextLogical(n); n->entryPresent = true; @@ -1411,7 +1478,8 @@ void addPointWrite(Node *&root, int64_t oldestVersion, } void addWriteRange(Node *&root, int64_t oldestVersion, - const ConflictSet::WriteRange &w) { + const ConflictSet::WriteRange &w, + NodeAllocators *allocators) { auto begin = std::span(w.begin.p, w.begin.len); auto end = std::span(w.end.p, w.end.len); @@ -1420,7 +1488,8 @@ void addWriteRange(Node *&root, int64_t oldestVersion, std::min(begin.size(), end.size())); if (lcp == int(begin.size()) && end.size() == begin.size() + 1 && end.back() == 0) { - return addPointWrite(root, oldestVersion, begin, w.writeVersion); + return addPointWrite(root, oldestVersion, begin, w.writeVersion, + allocators); } auto remaining = begin.subspan(0, lcp); @@ -1461,7 +1530,7 @@ void addWriteRange(Node *&root, int64_t oldestVersion, begin = begin.subspan(consumed, begin.size() - consumed); end = end.subspan(consumed, end.size() - consumed); - auto *beginNode = insert(useAsRoot, begin, w.writeVersion, true); + auto *beginNode = insert(useAsRoot, begin, w.writeVersion, true, allocators); const bool insertedBegin = !std::exchange(beginNode->entryPresent, true); @@ -1476,7 +1545,7 @@ void addWriteRange(Node *&root, int64_t oldestVersion, beginNode->entry.pointVersion = std::max(beginNode->entry.pointVersion, w.writeVersion); - auto *endNode = insert(useAsRoot, end, w.writeVersion, false); + auto *endNode = insert(useAsRoot, end, w.writeVersion, false, allocators); const bool insertedEnd = !std::exchange(endNode->entryPresent, true); @@ -1491,7 +1560,7 @@ void addWriteRange(Node *&root, int64_t oldestVersion, if (insertedEnd) { // beginNode may have been invalidated - beginNode = insert(useAsRoot, begin, w.writeVersion, true); + beginNode = insert(useAsRoot, begin, w.writeVersion, true, allocators); } for (beginNode = nextLogical(beginNode); beginNode != endNode;) { @@ -1499,7 +1568,7 @@ void addWriteRange(Node *&root, int64_t oldestVersion, beginNode = nextLogical(beginNode); old->entryPresent = false; if (old->numChildren == 0 && old->parent != nullptr) { - eraseChild(old->parent, old->parentsIndex); + eraseChild(old->parent, old->parentsIndex, allocators); } } } @@ -1639,12 +1708,12 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { const auto &w = writes[i]; if (w.end.len > 0) { keyUpdates += 2; - addWriteRange(root, oldestVersion, w); + addWriteRange(root, oldestVersion, w, &allocators); } else { keyUpdates += 1; addPointWrite(root, oldestVersion, std::span(w.begin.p, w.begin.len), - w.writeVersion); + w.writeVersion, &allocators); } } } @@ -1671,7 +1740,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { assert(n->entry.rangeVersion <= oldestVersion); prev->entryPresent = false; if (prev->numChildren == 0 && prev->parent != nullptr) { - eraseChild(prev->parent, prev->parentsIndex); + eraseChild(prev->parent, prev->parentsIndex, &allocators); } } @@ -1683,7 +1752,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { explicit Impl(int64_t oldestVersion) : oldestVersion(oldestVersion) { // Insert "" - root = newNode(); + root = allocators.node4.allocate(); root->maxVersion = oldestVersion; root->entry.pointVersion = oldestVersion; root->entry.rangeVersion = oldestVersion; @@ -1691,6 +1760,8 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { } ~Impl() { destroyTree(root); } + NodeAllocators allocators; + Arena removalKeyArena; std::span removalKey; int64_t keyUpdates = 0;