diff --git a/ConflictSet.cpp b/ConflictSet.cpp index 25cd774..2f8776e 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -210,7 +210,7 @@ enum Type : int8_t { Type_Node256, }; -template struct BoundedFreeListAllocator; +template struct NodeAllocator; struct TaggedNodePointer { TaggedNodePointer() = default; @@ -297,9 +297,9 @@ struct Node { } private: - template friend struct BoundedFreeListAllocator; + template friend struct NodeAllocator; // These are publically readable, but should only be written by - // BoundedFreeListAllocator + // NodeAllocator Type type; int32_t partialKeyCapacity; }; @@ -644,7 +644,7 @@ constexpr int kMinNodeSurplus = 104; constexpr int kBytesPerKey = 112; constexpr int kMinNodeSurplus = 80; #endif -// Cound the entry itself as a child +// Count the entry itself as a child constexpr int kMinChildrenNode0 = 1; constexpr int kMinChildrenNode3 = 2; constexpr int kMinChildrenNode16 = 4; @@ -669,48 +669,22 @@ static_assert(kNode3Surplus >= kMinNodeSurplus); static_assert(kBytesPerKey - sizeof(Node0) >= kMinNodeSurplus); -// setOldestVersion will additionally try to maintain this property: +// We'll additionally maintain this property: // `(children + entryPresent) * length >= capacity` // // Which should give us the budget to pay for the key bytes. (children + // entryPresent) is a lower bound on how many keys these bytes are a prefix of -constexpr int64_t kFreeListMaxMemory = 1 << 20; - -template struct BoundedFreeListAllocator { +// For now it's pretty much just a wrapper around malloc/free with some +// application-specific initialization. Maintaining a free list doesn't work +// that well since partial capacities mean the nodes have different sizes. If we +// come up with something better later we can implement it here. +template struct NodeAllocator { static_assert(sizeof(T) >= sizeof(void *)); static_assert(std::derived_from); static_assert(std::is_trivial_v); - T *allocate_helper(int partialKeyCapacity) { - if (freeList != nullptr) { - T *n = (T *)freeList; - VALGRIND_MAKE_MEM_DEFINED(freeList, sizeof(freeList)); - memcpy(&freeList, freeList, sizeof(freeList)); - VALGRIND_MAKE_MEM_UNDEFINED(n, sizeof(T)); - VALGRIND_MAKE_MEM_DEFINED(&n->partialKeyCapacity, - sizeof(n->partialKeyCapacity)); - VALGRIND_MAKE_MEM_DEFINED(&n->type, sizeof(n->type)); - assert(n->type == T::kType); - VALGRIND_MAKE_MEM_UNDEFINED(n + 1, n->partialKeyCapacity); - freeListBytes -= sizeof(T) + n->partialKeyCapacity; - if (n->partialKeyCapacity >= partialKeyCapacity) { - return n; - } else { - // The intent is to filter out too-small nodes in the freelist - removeNode(n); - safe_free(n, sizeof(T) + n->partialKeyCapacity); - } - } - - auto *result = (T *)safe_malloc(sizeof(T) + partialKeyCapacity); - result->type = T::kType; - result->partialKeyCapacity = partialKeyCapacity; - addNode(result); - return result; - } - T *allocate(int partialKeyCapacity) { T *result = allocate_helper(partialKeyCapacity); result->endOfRange = false; @@ -732,37 +706,27 @@ template struct BoundedFreeListAllocator { } void release(T *p) { - if (freeListBytes >= kFreeListMaxMemory) { - removeNode(p); - return safe_free(p, sizeof(T) + p->partialKeyCapacity); - } - memcpy((void *)p, &freeList, sizeof(freeList)); - freeList = p; - freeListBytes += sizeof(T) + p->partialKeyCapacity; - VALGRIND_MAKE_MEM_NOACCESS(freeList, sizeof(T) + p->partialKeyCapacity); + removeNode(p); + return safe_free(p, sizeof(T) + p->partialKeyCapacity); } - BoundedFreeListAllocator() = default; + NodeAllocator() = default; - BoundedFreeListAllocator(const BoundedFreeListAllocator &) = delete; - BoundedFreeListAllocator & - operator=(const BoundedFreeListAllocator &) = delete; - BoundedFreeListAllocator(BoundedFreeListAllocator &&) = delete; - BoundedFreeListAllocator &operator=(BoundedFreeListAllocator &&) = delete; + NodeAllocator(const NodeAllocator &) = delete; + NodeAllocator &operator=(const NodeAllocator &) = delete; + NodeAllocator(NodeAllocator &&) = delete; + NodeAllocator &operator=(NodeAllocator &&) = delete; - ~BoundedFreeListAllocator() { - for (void *iter = freeList; iter != nullptr;) { - VALGRIND_MAKE_MEM_DEFINED(iter, sizeof(Node)); - auto *tmp = (T *)iter; - memcpy(&iter, iter, sizeof(void *)); - removeNode((tmp)); - safe_free(tmp, sizeof(T) + tmp->partialKeyCapacity); - } - } + ~NodeAllocator() {} private: - int64_t freeListBytes = 0; - void *freeList = nullptr; + T *allocate_helper(int partialKeyCapacity) { + auto *result = (T *)safe_malloc(sizeof(T) + partialKeyCapacity); + result->type = T::kType; + result->partialKeyCapacity = partialKeyCapacity; + addNode(result); + return result; + } }; uint8_t *Node::partialKey() { @@ -896,11 +860,11 @@ struct WriteContext { private: Node *deferredList = nullptr; - BoundedFreeListAllocator node0; - BoundedFreeListAllocator node3; - BoundedFreeListAllocator node16; - BoundedFreeListAllocator node48; - BoundedFreeListAllocator node256; + NodeAllocator node0; + NodeAllocator node3; + NodeAllocator node16; + NodeAllocator node48; + NodeAllocator node256; }; int getNodeIndex(Node3 *self, uint8_t index) { @@ -1177,7 +1141,8 @@ void setMaxVersion(Node *n, InternalVersionT newMax) { } } -TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *); +// If impl is nullptr, then n->parent must not be nullptr +TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *impl); TaggedNodePointer getChild(Node0 *, uint8_t) { return nullptr; } TaggedNodePointer getChild(Node3 *self, uint8_t index) { @@ -1430,6 +1395,9 @@ TaggedNodePointer getFirstChildExists(Node *self) { // GCOVR_EXCL_STOP } +// self must not be the root +void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext); + void consumePartialKeyFull(TaggedNodePointer &self, TrivialSpan &key, InternalVersionT writeVersion, WriteContext *writeContext) { @@ -1466,9 +1434,8 @@ void consumePartialKeyFull(TaggedNodePointer &self, TrivialSpan &key, old->partialKeyLen - (partialKeyIndex + 1)); old->partialKeyLen -= partialKeyIndex + 1; - // We would consider decreasing capacity here, but we can't invalidate - // old since it's not on the search path. setOldestVersion will clean it - // up. + // Maintain memory capacity invariant + maybeDecreaseCapacity(old, writeContext); } key = key.subspan(partialKeyIndex, key.size() - partialKeyIndex); } @@ -1707,76 +1674,47 @@ downLeftSpine: return node; } -// Invalidates `self`, replacing it with a node of at least capacity. -// Does not return nodes to freelists when kUseFreeList is false. void freeAndMakeCapacityAtLeast(Node *&self, int capacity, - WriteContext *writeContext, - ConflictSet::Impl *impl, - const bool kUseFreeList) { + WriteContext *writeContext) { switch (self->getType()) { case Type_Node0: { auto *self0 = (Node0 *)self; auto *newSelf = writeContext->allocate(capacity); newSelf->copyChildrenAndKeyFrom(*self0); - getInTree(self, impl) = newSelf; - if (kUseFreeList) { - writeContext->deferRelease(self0, newSelf); - } else { - removeNode(self0); - safe_free(self0, self0->size()); - } + getInTree(self, nullptr) = newSelf; + writeContext->deferRelease(self0, newSelf); self = newSelf; } break; case Type_Node3: { auto *self3 = (Node3 *)self; auto *newSelf = writeContext->allocate(capacity); newSelf->copyChildrenAndKeyFrom(*self3); - getInTree(self, impl) = newSelf; - if (kUseFreeList) { - writeContext->deferRelease(self3, newSelf); - } else { - removeNode(self3); - safe_free(self3, self3->size()); - } + getInTree(self, nullptr) = newSelf; + writeContext->deferRelease(self3, newSelf); self = newSelf; } break; case Type_Node16: { auto *self16 = (Node16 *)self; auto *newSelf = writeContext->allocate(capacity); newSelf->copyChildrenAndKeyFrom(*self16); - getInTree(self, impl) = newSelf; - if (kUseFreeList) { - writeContext->deferRelease(self16, newSelf); - } else { - removeNode(self16); - safe_free(self16, self16->size()); - } + getInTree(self, nullptr) = newSelf; + writeContext->deferRelease(self16, newSelf); self = newSelf; } break; case Type_Node48: { auto *self48 = (Node48 *)self; auto *newSelf = writeContext->allocate(capacity); newSelf->copyChildrenAndKeyFrom(*self48); - getInTree(self, impl) = newSelf; - if (kUseFreeList) { - writeContext->deferRelease(self48, newSelf); - } else { - removeNode(self48); - safe_free(self48, self48->size()); - } + getInTree(self, nullptr) = newSelf; + writeContext->deferRelease(self48, newSelf); self = newSelf; } break; case Type_Node256: { auto *self256 = (Node256 *)self; auto *newSelf = writeContext->allocate(capacity); newSelf->copyChildrenAndKeyFrom(*self256); - getInTree(self, impl) = newSelf; - if (kUseFreeList) { - writeContext->deferRelease(self256, newSelf); - } else { - removeNode(self256); - safe_free(self256, self256->size()); - } + getInTree(self, nullptr) = newSelf; + writeContext->deferRelease(self256, newSelf); self = newSelf; } break; default: // GCOVR_EXCL_LINE @@ -1784,11 +1722,8 @@ void freeAndMakeCapacityAtLeast(Node *&self, int capacity, } } -// Fix larger-than-desired capacities. Does not return nodes to freelists, -// since that wouldn't actually reclaim the memory used for partial key -// capacity. -void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext, - ConflictSet::Impl *impl) { +// Fix larger-than-desired capacities. self must not be the root +void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext) { const int maxCapacity = (self->numChildren + int(self->entryPresent)) * (self->partialKeyLen + 1); @@ -1800,7 +1735,7 @@ void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext, if (self->getCapacity() <= maxCapacity) { return; } - freeAndMakeCapacityAtLeast(self, maxCapacity, writeContext, impl, false); + freeAndMakeCapacityAtLeast(self, maxCapacity, writeContext); } #if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__) @@ -1870,13 +1805,13 @@ void rezero(Node *n, InternalVersionT z) { #endif void mergeWithChild(TaggedNodePointer &self, WriteContext *writeContext, - ConflictSet::Impl *impl, Node3 *self3) { + Node3 *self3) { assert(!self3->entryPresent); Node *child = self3->children[0]; int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen; if (minCapacity > child->getCapacity()) { - freeAndMakeCapacityAtLeast(child, minCapacity, writeContext, impl, true); + freeAndMakeCapacityAtLeast(child, minCapacity, writeContext); } // Merge partial key with child @@ -1921,7 +1856,7 @@ void downsize(Node3 *self, WriteContext *writeContext, writeContext->deferRelease(self, newSelf); } else { assert(self->numChildren == 1 && !self->entryPresent); - mergeWithChild(getInTree(self, impl), writeContext, impl, self); + mergeWithChild(getInTree(self, impl), writeContext, self); } } @@ -2001,6 +1936,10 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl, if (needsDownsize(self)) { downsize(self, writeContext, impl); } + while (self->releaseDeferred) { + self = self->forwardTo; + } + maybeDecreaseCapacity(self, writeContext); if (result != nullptr) { while (result->releaseDeferred) { result = result->forwardTo; @@ -2088,6 +2027,11 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl, __builtin_unreachable(); // GCOVR_EXCL_LINE } + while (parent->releaseDeferred) { + parent = parent->forwardTo; + } + maybeDecreaseCapacity(parent, writeContext); + if (result != nullptr) { while (result->releaseDeferred) { result = result->forwardTo; @@ -5271,7 +5215,6 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { assert(n->entry.rangeVersion <= oldestVersion); n = erase(n, &writeContext, this, /*logical*/ false); } else { - maybeDecreaseCapacity(n, &writeContext, this); n = nextPhysical(n); } } @@ -5923,7 +5866,16 @@ checkMaxVersion(Node *root, Node *node, InternalVersionT oldestVersion, int(node->entryPresent), minNumChildren); success = false; } - // TODO check that the max capacity property eventually holds + + const int maxCapacity = + (node->numChildren + int(node->entryPresent)) * (node->partialKeyLen + 1); + if (node->getCapacity() > maxCapacity) { + fprintf(stderr, "%s has d capacity %d, which is more than the allowed %d\n", + getSearchPathPrintable(node).c_str(), node->getCapacity(), + maxCapacity); + success = false; + } + for (auto child = getChildGeq(node, 0); child != nullptr; child = getChildGeq(node, child->parentsIndex + 1)) { checkMemoryBoundInvariants(child, success);