diff --git a/ConflictSet.cpp b/ConflictSet.cpp index 8809fc4..57ae817 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -680,18 +680,34 @@ static_assert(kBytesPerKey - sizeof(Node0) >= kMinNodeSurplus); // Which should give us the budget to pay for the key bytes. (children + // entryPresent) is a lower bound on how many keys these bytes are a prefix of -// For now it's pretty much just a wrapper around malloc/free with some -// application-specific initialization. Maintaining a free list doesn't work -// that well since partial capacities mean the nodes have different sizes. If we -// come up with something better later we can implement it here. +constexpr int getMaxCapacity(int numChildren, int entryPresent, + int partialKeyLen) { + return (numChildren + entryPresent) * (partialKeyLen + 1); +} + +constexpr int getMaxCapacity(Node *self) { + return getMaxCapacity(self->numChildren, self->entryPresent, + self->partialKeyLen); +} + +constexpr int64_t kMaxFreeListBytes = 1 << 20; + +// Maintains a free list up to kMaxFreeListBytes. If the top element of the list +// doesn't meet the capacity constraints, it's freed and a new node is allocated +// with the minimum capacity. The hope is that "unfit" nodes don't get stuck in +// the free list. +// +// TODO valgrind annotations template struct NodeAllocator { - static_assert(sizeof(T) >= sizeof(void *)); static_assert(std::derived_from); static_assert(std::is_trivial_v); - T *allocate(int partialKeyCapacity) { - T *result = allocate_helper(partialKeyCapacity); + T *allocate(int minCapacity, int maxCapacity) { + assert(minCapacity <= maxCapacity); + assert(freeListSize >= 0); + assert(freeListSize <= kMaxFreeListBytes); + T *result = allocate_helper(minCapacity, maxCapacity); result->endOfRange = false; result->releaseDeferred = false; if constexpr (!std::is_same_v) { @@ -711,8 +727,41 @@ template struct NodeAllocator { } void release(T *p) { - removeNode(p); - return safe_free(p, sizeof(T) + p->partialKeyCapacity); + if (freeListSize + sizeof(T) + p->partialKeyCapacity > kMaxFreeListBytes) { + removeNode(p); + return safe_free(p, sizeof(T) + p->partialKeyCapacity); + } + p->parent = freeList; + freeList = p; + freeListSize += sizeof(T) + p->partialKeyCapacity; + } + + void deferRelease(T *p, Node *forwardTo) { + p->releaseDeferred = true; + p->forwardTo = forwardTo; + if (freeListSize + sizeof(T) + p->partialKeyCapacity > kMaxFreeListBytes) { + p->parent = deferredListOverflow; + deferredListOverflow = p; + } else { + if (deferredList == nullptr) { + deferredListFront = p; + } + p->parent = deferredList; + deferredList = p; + freeListSize += sizeof(T) + p->partialKeyCapacity; + } + } + + void releaseDeferred() { + if (deferredList != nullptr) { + deferredListFront->parent = freeList; + freeList = std::exchange(deferredList, nullptr); + } + for (T *n = std::exchange(deferredListOverflow, nullptr); n != nullptr;) { + auto *tmp = n; + n = (T *)n->parent; + release(tmp); + } } NodeAllocator() = default; @@ -722,13 +771,46 @@ template struct NodeAllocator { NodeAllocator(NodeAllocator &&) = delete; NodeAllocator &operator=(NodeAllocator &&) = delete; - ~NodeAllocator() {} + ~NodeAllocator() { + assert(deferredList == nullptr); + assert(deferredListOverflow == nullptr); + for (T *iter = freeList; iter != nullptr;) { + auto *tmp = iter; + iter = (T *)iter->parent; + removeNode(tmp); + safe_free(tmp, sizeof(T) + tmp->partialKeyCapacity); + } + } private: - T *allocate_helper(int partialKeyCapacity) { - auto *result = (T *)safe_malloc(sizeof(T) + partialKeyCapacity); + int64_t freeListSize = 0; + T *freeList = nullptr; + T *deferredList = nullptr; + // Used to concatenate deferredList to freeList + T *deferredListFront; + T *deferredListOverflow = nullptr; + + T *allocate_helper(int minCapacity, int maxCapacity) { + if (freeList != nullptr) { + freeListSize -= sizeof(T) + freeList->partialKeyCapacity; + assume(freeList->partialKeyCapacity >= 0); + assume(minCapacity >= 0); + assume(minCapacity <= maxCapacity); + if (freeList->partialKeyCapacity >= minCapacity && + freeList->partialKeyCapacity <= maxCapacity) { + auto *result = freeList; + freeList = (T *)freeList->parent; + return result; + } else { + auto *p = freeList; + freeList = (T *)p->parent; + removeNode(p); + safe_free(p, sizeof(T) + p->partialKeyCapacity); + } + } + auto *result = (T *)safe_malloc(sizeof(T) + minCapacity); result->type = T::kType; - result->partialKeyCapacity = partialKeyCapacity; + result->partialKeyCapacity = minCapacity; addNode(result); return result; } @@ -796,18 +878,19 @@ struct WriteContext { WriteContext() { memset(&accum, 0, sizeof(accum)); } - template T *allocate(int c) { + template T *allocate(int minCapacity, int maxCapacity) { + static_assert(!std::is_same_v); ++accum.nodes_allocated; if constexpr (std::is_same_v) { - return node0.allocate(c); + return node0.allocate(minCapacity, maxCapacity); } else if constexpr (std::is_same_v) { - return node3.allocate(c); + return node3.allocate(minCapacity, maxCapacity); } else if constexpr (std::is_same_v) { - return node16.allocate(c); + return node16.allocate(minCapacity, maxCapacity); } else if constexpr (std::is_same_v) { - return node48.allocate(c); + return node48.allocate(minCapacity, maxCapacity); } else if constexpr (std::is_same_v) { - return node256.allocate(c); + return node256.allocate(minCapacity, maxCapacity); } } template void release(T *c) { @@ -827,44 +910,32 @@ struct WriteContext { } // Place in a list to be released in the next call to releaseDeferred. - void deferRelease(Node *n, Node *forwardTo) { - n->releaseDeferred = true; - n->forwardTo = forwardTo; - n->parent = deferredList; - deferredList = n; + template void deferRelease(T *n, Node *forwardTo) { + static_assert(!std::is_same_v); + if constexpr (std::is_same_v) { + return node0.deferRelease(n, forwardTo); + } else if constexpr (std::is_same_v) { + return node3.deferRelease(n, forwardTo); + } else if constexpr (std::is_same_v) { + return node16.deferRelease(n, forwardTo); + } else if constexpr (std::is_same_v) { + return node48.deferRelease(n, forwardTo); + } else if constexpr (std::is_same_v) { + return node256.deferRelease(n, forwardTo); + } } // Release all nodes passed to deferRelease since the last call to // releaseDeferred. void releaseDeferred() { - for (Node *n = std::exchange(deferredList, nullptr); n != nullptr;) { - auto *tmp = n; - n = n->parent; - switch (tmp->getType()) { - case Type_Node0: - release(static_cast(tmp)); - break; - case Type_Node3: - release(static_cast(tmp)); - break; - case Type_Node16: - release(static_cast(tmp)); - break; - case Type_Node48: - release(static_cast(tmp)); - break; - case Type_Node256: - release(static_cast(tmp)); - break; - default: // GCOVR_EXCL_LINE - __builtin_unreachable(); // GCOVR_EXCL_LINE - } - } + node0.releaseDeferred(); + node3.releaseDeferred(); + node16.releaseDeferred(); + node48.releaseDeferred(); + node256.releaseDeferred(); } private: - Node *deferredList = nullptr; - NodeAllocator node0; NodeAllocator node3; NodeAllocator node16; @@ -1401,11 +1472,13 @@ TaggedNodePointer getFirstChildExists(Node *self) { } // self must not be the root -void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext); +void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext, + ConflictSet::Impl *impl); void consumePartialKeyFull(TaggedNodePointer &self, TrivialSpan &key, InternalVersionT writeVersion, - WriteContext *writeContext) { + WriteContext *writeContext, + ConflictSet::Impl *impl) { // Handle an existing partial key int commonLen = std::min(self->partialKeyLen, key.size()); int partialKeyIndex = @@ -1417,7 +1490,8 @@ void consumePartialKeyFull(TaggedNodePointer &self, TrivialSpan &key, InternalVersionT oldMaxVersion = exchangeMaxVersion(old, writeVersion); // *self will have one child (old) - auto *newSelf = writeContext->allocate(partialKeyIndex); + auto *newSelf = writeContext->allocate( + partialKeyIndex, getMaxCapacity(1, 0, partialKeyIndex)); newSelf->parent = old->parent; newSelf->parentsIndex = old->parentsIndex; @@ -1440,7 +1514,7 @@ void consumePartialKeyFull(TaggedNodePointer &self, TrivialSpan &key, old->partialKeyLen -= partialKeyIndex + 1; // Maintain memory capacity invariant - maybeDecreaseCapacity(old, writeContext); + maybeDecreaseCapacity(old, writeContext, impl); } key = key.subspan(partialKeyIndex, key.size() - partialKeyIndex); } @@ -1449,9 +1523,10 @@ void consumePartialKeyFull(TaggedNodePointer &self, TrivialSpan &key, // `key` such that `self` is along the search path of `key` inline __attribute__((always_inline)) void consumePartialKey(TaggedNodePointer &self, TrivialSpan &key, - InternalVersionT writeVersion, WriteContext *writeContext) { + InternalVersionT writeVersion, WriteContext *writeContext, + ConflictSet::Impl *impl) { if (self->partialKeyLen > 0) { - consumePartialKeyFull(self, key, writeVersion, writeContext); + consumePartialKeyFull(self, key, writeVersion, writeContext, impl); } } @@ -1461,7 +1536,8 @@ consumePartialKey(TaggedNodePointer &self, TrivialSpan &key, // `maxVersion` for result. TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key, InternalVersionT newMaxVersion, - WriteContext *writeContext) { + WriteContext *writeContext, + ConflictSet::Impl *impl) { int index = key.front(); key = key.subspan(1, key.size() - 1); @@ -1474,7 +1550,8 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key, auto *self3 = static_cast(self); int i = getNodeIndex(self3, index); if (i >= 0) { - consumePartialKey(self3->children[i], key, newMaxVersion, writeContext); + consumePartialKey(self3->children[i], key, newMaxVersion, writeContext, + impl); self3->childMaxVersion[i] = newMaxVersion; return self3->children[i]; } @@ -1483,7 +1560,8 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key, auto *self16 = static_cast(self); int i = getNodeIndex(self16, index); if (i >= 0) { - consumePartialKey(self16->children[i], key, newMaxVersion, writeContext); + consumePartialKey(self16->children[i], key, newMaxVersion, writeContext, + impl); self16->childMaxVersion[i] = newMaxVersion; return self16->children[i]; } @@ -1493,7 +1571,7 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key, int secondIndex = self48->index[index]; if (secondIndex >= 0) { consumePartialKey(self48->children[secondIndex], key, newMaxVersion, - writeContext); + writeContext, impl); self48->childMaxVersion[secondIndex] = newMaxVersion; self48->maxOfMax[secondIndex >> Node48::kMaxOfMaxShift] = std::max(self48->maxOfMax[secondIndex >> Node48::kMaxOfMaxShift], @@ -1504,7 +1582,7 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key, case Type_Node256: { auto *self256 = static_cast(self); if (auto &result = self256->children[index]; result != nullptr) { - consumePartialKey(result, key, newMaxVersion, writeContext); + consumePartialKey(result, key, newMaxVersion, writeContext, impl); self256->childMaxVersion[index] = newMaxVersion; self256->maxOfMax[index >> Node256::kMaxOfMaxShift] = std::max( self256->maxOfMax[index >> Node256::kMaxOfMaxShift], newMaxVersion); @@ -1515,9 +1593,10 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key, __builtin_unreachable(); // GCOVR_EXCL_LINE } - auto *newChild = writeContext->allocate(key.size()); + auto *newChild = writeContext->allocate( + key.size(), getMaxCapacity(0, 1, key.size())); newChild->numChildren = 0; - newChild->entryPresent = false; + newChild->entryPresent = false; // Will be set to true by the caller newChild->partialKeyLen = key.size(); newChild->parentsIndex = index; memcpy(newChild->partialKey(), key.data(), key.size()); @@ -1527,7 +1606,8 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key, case Type_Node0: { auto *self0 = static_cast(self); - auto *newSelf = writeContext->allocate(self->partialKeyLen); + auto *newSelf = writeContext->allocate( + self->partialKeyLen, getMaxCapacity(1, 1, self->partialKeyLen)); newSelf->copyChildrenAndKeyFrom(*self0); writeContext->deferRelease(self0, newSelf); self = newSelf; @@ -1537,7 +1617,9 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key, case Type_Node3: { if (self->numChildren == Node3::kMaxNodes) { auto *self3 = static_cast(self); - auto *newSelf = writeContext->allocate(self->partialKeyLen); + auto *newSelf = writeContext->allocate( + self->partialKeyLen, + getMaxCapacity(4, self->entryPresent, self->partialKeyLen)); newSelf->copyChildrenAndKeyFrom(*self3); writeContext->deferRelease(self3, newSelf); self = newSelf; @@ -1566,7 +1648,9 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key, case Type_Node16: { if (self->numChildren == Node16::kMaxNodes) { auto *self16 = static_cast(self); - auto *newSelf = writeContext->allocate(self->partialKeyLen); + auto *newSelf = writeContext->allocate( + self->partialKeyLen, + getMaxCapacity(17, self->entryPresent, self->partialKeyLen)); newSelf->copyChildrenAndKeyFrom(*self16); writeContext->deferRelease(self16, newSelf); self = newSelf; @@ -1597,7 +1681,9 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key, if (self->numChildren == 48) { auto *self48 = static_cast(self); - auto *newSelf = writeContext->allocate(self->partialKeyLen); + auto *newSelf = writeContext->allocate( + self->partialKeyLen, + getMaxCapacity(49, self->entryPresent, self->partialKeyLen)); newSelf->copyChildrenAndKeyFrom(*self48); writeContext->deferRelease(self48, newSelf); self = newSelf; @@ -1679,46 +1765,47 @@ downLeftSpine: return node; } -void freeAndMakeCapacityAtLeast(Node *&self, int capacity, - WriteContext *writeContext) { +void freeAndMakeCapacityBetween(Node *&self, int minCapacity, int maxCapacity, + WriteContext *writeContext, + ConflictSet::Impl *impl) { switch (self->getType()) { case Type_Node0: { auto *self0 = (Node0 *)self; - auto *newSelf = writeContext->allocate(capacity); + auto *newSelf = writeContext->allocate(minCapacity, maxCapacity); newSelf->copyChildrenAndKeyFrom(*self0); - getInTree(self, nullptr) = newSelf; + getInTree(self, impl) = newSelf; writeContext->deferRelease(self0, newSelf); self = newSelf; } break; case Type_Node3: { auto *self3 = (Node3 *)self; - auto *newSelf = writeContext->allocate(capacity); + auto *newSelf = writeContext->allocate(minCapacity, maxCapacity); newSelf->copyChildrenAndKeyFrom(*self3); - getInTree(self, nullptr) = newSelf; + getInTree(self, impl) = newSelf; writeContext->deferRelease(self3, newSelf); self = newSelf; } break; case Type_Node16: { auto *self16 = (Node16 *)self; - auto *newSelf = writeContext->allocate(capacity); + auto *newSelf = writeContext->allocate(minCapacity, maxCapacity); newSelf->copyChildrenAndKeyFrom(*self16); - getInTree(self, nullptr) = newSelf; + getInTree(self, impl) = newSelf; writeContext->deferRelease(self16, newSelf); self = newSelf; } break; case Type_Node48: { auto *self48 = (Node48 *)self; - auto *newSelf = writeContext->allocate(capacity); + auto *newSelf = writeContext->allocate(minCapacity, maxCapacity); newSelf->copyChildrenAndKeyFrom(*self48); - getInTree(self, nullptr) = newSelf; + getInTree(self, impl) = newSelf; writeContext->deferRelease(self48, newSelf); self = newSelf; } break; case Type_Node256: { auto *self256 = (Node256 *)self; - auto *newSelf = writeContext->allocate(capacity); + auto *newSelf = writeContext->allocate(minCapacity, maxCapacity); newSelf->copyChildrenAndKeyFrom(*self256); - getInTree(self, nullptr) = newSelf; + getInTree(self, impl) = newSelf; writeContext->deferRelease(self256, newSelf); self = newSelf; } break; @@ -1728,7 +1815,8 @@ void freeAndMakeCapacityAtLeast(Node *&self, int capacity, } // Fix larger-than-desired capacities. self must not be the root -void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext) { +void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext, + ConflictSet::Impl *impl) { const int maxCapacity = (self->numChildren + int(self->entryPresent)) * (self->partialKeyLen + 1); @@ -1740,7 +1828,8 @@ void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext) { if (self->getCapacity() <= maxCapacity) { return; } - freeAndMakeCapacityAtLeast(self, maxCapacity, writeContext); + freeAndMakeCapacityBetween(self, self->partialKeyLen, maxCapacity, + writeContext, impl); } #if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__) @@ -1810,13 +1899,16 @@ void rezero(Node *n, InternalVersionT z) { #endif void mergeWithChild(TaggedNodePointer &self, WriteContext *writeContext, - Node3 *self3) { + Node3 *self3, ConflictSet::Impl *impl) { assert(!self3->entryPresent); Node *child = self3->children[0]; - int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen; + const int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen; + const int maxCapacity = + getMaxCapacity(child->numChildren, child->entryPresent, minCapacity); if (minCapacity > child->getCapacity()) { - freeAndMakeCapacityAtLeast(child, minCapacity, writeContext); + freeAndMakeCapacityBetween(child, minCapacity, maxCapacity, writeContext, + impl); } // Merge partial key with child @@ -1855,20 +1947,23 @@ bool needsDownsize(Node *n) { void downsize(Node3 *self, WriteContext *writeContext, ConflictSet::Impl *impl) { if (self->numChildren == 0) { - auto *newSelf = writeContext->allocate(self->partialKeyLen); + auto *newSelf = writeContext->allocate( + self->partialKeyLen, getMaxCapacity(0, 1, self->partialKeyLen)); newSelf->copyChildrenAndKeyFrom(*self); getInTree(self, impl) = newSelf; writeContext->deferRelease(self, newSelf); } else { assert(self->numChildren == 1 && !self->entryPresent); - mergeWithChild(getInTree(self, impl), writeContext, self); + mergeWithChild(getInTree(self, impl), writeContext, self, impl); } } void downsize(Node16 *self, WriteContext *writeContext, ConflictSet::Impl *impl) { assert(self->numChildren + int(self->entryPresent) < kMinChildrenNode16); - auto *newSelf = writeContext->allocate(self->partialKeyLen); + auto *newSelf = writeContext->allocate( + self->partialKeyLen, + getMaxCapacity(kMinChildrenNode16 - 1, 0, self->partialKeyLen)); newSelf->copyChildrenAndKeyFrom(*self); getInTree(self, impl) = newSelf; writeContext->deferRelease(self, newSelf); @@ -1877,7 +1972,9 @@ void downsize(Node16 *self, WriteContext *writeContext, void downsize(Node48 *self, WriteContext *writeContext, ConflictSet::Impl *impl) { assert(self->numChildren + int(self->entryPresent) < kMinChildrenNode48); - auto *newSelf = writeContext->allocate(self->partialKeyLen); + auto *newSelf = writeContext->allocate( + self->partialKeyLen, + getMaxCapacity(kMinChildrenNode48 - 1, 0, self->partialKeyLen)); newSelf->copyChildrenAndKeyFrom(*self); getInTree(self, impl) = newSelf; writeContext->deferRelease(self, newSelf); @@ -1887,7 +1984,9 @@ void downsize(Node256 *self, WriteContext *writeContext, ConflictSet::Impl *impl) { assert(self->numChildren + int(self->entryPresent) < kMinChildrenNode256); auto *self256 = (Node256 *)self; - auto *newSelf = writeContext->allocate(self->partialKeyLen); + auto *newSelf = writeContext->allocate( + self->partialKeyLen, + getMaxCapacity(kMinChildrenNode256 - 1, 0, self->partialKeyLen)); newSelf->copyChildrenAndKeyFrom(*self256); getInTree(self, impl) = newSelf; writeContext->deferRelease(self256, newSelf); @@ -1944,7 +2043,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl, while (self->releaseDeferred) { self = self->forwardTo; } - maybeDecreaseCapacity(self, writeContext); + maybeDecreaseCapacity(self, writeContext, impl); if (result != nullptr) { while (result->releaseDeferred) { result = result->forwardTo; @@ -2035,7 +2134,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl, while (parent->releaseDeferred) { parent = parent->forwardTo; } - maybeDecreaseCapacity(parent, writeContext); + maybeDecreaseCapacity(parent, writeContext, impl); if (result != nullptr) { while (result->releaseDeferred) { @@ -2740,13 +2839,12 @@ checkMaxBetweenExclusiveImpl(Node256 *n, int begin, int end, // of the result will have `maxVersion` set to `writeVersion` as a // postcondition. Nodes along the search path may be invalidated. Callers must // ensure that the max version of the self argument is updated. -[[nodiscard]] TaggedNodePointer *insert(TaggedNodePointer *self, - TrivialSpan key, - InternalVersionT writeVersion, - WriteContext *writeContext) { +[[nodiscard]] TaggedNodePointer * +insert(TaggedNodePointer *self, TrivialSpan key, InternalVersionT writeVersion, + WriteContext *writeContext, ConflictSet::Impl *impl) { for (; key.size() != 0; ++writeContext->accum.insert_iterations) { - self = &getOrCreateChild(*self, key, writeVersion, writeContext); + self = &getOrCreateChild(*self, key, writeVersion, writeContext, impl); } return self; } @@ -2804,9 +2902,10 @@ void eraseTree(Node *root, WriteContext *writeContext) { } void addPointWrite(TaggedNodePointer &root, TrivialSpan key, - InternalVersionT writeVersion, WriteContext *writeContext) { + InternalVersionT writeVersion, WriteContext *writeContext, + ConflictSet::Impl *impl) { ++writeContext->accum.point_writes; - auto n = *insert(&root, key, writeVersion, writeContext); + auto n = *insert(&root, key, writeVersion, writeContext, impl); if (!n->entryPresent) { ++writeContext->accum.entries_inserted; auto *p = nextLogical(n); @@ -2940,8 +3039,8 @@ AddedWriteRange addWriteRange(Node *beginRoot, TrivialSpan begin, Node *endRoot, ++writeContext->accum.range_writes; - Node *beginNode = - *insert(&getInTree(beginRoot, impl), begin, writeVersion, writeContext); + Node *beginNode = *insert(&getInTree(beginRoot, impl), begin, writeVersion, + writeContext, impl); addKey(beginNode); if (!beginNode->entryPresent) { ++writeContext->accum.entries_inserted; @@ -2957,7 +3056,7 @@ AddedWriteRange addWriteRange(Node *beginRoot, TrivialSpan begin, Node *endRoot, beginNode->entry.pointVersion = writeVersion; Node *endNode = - *insert(&getInTree(endRoot, impl), end, writeVersion, writeContext); + *insert(&getInTree(endRoot, impl), end, writeVersion, writeContext, impl); addKey(endNode); if (!endNode->entryPresent) { @@ -3003,10 +3102,10 @@ void addWriteRange(TaggedNodePointer &root, TrivialSpan begin, TrivialSpan end, std::min(begin.size(), end.size())); if (lcp == begin.size() && end.size() == begin.size() + 1 && end.back() == 0) { - return addPointWrite(root, begin, writeVersion, writeContext); + return addPointWrite(root, begin, writeVersion, writeContext, impl); } auto useAsRoot = - insert(&root, begin.subspan(0, lcp), writeVersion, writeContext); + insert(&root, begin.subspan(0, lcp), writeVersion, writeContext, impl); auto [beginNode, endNode] = addWriteRange( *useAsRoot, begin.subspan(lcp, begin.size() - lcp), *useAsRoot, @@ -5026,8 +5125,8 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { } if (context.results[i].endInsertionPoint == nullptr) { addPointWrite(getInTree(context.results[i].insertionPoint, this), - context.results[i].remaining, writeVersion, - &writeContext); + context.results[i].remaining, writeVersion, &writeContext, + this); } else { if (firstRangeWrite == nullptr) { firstRangeWrite = context.results + i; @@ -5104,7 +5203,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { &writeContext, this); } else { addPointWrite(root, begin, InternalVersionT(writeVersion), - &writeContext); + &writeContext, this); } } } @@ -5302,7 +5401,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { keyUpdates = 10; // Insert "" - root = writeContext.allocate(0); + root = writeContext.allocate(0, 0); root->numChildren = 0; root->parent = nullptr; root->entryPresent = false;