diff --git a/ConflictSet.cpp b/ConflictSet.cpp index b563d34..b5a6d64 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -224,6 +224,9 @@ struct Node0 : Node { void copyChildrenAndKeyFrom(const Node0 &other); void copyChildrenAndKeyFrom(const struct Node3 &other); + void copyChildrenAndKeyFrom(const struct Node16 &other); + void copyChildrenAndKeyFrom(const struct Node48 &other); + void copyChildrenAndKeyFrom(const struct Node256 &other); size_t size() const { return sizeof(Node0) + getCapacity(); } }; @@ -240,6 +243,8 @@ struct Node3 : Node { void copyChildrenAndKeyFrom(const Node0 &other); void copyChildrenAndKeyFrom(const Node3 &other); void copyChildrenAndKeyFrom(const struct Node16 &other); + void copyChildrenAndKeyFrom(const struct Node48 &other); + void copyChildrenAndKeyFrom(const struct Node256 &other); size_t size() const { return sizeof(Node3) + getCapacity(); } }; @@ -256,6 +261,7 @@ struct Node16 : Node { void copyChildrenAndKeyFrom(const Node3 &other); void copyChildrenAndKeyFrom(const Node16 &other); void copyChildrenAndKeyFrom(const struct Node48 &other); + void copyChildrenAndKeyFrom(const struct Node256 &other); size_t size() const { return sizeof(Node16) + getCapacity(); } }; @@ -313,6 +319,24 @@ inline void Node0::copyChildrenAndKeyFrom(const Node3 &other) { memcpy(partialKey(), &other + 1, partialKeyLen); } +inline void Node0::copyChildrenAndKeyFrom(const Node16 &other) { + memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin, + kNodeCopySize); + memcpy(partialKey(), &other + 1, partialKeyLen); +} + +inline void Node0::copyChildrenAndKeyFrom(const Node48 &other) { + memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin, + kNodeCopySize); + memcpy(partialKey(), &other + 1, partialKeyLen); +} + +inline void Node0::copyChildrenAndKeyFrom(const Node256 &other) { + memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin, + kNodeCopySize); + memcpy(partialKey(), &other + 1, partialKeyLen); +} + inline void Node3::copyChildrenAndKeyFrom(const Node0 &other) { memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin, kNodeCopySize); @@ -344,17 +368,53 @@ inline void Node3::copyChildrenAndKeyFrom(const Node16 &other) { } } +inline void Node3::copyChildrenAndKeyFrom(const Node48 &other) { + memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin, + kNodeCopySize); + int i = 0; + other.bitSet.forEachSet([&](int c) { + // Suppress a false positive -Waggressive-loop-optimizations warning + // in gcc + assume(i < kMaxNodes); + index[i] = c; + children[i] = other.children[other.index[c]]; + childMaxVersion[i] = other.childMaxVersion[other.index[c]]; + assert(children[i]->parent == &other); + children[i]->parent = this; + ++i; + }); + memcpy(partialKey(), &other + 1, partialKeyLen); +} + +inline void Node3::copyChildrenAndKeyFrom(const Node256 &other) { + memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin, + kNodeCopySize); + int i = 0; + other.bitSet.forEachSet([&](int c) { + // Suppress a false positive -Waggressive-loop-optimizations warning + // in gcc + assume(i < kMaxNodes); + index[i] = c; + children[i] = other.children[c]; + childMaxVersion[i] = other.childMaxVersion[c]; + assert(children[i]->parent == &other); + children[i]->parent = this; + ++i; + }); + memcpy(partialKey(), &other + 1, partialKeyLen); +} + inline void Node16::copyChildrenAndKeyFrom(const Node3 &other) { memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin, kNodeCopySize); - memcpy(index, other.index, Node3::kMaxNodes); + memcpy(index, other.index, other.kMaxNodes); memcpy(children, other.children, - Node3::kMaxNodes * sizeof(children[0])); // NOLINT + other.kMaxNodes * sizeof(children[0])); // NOLINT memcpy(childMaxVersion, other.childMaxVersion, - Node3::kMaxNodes * sizeof(childMaxVersion[0])); + other.kMaxNodes * sizeof(childMaxVersion[0])); memcpy(partialKey(), &other + 1, partialKeyLen); - assert(numChildren == Node3::kMaxNodes); - for (int i = 0; i < Node3::kMaxNodes; ++i) { + assert(numChildren == other.kMaxNodes); + for (int i = 0; i < other.kMaxNodes; ++i) { assert(children[i]->parent == &other); children[i]->parent = this; } @@ -380,7 +440,7 @@ inline void Node16::copyChildrenAndKeyFrom(const Node48 &other) { other.bitSet.forEachSet([&](int c) { // Suppress a false positive -Waggressive-loop-optimizations warning // in gcc - assume(i < Node16::kMaxNodes); + assume(i < kMaxNodes); index[i] = c; children[i] = other.children[other.index[c]]; childMaxVersion[i] = other.childMaxVersion[other.index[c]]; @@ -391,10 +451,28 @@ inline void Node16::copyChildrenAndKeyFrom(const Node48 &other) { memcpy(partialKey(), &other + 1, partialKeyLen); } +inline void Node16::copyChildrenAndKeyFrom(const Node256 &other) { + memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin, + kNodeCopySize); + int i = 0; + other.bitSet.forEachSet([&](int c) { + // Suppress a false positive -Waggressive-loop-optimizations warning + // in gcc + assume(i < kMaxNodes); + index[i] = c; + children[i] = other.children[c]; + childMaxVersion[i] = other.childMaxVersion[c]; + assert(children[i]->parent == &other); + children[i]->parent = this; + ++i; + }); + memcpy(partialKey(), &other + 1, partialKeyLen); +} + inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) { memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin, kNodeCopySize); - assert(numChildren == Node16::kMaxNodes); + assert(numChildren == other.kMaxNodes); memset(index, -1, sizeof(index)); memset(children, 0, sizeof(children)); const auto z = InternalVersionT::zero; @@ -403,7 +481,7 @@ inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) { } memcpy(partialKey(), &other + 1, partialKeyLen); bitSet.init(); - nextFree = Node16::kMaxNodes; + nextFree = other.kMaxNodes; int i = 0; for (auto x : other.index) { bitSet.set(x); @@ -413,8 +491,8 @@ inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) { assert(children[i]->parent == &other); children[i]->parent = this; reverseIndex[i] = x; - maxOfMax[i >> Node48::kMaxOfMaxShift] = - std::max(maxOfMax[i >> Node48::kMaxOfMaxShift], childMaxVersion[i]); + maxOfMax[i >> kMaxOfMaxShift] = + std::max(maxOfMax[i >> kMaxOfMaxShift], childMaxVersion[i]); ++i; } } @@ -456,15 +534,15 @@ inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) { bitSet.forEachSet([&](int c) { // Suppress a false positive -Waggressive-loop-optimizations warning // in gcc. - assume(i < Node48::kMaxNodes); + assume(i < kMaxNodes); index[c] = i; children[i] = other.children[c]; childMaxVersion[i] = other.childMaxVersion[c]; assert(children[i]->parent == &other); children[i]->parent = this; reverseIndex[i] = c; - maxOfMax[i >> Node48::kMaxOfMaxShift] = - std::max(maxOfMax[i >> Node48::kMaxOfMaxShift], childMaxVersion[i]); + maxOfMax[i >> kMaxOfMaxShift] = + std::max(maxOfMax[i >> kMaxOfMaxShift], childMaxVersion[i]); ++i; }); memcpy(partialKey(), &other + 1, partialKeyLen); @@ -487,8 +565,8 @@ inline void Node256::copyChildrenAndKeyFrom(const Node48 &other) { childMaxVersion[c] = other.childMaxVersion[other.index[c]]; assert(children[c]->parent == &other); children[c]->parent = this; - maxOfMax[c >> Node256::kMaxOfMaxShift] = - std::max(maxOfMax[c >> Node256::kMaxOfMaxShift], childMaxVersion[c]); + maxOfMax[c >> kMaxOfMaxShift] = + std::max(maxOfMax[c >> kMaxOfMaxShift], childMaxVersion[c]); }); memcpy(partialKey(), &other + 1, partialKeyLen); } @@ -740,6 +818,7 @@ struct WriteContext { int64_t nodes_allocated; int64_t nodes_released; int64_t point_writes; + int64_t prefix_writes; int64_t range_writes; int64_t write_bytes; } accum; @@ -1635,10 +1714,224 @@ void maybeDownsize(Node *self, WriteContext *tls, ConflictSet::Impl *impl, } } +void destroyTree(Node *root, int64_t *entriesErased) { + Arena arena; + auto toFree = vector(arena); + toFree.push_back(root); + +#if SHOW_MEMORY + for (auto *iter = root; iter != nullptr; iter = nextPhysical(iter)) { + removeNode(iter); + removeKey(iter); + } +#endif + + while (toFree.size() > 0) { + auto *n = toFree.back(); + toFree.pop_back(); + *entriesErased += n->entryPresent; + + switch (n->getType()) { + case Type_Node0: { + } break; + case Type_Node3: { + auto *n3 = static_cast(n); + toFree.append(std::span(n3->children, n3->numChildren)); + } break; + case Type_Node16: { + auto *n16 = static_cast(n); + toFree.append(std::span(n16->children, n16->numChildren)); + } break; + case Type_Node48: { + auto *n48 = static_cast(n); + toFree.append(std::span(n48->children, n48->numChildren)); + } break; + case Type_Node256: { + auto *n256 = static_cast(n); + auto *out = toFree.unsafePrepareAppend(n256->numChildren).data(); + n256->bitSet.forEachSet([&](int i) { *out++ = n256->children[i]; }); + assert(out == toFree.end()); + } break; + default: // GCOVR_EXCL_LINE + __builtin_unreachable(); // GCOVR_EXCL_LINE + } + safe_free(n, n->size()); + } +} + +void eraseBetween(Node3 *&n, int begin, int end, WriteContext *tls) { + const unsigned shiftUpperBound = end - begin; + const unsigned shiftAmount = begin; + auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; }; + Node **nodeOut = n->children; + uint8_t *indexOut = n->index; + InternalVersionT *maxVOut = n->childMaxVersion; + for (int i = 0; i < n->numChildren; ++i) { + if (inBounds(n->index[i])) { + destroyTree(n->children[i], &tls->accum.entries_erased); + } else { + *nodeOut++ = n->children[i]; + *indexOut++ = n->index[i]; + *maxVOut++ = n->childMaxVersion[i]; + } + } + n->numChildren = nodeOut - n->children; + + if (n->numChildren == 0) { + auto *newNode = tls->allocate(n->partialKeyLen); + newNode->copyChildrenAndKeyFrom(*n); + tls->release(n); + (Node *&)n = newNode; + } +} + +void eraseBetween(Node16 *&n, int begin, int end, WriteContext *tls) { + const unsigned shiftUpperBound = end - begin; + const unsigned shiftAmount = begin; + auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; }; + Node **nodeOut = n->children; + uint8_t *indexOut = n->index; + InternalVersionT *maxVOut = n->childMaxVersion; + for (int i = 0; i < n->numChildren; ++i) { + if (inBounds(n->index[i])) { + destroyTree(n->children[i], &tls->accum.entries_erased); + } else { + *nodeOut++ = n->children[i]; + *indexOut++ = n->index[i]; + *maxVOut++ = n->childMaxVersion[i]; + } + } + n->numChildren = nodeOut - n->children; + + if (n->numChildren == 0) { + auto *newNode = tls->allocate(n->partialKeyLen); + newNode->copyChildrenAndKeyFrom(*n); + tls->release(n); + (Node *&)n = newNode; + } else if (n->numChildren <= Node3::kMaxNodes) { + auto *newNode = tls->allocate(n->partialKeyLen); + newNode->copyChildrenAndKeyFrom(*n); + tls->release(n); + (Node *&)n = newNode; + } +} + +void eraseBetween(Node48 *&n, int begin, int end, WriteContext *tls) { + const unsigned shiftUpperBound = end - begin; + const unsigned shiftAmount = begin; + auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; }; + Node **nodeOut = n->children; + uint8_t *indexOut = n->reverseIndex; + InternalVersionT *maxVOut = n->childMaxVersion; + for (auto &v : n->maxOfMax) { + v = tls->zero; + } + n->bitSet = {}; + memset(n->index, -1, sizeof(n->index)); + n->nextFree = 0; + for (int i = 0; i < n->numChildren; ++i) { + if (inBounds(n->reverseIndex[i])) { + destroyTree(n->children[i], &tls->accum.entries_erased); + } else { + *nodeOut++ = n->children[i]; + *indexOut++ = n->reverseIndex[i]; + *maxVOut++ = n->childMaxVersion[i]; + n->maxOfMax[n->reverseIndex[i] >> Node48::kMaxOfMaxShift] = + std::max(n->maxOfMax[n->reverseIndex[i] >> Node48::kMaxOfMaxShift], + n->childMaxVersion[i]); + n->bitSet.set(n->reverseIndex[i]); + n->index[n->reverseIndex[i]] = n->nextFree++; + } + } + n->numChildren = n->nextFree; + + if (n->numChildren == 0) { + auto *newNode = tls->allocate(n->partialKeyLen); + newNode->copyChildrenAndKeyFrom(*n); + tls->release(n); + (Node *&)n = newNode; + } else if (n->numChildren <= Node3::kMaxNodes) { + auto *newNode = tls->allocate(n->partialKeyLen); + newNode->copyChildrenAndKeyFrom(*n); + tls->release(n); + (Node *&)n = newNode; + } else if (n->numChildren <= Node16::kMaxNodes) { + auto *newNode = tls->allocate(n->partialKeyLen); + newNode->copyChildrenAndKeyFrom(*n); + tls->release(n); + (Node *&)n = newNode; + } +} + +void eraseBetween(Node256 *&n, int begin, int end, WriteContext *tls) { + const unsigned shiftUpperBound = end - begin; + const unsigned shiftAmount = begin; + auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; }; + n->numChildren = 0; + BitSet newBitSet; + n->bitSet.forEachSet([&](int i) { + if (inBounds(i)) { + destroyTree(std::exchange(n->children[i], nullptr), + &tls->accum.entries_erased); + } else { + ++n->numChildren; + newBitSet.set(i); + } + }); + n->bitSet = newBitSet; + // Don't need to update childMaxVersion or maxOfMax because of monotonicity + if (n->numChildren == 0) { + auto *newNode = tls->allocate(n->partialKeyLen); + newNode->copyChildrenAndKeyFrom(*n); + tls->release(n); + (Node *&)n = newNode; + } else if (n->numChildren <= Node3::kMaxNodes) { + auto *newNode = tls->allocate(n->partialKeyLen); + newNode->copyChildrenAndKeyFrom(*n); + tls->release(n); + (Node *&)n = newNode; + } else if (n->numChildren <= Node16::kMaxNodes) { + auto *newNode = tls->allocate(n->partialKeyLen); + newNode->copyChildrenAndKeyFrom(*n); + tls->release(n); + (Node *&)n = newNode; + } + if (n->numChildren <= Node48::kMaxNodes) { + auto *newNode = tls->allocate(n->partialKeyLen); + newNode->copyChildrenAndKeyFrom(*n); + tls->release(n); + (Node *&)n = newNode; + } +} + +// Erase all nodes with a search path starting with n + [child], +// where child in [begin, end). +void eraseBetween(Node *&n, int begin, int end, WriteContext *tls) { + switch (n->getType()) { + case Type_Node0: + break; + case Type_Node3: + eraseBetween((Node3 *&)n, begin, end, tls); + break; + case Type_Node16: + eraseBetween((Node16 *&)n, begin, end, tls); + break; + case Type_Node48: + eraseBetween((Node48 *&)n, begin, end, tls); + break; + case Type_Node256: + eraseBetween((Node256 *&)n, begin, end, tls); + break; + default: // GCOVR_EXCL_LINE + __builtin_unreachable(); // GCOVR_EXCL_LINE + } +} + // Precondition: self is not the root. May invalidate nodes along the search // path to self. May invalidate children of self->parent. Returns a pointer to -// the node after self. If erase invalidates the pointee of `dontInvalidate`, it -// will update it to its new pointee as well. Precondition: `self->entryPresent` +// the node after self. If erase invalidates the pointee of `dontInvalidate`, +// it will update it to its new pointee as well. Precondition: +// `self->entryPresent` Node *erase(Node *self, WriteContext *tls, ConflictSet::Impl *impl, bool logical, Node *&dontInvalidate) { ++tls->accum.entries_erased; @@ -1892,8 +2185,8 @@ bool checkPrefixRead(Node *n, const std::span key, remaining = remaining.subspan(commonLen, remaining.size() - commonLen); } else if (n->partialKeyLen > int(remaining.size())) { // n is the first physical node greater than remaining, and there's no - // eq node. All physical nodes that start with prefix are reachable from - // n. + // eq node. All physical nodes that start with prefix are reachable + // from n. if (maxVersion(n, impl) > readVersion) { return false; } @@ -1942,8 +2235,8 @@ compare16_32bit_avx512(const InternalVersionT *vs, InternalVersionT rv) { #endif // GCOVR_EXCL_STOP -// Returns true if v[i] <= readVersion for all i such that begin <= is[i] < end -// Preconditions: begin <= end, end - begin < 256 +// Returns true if v[i] <= readVersion for all i such that begin <= is[i] < +// end Preconditions: begin <= end, end - begin < 256 template bool scan16(const InternalVersionT *vs, const uint8_t *is, int begin, int end, InternalVersionT readVersion) { @@ -2074,9 +2367,10 @@ bool scan16(const InternalVersionT *vs, int begin, int end, #endif } -// Return whether or not the max version among all keys starting with the search -// path of n + [child], where child in (begin, end) is <= readVersion. Does not -// account for the range version of firstGt(searchpath(n) + [end - 1]) +// Return whether or not the max version among all keys starting with the +// search path of n + [child], where child in (begin, end) is <= readVersion. +// Does not account for the range version of firstGt(searchpath(n) + [end - +// 1]) template bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end, InternalVersionT readVersion, @@ -2245,8 +2539,8 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end, } else { return true; } - // [begin, end) is now the half-open interval of children we're interested - // in. + // [begin, end) is now the half-open interval of children we're + // interested in. assert(begin < end); } @@ -2281,8 +2575,8 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end, } else { return true; } - // [begin, end) is now the half-open interval of children we're interested - // in. + // [begin, end) is now the half-open interval of children we're + // interested in. assert(begin < end); } @@ -2339,8 +2633,9 @@ checkMaxBetweenExclusive(Node *n, int begin, int end, // GCOVR_EXCL_STOP __attribute__((target("default"))) #endif -bool checkMaxBetweenExclusive(Node *n, int begin, int end, - InternalVersionT readVersion, ReadContext *tls) { + bool + checkMaxBetweenExclusive(Node * n, int begin, int end, + InternalVersionT readVersion, ReadContext *tls) { return checkMaxBetweenExclusiveImpl(n, begin, end, readVersion, tls); } @@ -2361,8 +2656,8 @@ Vector getSearchPath(Arena &arena, Node *n) { return result; } // GCOVR_EXCL_LINE -// Return true if the max version among all keys that start with key + [child], -// where begin < child < end, is <= readVersion. +// Return true if the max version among all keys that start with key + +// [child], where begin < child < end, is <= readVersion. // // Precondition: transitively, no child of n has a search path that's a longer // prefix of key than n @@ -2432,8 +2727,8 @@ downLeftSpine: } namespace { -// Return true if the max version among all keys that start with key[:prefixLen] -// that are >= key is <= readVersion +// Return true if the max version among all keys that start with +// key[:prefixLen] that are >= key is <= readVersion struct CheckRangeLeftSide { CheckRangeLeftSide(Node *n, std::span key, int prefixLen, InternalVersionT readVersion, ReadContext *tls) @@ -2547,8 +2842,8 @@ struct CheckRangeLeftSide { } }; -// Return true if the max version among all keys that start with key[:prefixLen] -// that are < key is <= readVersion +// Return true if the max version among all keys that start with +// key[:prefixLen] that are < key is <= readVersion struct CheckRangeRightSide { CheckRangeRightSide(Node *n, std::span key, int prefixLen, InternalVersionT readVersion, ReadContext *tls) @@ -2794,9 +3089,9 @@ checkMaxBetweenExclusiveImpl(Node *n, int begin, int end, InternalVersionT readVersion, ReadContext *); #endif -// Returns a pointer the pointer to the newly inserted node in the tree. Caller -// must set `entryPresent`, and `entry` fields. All nodes along the search path -// of the result will have `maxVersion` set to `writeVersion` as a +// Returns a pointer the pointer to the newly inserted node in the tree. +// Caller must set `entryPresent`, and `entry` fields. All nodes along the +// search path of the result will have `maxVersion` set to `writeVersion` as a // postcondition. Nodes along the search path may be invalidated. [[nodiscard]] Node **insert(Node **self, std::span key, @@ -2812,51 +3107,6 @@ Node **insert(Node **self, std::span key, return self; } -void destroyTree(Node *root, int64_t *entriesErased) { - Arena arena; - auto toFree = vector(arena); - toFree.push_back(root); - -#if SHOW_MEMORY - for (auto *iter = root; iter != nullptr; iter = nextPhysical(iter)) { - removeNode(iter); - removeKey(iter); - } -#endif - - while (toFree.size() > 0) { - auto *n = toFree.back(); - toFree.pop_back(); - *entriesErased += n->entryPresent; - - switch (n->getType()) { - case Type_Node0: { - } break; - case Type_Node3: { - auto *n3 = static_cast(n); - toFree.append(std::span(n3->children, n3->numChildren)); - } break; - case Type_Node16: { - auto *n16 = static_cast(n); - toFree.append(std::span(n16->children, n16->numChildren)); - } break; - case Type_Node48: { - auto *n48 = static_cast(n); - toFree.append(std::span(n48->children, n48->numChildren)); - } break; - case Type_Node256: { - auto *n256 = static_cast(n); - auto *out = toFree.unsafePrepareAppend(n256->numChildren).data(); - n256->bitSet.forEachSet([&](int i) { *out++ = n256->children[i]; }); - assert(out == toFree.end()); - } break; - default: // GCOVR_EXCL_LINE - __builtin_unreachable(); // GCOVR_EXCL_LINE - } - safe_free(n, n->size()); - } -} - void addPointWrite(Node *&root, std::span key, InternalVersionT writeVersion, WriteContext *tls, ConflictSet::Impl *impl) { @@ -2916,6 +3166,55 @@ void fixupMaxVersion(Node *node, ConflictSet::Impl *impl, WriteContext *tls) { setMaxVersion(node, impl, max); } +void addPrefixWrite(Node *&root, std::span begin, + std::span end, InternalVersionT writeVersion, + WriteContext *tls, ConflictSet::Impl *impl) { + ++tls->accum.prefix_writes; + int lcp = begin.size() - 1; + + Node **useAsRoot = + insert(&root, begin.subspan(0, lcp), writeVersion, tls, impl); + + auto *beginNode = + *insert(useAsRoot, begin.subspan(lcp, 1), writeVersion, tls, impl); + + const bool insertedBegin = !beginNode->entryPresent; + + addKey(beginNode); + beginNode->entryPresent = true; + + if (insertedBegin) { + ++tls->accum.entries_inserted; + auto *p = nextLogical(beginNode); + beginNode->entry.rangeVersion = + p == nullptr ? tls->zero : std::max(p->entry.rangeVersion, tls->zero); + beginNode->entry.pointVersion = writeVersion; + } + assert(writeVersion >= beginNode->entry.pointVersion); + beginNode->entry.pointVersion = writeVersion; + + auto *endNode = + *insert(useAsRoot, end.subspan(lcp, 1), writeVersion, tls, impl); + + const bool insertedEnd = !endNode->entryPresent; + + addKey(endNode); + endNode->entryPresent = true; + + if (insertedEnd) { + ++tls->accum.entries_inserted; + auto *p = nextLogical(endNode); + endNode->entry.pointVersion = + p == nullptr ? tls->zero : std::max(p->entry.rangeVersion, tls->zero); + } + endNode->entry.rangeVersion = writeVersion; + + eraseBetween(getInTree(beginNode, impl), 0, 256, tls); + + // Inserting end trashed endNode's maxVersion. Fix that + fixupMaxVersion(endNode, impl, tls); +} + void addWriteRange(Node *&root, std::span begin, std::span end, InternalVersionT writeVersion, WriteContext *tls, ConflictSet::Impl *impl) { @@ -2926,6 +3225,10 @@ void addWriteRange(Node *&root, std::span begin, end.back() == 0) { return addPointWrite(root, begin, writeVersion, tls, impl); } + if (lcp == int(begin.size() - 1) && end.size() == begin.size() && + int(begin.back()) + 1 == int(end.back())) { + return addPrefixWrite(root, begin, end, writeVersion, tls, impl); + } ++tls->accum.range_writes; const bool beginIsPrefix = lcp == int(begin.size()); @@ -3128,6 +3431,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { memory_bytes.set(totalBytes); point_writes_total.add(tls.accum.point_writes); + prefix_writes_total.add(tls.accum.prefix_writes); range_writes_total.add(tls.accum.range_writes); nodes_allocated_total.add(tls.accum.nodes_allocated); nodes_released_total.add(tls.accum.nodes_released); @@ -3153,17 +3457,17 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { int64_t set_oldest_iterations_accum = 0; for (; fuel > 0 && n != nullptr; ++set_oldest_iterations_accum) { rezero(n, oldestVersion); - // The "make sure gc keeps up with writes" calculations assume that we're - // scanning key by key, not node by node. Make sure we only spend fuel - // when there's a logical entry. + // The "make sure gc keeps up with writes" calculations assume that + // we're scanning key by key, not node by node. Make sure we only spend + // fuel when there's a logical entry. fuel -= n->entryPresent; if (n->entryPresent && std::max(n->entry.pointVersion, n->entry.rangeVersion) <= oldestVersion) { // Any transaction n would have prevented from committing is // going to fail with TooOld anyway. - // There's no way to insert a range such that range version of the right - // node is greater than the point version of the left node + // There's no way to insert a range such that range version of the + // right node is greater than the point version of the left node assert(n->entry.rangeVersion <= oldestVersion); Node *dummy = nullptr; n = erase(n, &tls, this, /*logical*/ false, dummy); @@ -3200,9 +3504,9 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { this->oldestVersion = oldestVersion; InternalVersionT::zero = tls.zero = oldestVersion; #ifdef NDEBUG - // This is here for performance reasons, since we want to amortize the cost - // of storing the search path as a string. In tests, we want to exercise the - // rest of the code often. + // This is here for performance reasons, since we want to amortize the + // cost of storing the search path as a string. In tests, we want to + // exercise the rest of the code often. if (keyUpdates < 100) { return; } @@ -3326,8 +3630,8 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { "Total number of checks where the result is \"too old\""); COUNTER(check_bytes_total, "Total number of key bytes checked"); COUNTER(point_writes_total, "Total number of point writes"); - COUNTER(range_writes_total, - "Total number of range writes (includes prefix writes)"); + COUNTER(prefix_writes_total, "Total number of prefix writes"); + COUNTER(range_writes_total, "Total number of range writes"); GAUGE(memory_bytes, "Total number of bytes in use"); COUNTER(nodes_allocated_total, "The total number of physical tree nodes allocated"); @@ -3341,16 +3645,14 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { "The total number of entries inserted in the tree"); COUNTER(entries_erased_total, "The total number of entries erased from the tree"); - COUNTER( - gc_iterations_total, - "The total number of iterations of the main loop for garbage collection"); + COUNTER(gc_iterations_total, "The total number of iterations of the main " + "loop for garbage collection"); COUNTER(write_bytes_total, "Total number of key bytes in calls to addWrites"); GAUGE(oldest_version, "The lowest version that doesn't result in \"TooOld\" for checks"); GAUGE(newest_version, "The version of the most recent call to addWrites"); - GAUGE( - oldest_extant_version, - "A lower bound on the lowest version associated with an existing entry"); + GAUGE(oldest_extant_version, "A lower bound on the lowest version " + "associated with an existing entry"); // ==================== END METRICS DEFINITIONS ==================== #undef GAUGE #undef COUNTER @@ -3725,8 +4027,8 @@ std::string strinc(std::string_view str, bool &ok) { if ((uint8_t &)(str[index]) != 255) break; - // Must not be called with a string that consists only of zero or more '\xff' - // bytes. + // Must not be called with a string that consists only of zero or more + // '\xff' bytes. if (index < 0) { ok = false; return {};