diff --git a/ConflictSet.cpp b/ConflictSet.cpp index ec32d43..a718baa 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -203,6 +203,61 @@ enum Type : int8_t { template struct BoundedFreeListAllocator; +struct TaggedNodePointer { + TaggedNodePointer() = default; + operator struct Node *() { return (struct Node *)withoutType(); } + operator struct Node0 *() { + assert(getType() == Type_Node0); + return (struct Node0 *)withoutType(); + } + operator struct Node3 *() { + assert(getType() == Type_Node3); + return (struct Node3 *)withoutType(); + } + operator struct Node16 *() { + assert(getType() == Type_Node16); + return (struct Node16 *)withoutType(); + } + operator struct Node48 *() { + assert(getType() == Type_Node48); + return (struct Node48 *)withoutType(); + } + operator struct Node256 *() { + assert(getType() == Type_Node256); + return (struct Node256 *)withoutType(); + } + /*implicit*/ TaggedNodePointer(std::nullptr_t) : p(0) {} + /*implicit*/ TaggedNodePointer(Node0 *x) + : TaggedNodePointer((struct Node *)x, Type_Node0) {} + /*implicit*/ TaggedNodePointer(Node3 *x) + : TaggedNodePointer((struct Node *)x, Type_Node3) {} + /*implicit*/ TaggedNodePointer(Node16 *x) + : TaggedNodePointer((struct Node *)x, Type_Node16) {} + /*implicit*/ TaggedNodePointer(Node48 *x) + : TaggedNodePointer((struct Node *)x, Type_Node48) {} + /*implicit*/ TaggedNodePointer(Node256 *x) + : TaggedNodePointer((struct Node *)x, Type_Node256) {} + + bool operator!=(std::nullptr_t) { return p != 0; } + bool operator==(std::nullptr_t) { return p == 0; } + bool operator==(const TaggedNodePointer &) const = default; + bool operator==(Node *n) const { return (uintptr_t)n == withoutType(); } + Node *operator->() { return (Node *)withoutType(); } + Type getType(); + + TaggedNodePointer(const TaggedNodePointer &) = default; + TaggedNodePointer &operator=(const TaggedNodePointer &) = default; + /*implicit*/ TaggedNodePointer(Node *n); + +private: + TaggedNodePointer(struct Node *p, Type) : p((uintptr_t)p) { + assert((this->p & 7) == 0); + assume(p != 0); + } + uintptr_t withoutType() const { return p; } + uintptr_t p; +}; + struct Node { /* begin section that's copied to the next node */ @@ -228,6 +283,14 @@ private: int32_t partialKeyCapacity; }; +TaggedNodePointer::TaggedNodePointer(Node *n) + : TaggedNodePointer(n, n->getType()) {} + +Type TaggedNodePointer::getType() { + assert(p != 0); + return ((Node *)p)->getType(); +} + constexpr int kNodeCopyBegin = offsetof(Node, entry); constexpr int kNodeCopySize = offsetof(Node, parentsIndex) + sizeof(Node::parentsIndex) - kNodeCopyBegin; @@ -251,7 +314,7 @@ struct Node3 : Node { constexpr static auto kMaxNodes = 3; constexpr static auto kType = Type_Node3; - Node *children[kMaxNodes]; + TaggedNodePointer children[kMaxNodes]; InternalVersionT childMaxVersion[kMaxNodes]; // Sorted uint8_t index[kMaxNodes]; @@ -267,7 +330,7 @@ struct Node16 : Node { constexpr static auto kType = Type_Node16; constexpr static auto kMaxNodes = 16; - Node *children[kMaxNodes]; + TaggedNodePointer children[kMaxNodes]; InternalVersionT childMaxVersion[kMaxNodes]; // Sorted uint8_t index[kMaxNodes]; @@ -288,7 +351,7 @@ struct Node48 : Node { constexpr static int kMaxOfMaxTotalPages = kMaxNodes / kMaxOfMaxPageSize; BitSet bitSet; - Node *children[kMaxNodes]; + TaggedNodePointer children[kMaxNodes]; InternalVersionT childMaxVersion[kMaxNodes]; InternalVersionT maxOfMax[kMaxOfMaxTotalPages]; uint8_t reverseIndex[kMaxNodes]; @@ -310,7 +373,7 @@ struct Node256 : Node { constexpr static int kMaxOfMaxTotalPages = kMaxNodes / kMaxOfMaxPageSize; BitSet bitSet; - Node *children[kMaxNodes]; + TaggedNodePointer children[kMaxNodes]; InternalVersionT childMaxVersion[kMaxNodes]; InternalVersionT maxOfMax[kMaxOfMaxTotalPages]; @@ -888,26 +951,26 @@ int getNodeIndexExists(Node16 *self, uint8_t index) { } // Precondition - an entry for index must exist in the node -Node *&getChildExists(Node3 *self, uint8_t index) { +TaggedNodePointer &getChildExists(Node3 *self, uint8_t index) { return self->children[getNodeIndexExists(self, index)]; } // Precondition - an entry for index must exist in the node -Node *&getChildExists(Node16 *self, uint8_t index) { +TaggedNodePointer &getChildExists(Node16 *self, uint8_t index) { return self->children[getNodeIndexExists(self, index)]; } // Precondition - an entry for index must exist in the node -Node *&getChildExists(Node48 *self, uint8_t index) { +TaggedNodePointer &getChildExists(Node48 *self, uint8_t index) { assert(self->bitSet.test(index)); return self->children[self->index[index]]; } // Precondition - an entry for index must exist in the node -Node *&getChildExists(Node256 *self, uint8_t index) { +TaggedNodePointer &getChildExists(Node256 *self, uint8_t index) { assert(self->bitSet.test(index)); return self->children[index]; } // Precondition - an entry for index must exist in the node -Node *&getChildExists(Node *self, uint8_t index) { +TaggedNodePointer &getChildExists(Node *self, uint8_t index) { switch (self->getType()) { case Type_Node0: // GCOVR_EXCL_LINE __builtin_unreachable(); // GCOVR_EXCL_LINE @@ -1038,7 +1101,7 @@ void setMaxVersion(Node *n, InternalVersionT newMax) { } } -Node *&getInTree(Node *n, ConflictSet::Impl *); +TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *); Node *getChild(Node0 *, uint8_t) { return nullptr; } Node *getChild(Node3 *self, uint8_t index) { @@ -1245,14 +1308,15 @@ Node *getFirstChildExists(Node *self) { } } -void consumePartialKeyFull(Node *&self, std::span &key, +void consumePartialKeyFull(TaggedNodePointer &self, + std::span &key, InternalVersionT writeVersion, WriteContext *tls) { // Handle an existing partial key int commonLen = std::min(self->partialKeyLen, key.size()); int partialKeyIndex = longestCommonPrefix(self->partialKey(), key.data(), commonLen); if (partialKeyIndex < self->partialKeyLen) { - auto *old = self; + Node *old = self; // Since root cannot have a partial key assert(old->parent != nullptr); InternalVersionT oldMaxVersion = exchangeMaxVersion(old, writeVersion); @@ -1290,7 +1354,7 @@ void consumePartialKeyFull(Node *&self, std::span &key, // Consume any partial key of `self`, and update `self` and // `key` such that `self` is along the search path of `key` inline __attribute__((always_inline)) void -consumePartialKey(Node *&self, std::span &key, +consumePartialKey(TaggedNodePointer &self, std::span &key, InternalVersionT writeVersion, WriteContext *tls) { if (self->partialKeyLen > 0) { consumePartialKeyFull(self, key, writeVersion, tls); @@ -1301,8 +1365,10 @@ consumePartialKey(Node *&self, std::span &key, // such that the search path of the result + key is the same as the search path // of self + key before the call. Creates a node if necessary. Updates // `maxVersion` for result. -Node *&getOrCreateChild(Node *&self, std::span &key, - InternalVersionT newMaxVersion, WriteContext *tls) { +TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, + std::span &key, + InternalVersionT newMaxVersion, + WriteContext *tls) { int index = key.front(); key = key.subspan(1, key.size() - 1); @@ -1660,10 +1726,11 @@ void rezero(Node *n, InternalVersionT z) { } #endif -void mergeWithChild(Node *&self, WriteContext *tls, ConflictSet::Impl *impl, - Node *&dontInvalidate, Node3 *self3) { +void mergeWithChild(TaggedNodePointer &self, WriteContext *tls, + ConflictSet::Impl *impl, Node *&dontInvalidate, + Node3 *self3) { assert(!self3->entryPresent); - auto *child = self3->children[0]; + Node *child = self3->children[0]; int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen; if (minCapacity > child->getCapacity()) { @@ -2255,7 +2322,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end, if (!mask) { return true; } - auto *child = self->children[std::countr_zero(mask)]; + Node *child = self->children[std::countr_zero(mask)]; const bool firstRangeOk = !child->entryPresent || child->entry.rangeVersion <= readVersion; uint32_t compared = 0; @@ -2330,7 +2397,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end, if (!mask) { return true; } - auto *child = self->children[std::countr_zero(mask)]; + Node *child = self->children[std::countr_zero(mask)]; const bool firstRangeOk = !child->entryPresent || child->entry.rangeVersion <= readVersion; @@ -2375,7 +2442,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end, { int c = self->bitSet.firstSetGeq(begin + 1); if (c >= 0 && c < end) { - auto *child = self->children[self->index[c]]; + Node *child = self->children[self->index[c]]; if (child->entryPresent && child->entry.rangeVersion > readVersion) { return false; } @@ -2409,7 +2476,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end, { int c = self->bitSet.firstSetGeq(begin + 1); if (c >= 0 && c < end) { - auto *child = self->children[c]; + Node *child = self->children[c]; if (child->entryPresent && child->entry.rangeVersion > readVersion) { return false; } @@ -2835,8 +2902,10 @@ checkMaxBetweenExclusiveImpl(Node *n, int begin, int end, // of the result will have `maxVersion` set to `writeVersion` as a // postcondition. Nodes along the search path may be invalidated. Callers must // ensure that the max version of the self argument is updated. -[[nodiscard]] Node **insert(Node **self, std::span key, - InternalVersionT writeVersion, WriteContext *tls) { +[[nodiscard]] TaggedNodePointer *insert(TaggedNodePointer *self, + std::span key, + InternalVersionT writeVersion, + WriteContext *tls) { for (; key.size() != 0; ++tls->accum.insert_iterations) { self = &getOrCreateChild(*self, key, writeVersion, tls); @@ -2864,17 +2933,23 @@ void eraseTree(Node *root, WriteContext *tls) { } break; case Type_Node3: { auto *n3 = static_cast(n); - toFree.append(std::span(n3->children, n3->numChildren)); + for (int i = 0; i < n3->numChildren; ++i) { + toFree.push_back(n3->children[i]); + } tls->release(n3); } break; case Type_Node16: { auto *n16 = static_cast(n); - toFree.append(std::span(n16->children, n16->numChildren)); + for (int i = 0; i < n16->numChildren; ++i) { + toFree.push_back(n16->children[i]); + } tls->release(n16); } break; case Type_Node48: { auto *n48 = static_cast(n); - toFree.append(std::span(n48->children, n48->numChildren)); + for (int i = 0; i < n48->numChildren; ++i) { + toFree.push_back(n48->children[i]); + } tls->release(n48); } break; case Type_Node256: { @@ -2890,10 +2965,10 @@ void eraseTree(Node *root, WriteContext *tls) { } } -void addPointWrite(Node *&root, std::span key, +void addPointWrite(TaggedNodePointer &root, std::span key, InternalVersionT writeVersion, WriteContext *tls) { ++tls->accum.point_writes; - auto *n = *insert(&root, key, writeVersion, tls); + auto n = *insert(&root, key, writeVersion, tls); if (!n->entryPresent) { ++tls->accum.entries_inserted; auto *p = nextLogical(n); @@ -3011,7 +3086,7 @@ void fixupMaxVersion(Node *node, WriteContext *tls) { setMaxVersion(node, max); } -void addWriteRange(Node *&root, std::span begin, +void addWriteRange(TaggedNodePointer &root, std::span begin, std::span end, InternalVersionT writeVersion, WriteContext *tls, ConflictSet::Impl *impl) { @@ -3024,12 +3099,12 @@ void addWriteRange(Node *&root, std::span begin, ++tls->accum.range_writes; const bool beginIsPrefix = lcp == int(begin.size()); - Node **useAsRoot = insert(&root, begin.subspan(0, lcp), writeVersion, tls); + auto useAsRoot = insert(&root, begin.subspan(0, lcp), writeVersion, tls); begin = begin.subspan(lcp, begin.size() - lcp); end = end.subspan(lcp, end.size() - lcp); - auto *beginNode = *insert(useAsRoot, begin, writeVersion, tls); + Node *beginNode = *insert(useAsRoot, begin, writeVersion, tls); addKey(beginNode); if (!beginNode->entryPresent) { ++tls->accum.entries_inserted; @@ -3040,7 +3115,7 @@ void addWriteRange(Node *&root, std::span begin, } beginNode->entry.pointVersion = writeVersion; - auto *endNode = *insert(useAsRoot, end, writeVersion, tls); + Node *endNode = *insert(useAsRoot, end, writeVersion, tls); addKey(endNode); if (!endNode->entryPresent) { ++tls->accum.entries_inserted; @@ -3060,7 +3135,7 @@ void addWriteRange(Node *&root, std::span begin, assert(!beginNode->endOfRange); assert(!endNode->endOfRange); endNode->endOfRange = true; - auto *iter = beginNode; + Node *iter = beginNode; for (iter = nextLogical(iter); !iter->endOfRange; iter = erase(iter, tls, impl, /*logical*/ true)) { assert(!iter->endOfRange); @@ -3375,7 +3450,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { std::span removalKey; int64_t keyUpdates; - Node *root; + TaggedNodePointer root; InternalVersionT oldestVersion; int64_t oldestVersionFullPrecision; int64_t oldestExtantVersion; @@ -3456,7 +3531,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { } }; -Node *&getInTree(Node *n, ConflictSet::Impl *impl) { +TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *impl) { return n->parent == nullptr ? impl->root : getChildExists(n->parent, n->parentsIndex); }