weaselab/conflict-set
2
0
Fork 0
Code Issues 4 Packages Releases 13 Activity

Introduce TaggedNodePointer (not actually a tagged pointer yet)

Browse Source
This commit is contained in:
Andrew Noyes
2024-10-10 10:09:52 -07:00
parent c3190c11ac
commit 3739ccaaf2
1 changed files with 111 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
ConflictSet.cpp
View File

@@ -203,6 +203,61 @@ enum Type : int8_t {
template <class T> struct BoundedFreeListAllocator; template <class T> struct BoundedFreeListAllocator;
struct TaggedNodePointer {
TaggedNodePointer() = default;
operator struct Node *() { return (struct Node *)withoutType(); }
operator struct Node0 *() {
assert(getType() == Type_Node0);
return (struct Node0 *)withoutType();
}
operator struct Node3 *() {
assert(getType() == Type_Node3);
return (struct Node3 *)withoutType();
}
operator struct Node16 *() {
assert(getType() == Type_Node16);
return (struct Node16 *)withoutType();
}
operator struct Node48 *() {
assert(getType() == Type_Node48);
return (struct Node48 *)withoutType();
}
operator struct Node256 *() {
assert(getType() == Type_Node256);
return (struct Node256 *)withoutType();
}
/*implicit*/ TaggedNodePointer(std::nullptr_t) : p(0) {}
/*implicit*/ TaggedNodePointer(Node0 *x)
: TaggedNodePointer((struct Node *)x, Type_Node0) {}
/*implicit*/ TaggedNodePointer(Node3 *x)
: TaggedNodePointer((struct Node *)x, Type_Node3) {}
/*implicit*/ TaggedNodePointer(Node16 *x)
: TaggedNodePointer((struct Node *)x, Type_Node16) {}
/*implicit*/ TaggedNodePointer(Node48 *x)
: TaggedNodePointer((struct Node *)x, Type_Node48) {}
/*implicit*/ TaggedNodePointer(Node256 *x)
: TaggedNodePointer((struct Node *)x, Type_Node256) {}
bool operator!=(std::nullptr_t) { return p != 0; }
bool operator==(std::nullptr_t) { return p == 0; }
bool operator==(const TaggedNodePointer &) const = default;
bool operator==(Node *n) const { return (uintptr_t)n == withoutType(); }
Node *operator->() { return (Node *)withoutType(); }
Type getType();
TaggedNodePointer(const TaggedNodePointer &) = default;
TaggedNodePointer &operator=(const TaggedNodePointer &) = default;
/*implicit*/ TaggedNodePointer(Node *n);
private:
TaggedNodePointer(struct Node *p, Type) : p((uintptr_t)p) {
assert((this->p & 7) == 0);
assume(p != 0);
}
uintptr_t withoutType() const { return p; }
uintptr_t p;
};
struct Node { struct Node {
/* begin section that's copied to the next node */ /* begin section that's copied to the next node */
@@ -228,6 +283,14 @@ private:
int32_t partialKeyCapacity; int32_t partialKeyCapacity;
}; };
TaggedNodePointer::TaggedNodePointer(Node *n)
: TaggedNodePointer(n, n->getType()) {}
Type TaggedNodePointer::getType() {
assert(p != 0);
return ((Node *)p)->getType();
}
constexpr int kNodeCopyBegin = offsetof(Node, entry); constexpr int kNodeCopyBegin = offsetof(Node, entry);
constexpr int kNodeCopySize = constexpr int kNodeCopySize =
offsetof(Node, parentsIndex) + sizeof(Node::parentsIndex) - kNodeCopyBegin; offsetof(Node, parentsIndex) + sizeof(Node::parentsIndex) - kNodeCopyBegin;
@@ -251,7 +314,7 @@ struct Node3 : Node {
constexpr static auto kMaxNodes = 3; constexpr static auto kMaxNodes = 3;
constexpr static auto kType = Type_Node3; constexpr static auto kType = Type_Node3;
Node *children[kMaxNodes]; TaggedNodePointer children[kMaxNodes];
InternalVersionT childMaxVersion[kMaxNodes]; InternalVersionT childMaxVersion[kMaxNodes];
// Sorted // Sorted
uint8_t index[kMaxNodes]; uint8_t index[kMaxNodes];
@@ -267,7 +330,7 @@ struct Node16 : Node {
constexpr static auto kType = Type_Node16; constexpr static auto kType = Type_Node16;
constexpr static auto kMaxNodes = 16; constexpr static auto kMaxNodes = 16;
Node *children[kMaxNodes]; TaggedNodePointer children[kMaxNodes];
InternalVersionT childMaxVersion[kMaxNodes]; InternalVersionT childMaxVersion[kMaxNodes];
// Sorted // Sorted
uint8_t index[kMaxNodes]; uint8_t index[kMaxNodes];
@@ -288,7 +351,7 @@ struct Node48 : Node {
constexpr static int kMaxOfMaxTotalPages = kMaxNodes / kMaxOfMaxPageSize; constexpr static int kMaxOfMaxTotalPages = kMaxNodes / kMaxOfMaxPageSize;
BitSet bitSet; BitSet bitSet;
Node *children[kMaxNodes]; TaggedNodePointer children[kMaxNodes];
InternalVersionT childMaxVersion[kMaxNodes]; InternalVersionT childMaxVersion[kMaxNodes];
InternalVersionT maxOfMax[kMaxOfMaxTotalPages]; InternalVersionT maxOfMax[kMaxOfMaxTotalPages];
uint8_t reverseIndex[kMaxNodes]; uint8_t reverseIndex[kMaxNodes];
@@ -310,7 +373,7 @@ struct Node256 : Node {
constexpr static int kMaxOfMaxTotalPages = kMaxNodes / kMaxOfMaxPageSize; constexpr static int kMaxOfMaxTotalPages = kMaxNodes / kMaxOfMaxPageSize;
BitSet bitSet; BitSet bitSet;
Node *children[kMaxNodes]; TaggedNodePointer children[kMaxNodes];
InternalVersionT childMaxVersion[kMaxNodes]; InternalVersionT childMaxVersion[kMaxNodes];
InternalVersionT maxOfMax[kMaxOfMaxTotalPages]; InternalVersionT maxOfMax[kMaxOfMaxTotalPages];
@@ -888,26 +951,26 @@ int getNodeIndexExists(Node16 *self, uint8_t index) {
} }
// Precondition - an entry for index must exist in the node // Precondition - an entry for index must exist in the node
Node *&getChildExists(Node3 *self, uint8_t index) { TaggedNodePointer &getChildExists(Node3 *self, uint8_t index) {
return self->children[getNodeIndexExists(self, index)]; return self->children[getNodeIndexExists(self, index)];
} }
// Precondition - an entry for index must exist in the node // Precondition - an entry for index must exist in the node
Node *&getChildExists(Node16 *self, uint8_t index) { TaggedNodePointer &getChildExists(Node16 *self, uint8_t index) {
return self->children[getNodeIndexExists(self, index)]; return self->children[getNodeIndexExists(self, index)];
} }
// Precondition - an entry for index must exist in the node // Precondition - an entry for index must exist in the node
Node *&getChildExists(Node48 *self, uint8_t index) { TaggedNodePointer &getChildExists(Node48 *self, uint8_t index) {
assert(self->bitSet.test(index)); assert(self->bitSet.test(index));
return self->children[self->index[index]]; return self->children[self->index[index]];
} }
// Precondition - an entry for index must exist in the node // Precondition - an entry for index must exist in the node
Node *&getChildExists(Node256 *self, uint8_t index) { TaggedNodePointer &getChildExists(Node256 *self, uint8_t index) {
assert(self->bitSet.test(index)); assert(self->bitSet.test(index));
return self->children[index]; return self->children[index];
} }
// Precondition - an entry for index must exist in the node // Precondition - an entry for index must exist in the node
Node *&getChildExists(Node *self, uint8_t index) { TaggedNodePointer &getChildExists(Node *self, uint8_t index) {
switch (self->getType()) { switch (self->getType()) {
case Type_Node0: // GCOVR_EXCL_LINE case Type_Node0: // GCOVR_EXCL_LINE
__builtin_unreachable(); // GCOVR_EXCL_LINE __builtin_unreachable(); // GCOVR_EXCL_LINE
@@ -1038,7 +1101,7 @@ void setMaxVersion(Node *n, InternalVersionT newMax) {
} }
} }
Node *&getInTree(Node *n, ConflictSet::Impl *); TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *);
Node *getChild(Node0 *, uint8_t) { return nullptr; } Node *getChild(Node0 *, uint8_t) { return nullptr; }
Node *getChild(Node3 *self, uint8_t index) { Node *getChild(Node3 *self, uint8_t index) {
@@ -1245,14 +1308,15 @@ Node *getFirstChildExists(Node *self) {
} }
} }
void consumePartialKeyFull(Node *&self, std::span<const uint8_t> &key, void consumePartialKeyFull(TaggedNodePointer &self,
std::span<const uint8_t> &key,
InternalVersionT writeVersion, WriteContext *tls) { InternalVersionT writeVersion, WriteContext *tls) {
// Handle an existing partial key // Handle an existing partial key
int commonLen = std::min<int>(self->partialKeyLen, key.size()); int commonLen = std::min<int>(self->partialKeyLen, key.size());
int partialKeyIndex = int partialKeyIndex =
longestCommonPrefix(self->partialKey(), key.data(), commonLen); longestCommonPrefix(self->partialKey(), key.data(), commonLen);
if (partialKeyIndex < self->partialKeyLen) { if (partialKeyIndex < self->partialKeyLen) {
auto *old = self; Node *old = self;
// Since root cannot have a partial key // Since root cannot have a partial key
assert(old->parent != nullptr); assert(old->parent != nullptr);
InternalVersionT oldMaxVersion = exchangeMaxVersion(old, writeVersion); InternalVersionT oldMaxVersion = exchangeMaxVersion(old, writeVersion);
@@ -1290,7 +1354,7 @@ void consumePartialKeyFull(Node *&self, std::span<const uint8_t> &key,
// Consume any partial key of `self`, and update `self` and // Consume any partial key of `self`, and update `self` and
// `key` such that `self` is along the search path of `key` // `key` such that `self` is along the search path of `key`
inline __attribute__((always_inline)) void inline __attribute__((always_inline)) void
consumePartialKey(Node *&self, std::span<const uint8_t> &key, consumePartialKey(TaggedNodePointer &self, std::span<const uint8_t> &key,
InternalVersionT writeVersion, WriteContext *tls) { InternalVersionT writeVersion, WriteContext *tls) {
if (self->partialKeyLen > 0) { if (self->partialKeyLen > 0) {
consumePartialKeyFull(self, key, writeVersion, tls); consumePartialKeyFull(self, key, writeVersion, tls);
@@ -1301,8 +1365,10 @@ consumePartialKey(Node *&self, std::span<const uint8_t> &key,
// such that the search path of the result + key is the same as the search path // such that the search path of the result + key is the same as the search path
// of self + key before the call. Creates a node if necessary. Updates // of self + key before the call. Creates a node if necessary. Updates
// `maxVersion` for result. // `maxVersion` for result.
Node *&getOrCreateChild(Node *&self, std::span<const uint8_t> &key, TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self,
InternalVersionT newMaxVersion, WriteContext *tls) { std::span<const uint8_t> &key,
InternalVersionT newMaxVersion,
WriteContext *tls) {
int index = key.front(); int index = key.front();
key = key.subspan(1, key.size() - 1); key = key.subspan(1, key.size() - 1);
@@ -1660,10 +1726,11 @@ void rezero(Node *n, InternalVersionT z) {
} }
#endif #endif
void mergeWithChild(Node *&self, WriteContext *tls, ConflictSet::Impl *impl, void mergeWithChild(TaggedNodePointer &self, WriteContext *tls,
Node *&dontInvalidate, Node3 *self3) { ConflictSet::Impl *impl, Node *&dontInvalidate,
Node3 *self3) {
assert(!self3->entryPresent); assert(!self3->entryPresent);
auto *child = self3->children[0]; Node *child = self3->children[0];
int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen; int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen;
if (minCapacity > child->getCapacity()) { if (minCapacity > child->getCapacity()) {
@@ -2255,7 +2322,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
if (!mask) { if (!mask) {
return true; return true;
} }
auto *child = self->children[std::countr_zero(mask)]; Node *child = self->children[std::countr_zero(mask)];
const bool firstRangeOk = const bool firstRangeOk =
!child->entryPresent || child->entry.rangeVersion <= readVersion; !child->entryPresent || child->entry.rangeVersion <= readVersion;
uint32_t compared = 0; uint32_t compared = 0;
@@ -2330,7 +2397,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
if (!mask) { if (!mask) {
return true; return true;
} }
auto *child = self->children[std::countr_zero(mask)]; Node *child = self->children[std::countr_zero(mask)];
const bool firstRangeOk = const bool firstRangeOk =
!child->entryPresent || child->entry.rangeVersion <= readVersion; !child->entryPresent || child->entry.rangeVersion <= readVersion;
@@ -2375,7 +2442,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
{ {
int c = self->bitSet.firstSetGeq(begin + 1); int c = self->bitSet.firstSetGeq(begin + 1);
if (c >= 0 && c < end) { if (c >= 0 && c < end) {
auto *child = self->children[self->index[c]]; Node *child = self->children[self->index[c]];
if (child->entryPresent && child->entry.rangeVersion > readVersion) { if (child->entryPresent && child->entry.rangeVersion > readVersion) {
return false; return false;
} }
@@ -2409,7 +2476,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
{ {
int c = self->bitSet.firstSetGeq(begin + 1); int c = self->bitSet.firstSetGeq(begin + 1);
if (c >= 0 && c < end) { if (c >= 0 && c < end) {
auto *child = self->children[c]; Node *child = self->children[c];
if (child->entryPresent && child->entry.rangeVersion > readVersion) { if (child->entryPresent && child->entry.rangeVersion > readVersion) {
return false; return false;
} }
@@ -2835,8 +2902,10 @@ checkMaxBetweenExclusiveImpl<true>(Node *n, int begin, int end,
// of the result will have `maxVersion` set to `writeVersion` as a // of the result will have `maxVersion` set to `writeVersion` as a
// postcondition. Nodes along the search path may be invalidated. Callers must // postcondition. Nodes along the search path may be invalidated. Callers must
// ensure that the max version of the self argument is updated. // ensure that the max version of the self argument is updated.
[[nodiscard]] Node **insert(Node **self, std::span<const uint8_t> key, [[nodiscard]] TaggedNodePointer *insert(TaggedNodePointer *self,
InternalVersionT writeVersion, WriteContext *tls) { std::span<const uint8_t> key,
InternalVersionT writeVersion,
WriteContext *tls) {
for (; key.size() != 0; ++tls->accum.insert_iterations) { for (; key.size() != 0; ++tls->accum.insert_iterations) {
self = &getOrCreateChild(*self, key, writeVersion, tls); self = &getOrCreateChild(*self, key, writeVersion, tls);
@@ -2864,17 +2933,23 @@ void eraseTree(Node *root, WriteContext *tls) {
} break; } break;
case Type_Node3: { case Type_Node3: {
auto *n3 = static_cast<Node3 *>(n); auto *n3 = static_cast<Node3 *>(n);
toFree.append(std::span<Node *>(n3->children, n3->numChildren)); for (int i = 0; i < n3->numChildren; ++i) {
toFree.push_back(n3->children[i]);
}
tls->release(n3); tls->release(n3);
} break; } break;
case Type_Node16: { case Type_Node16: {
auto *n16 = static_cast<Node16 *>(n); auto *n16 = static_cast<Node16 *>(n);
toFree.append(std::span<Node *>(n16->children, n16->numChildren)); for (int i = 0; i < n16->numChildren; ++i) {
toFree.push_back(n16->children[i]);
}
tls->release(n16); tls->release(n16);
} break; } break;
case Type_Node48: { case Type_Node48: {
auto *n48 = static_cast<Node48 *>(n); auto *n48 = static_cast<Node48 *>(n);
toFree.append(std::span<Node *>(n48->children, n48->numChildren)); for (int i = 0; i < n48->numChildren; ++i) {
toFree.push_back(n48->children[i]);
}
tls->release(n48); tls->release(n48);
} break; } break;
case Type_Node256: { case Type_Node256: {
@@ -2890,10 +2965,10 @@ void eraseTree(Node *root, WriteContext *tls) {
} }
} }
void addPointWrite(Node *&root, std::span<const uint8_t> key, void addPointWrite(TaggedNodePointer &root, std::span<const uint8_t> key,
InternalVersionT writeVersion, WriteContext *tls) { InternalVersionT writeVersion, WriteContext *tls) {
++tls->accum.point_writes; ++tls->accum.point_writes;
auto *n = *insert(&root, key, writeVersion, tls); auto n = *insert(&root, key, writeVersion, tls);
if (!n->entryPresent) { if (!n->entryPresent) {
++tls->accum.entries_inserted; ++tls->accum.entries_inserted;
auto *p = nextLogical(n); auto *p = nextLogical(n);
@@ -3011,7 +3086,7 @@ void fixupMaxVersion(Node *node, WriteContext *tls) {
setMaxVersion(node, max); setMaxVersion(node, max);
} }
void addWriteRange(Node *&root, std::span<const uint8_t> begin, void addWriteRange(TaggedNodePointer &root, std::span<const uint8_t> begin,
std::span<const uint8_t> end, InternalVersionT writeVersion, std::span<const uint8_t> end, InternalVersionT writeVersion,
WriteContext *tls, ConflictSet::Impl *impl) { WriteContext *tls, ConflictSet::Impl *impl) {
@@ -3024,12 +3099,12 @@ void addWriteRange(Node *&root, std::span<const uint8_t> begin,
++tls->accum.range_writes; ++tls->accum.range_writes;
const bool beginIsPrefix = lcp == int(begin.size()); const bool beginIsPrefix = lcp == int(begin.size());
Node **useAsRoot = insert(&root, begin.subspan(0, lcp), writeVersion, tls); auto useAsRoot = insert(&root, begin.subspan(0, lcp), writeVersion, tls);
begin = begin.subspan(lcp, begin.size() - lcp); begin = begin.subspan(lcp, begin.size() - lcp);
end = end.subspan(lcp, end.size() - lcp); end = end.subspan(lcp, end.size() - lcp);
auto *beginNode = *insert(useAsRoot, begin, writeVersion, tls); Node *beginNode = *insert(useAsRoot, begin, writeVersion, tls);
addKey(beginNode); addKey(beginNode);
if (!beginNode->entryPresent) { if (!beginNode->entryPresent) {
++tls->accum.entries_inserted; ++tls->accum.entries_inserted;
@@ -3040,7 +3115,7 @@ void addWriteRange(Node *&root, std::span<const uint8_t> begin,
} }
beginNode->entry.pointVersion = writeVersion; beginNode->entry.pointVersion = writeVersion;
auto *endNode = *insert(useAsRoot, end, writeVersion, tls); Node *endNode = *insert(useAsRoot, end, writeVersion, tls);
addKey(endNode); addKey(endNode);
if (!endNode->entryPresent) { if (!endNode->entryPresent) {
++tls->accum.entries_inserted; ++tls->accum.entries_inserted;
@@ -3060,7 +3135,7 @@ void addWriteRange(Node *&root, std::span<const uint8_t> begin,
assert(!beginNode->endOfRange); assert(!beginNode->endOfRange);
assert(!endNode->endOfRange); assert(!endNode->endOfRange);
endNode->endOfRange = true; endNode->endOfRange = true;
auto *iter = beginNode; Node *iter = beginNode;
for (iter = nextLogical(iter); !iter->endOfRange; for (iter = nextLogical(iter); !iter->endOfRange;
iter = erase(iter, tls, impl, /*logical*/ true)) { iter = erase(iter, tls, impl, /*logical*/ true)) {
assert(!iter->endOfRange); assert(!iter->endOfRange);
@@ -3375,7 +3450,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
std::span<const uint8_t> removalKey; std::span<const uint8_t> removalKey;
int64_t keyUpdates; int64_t keyUpdates;
Node *root; TaggedNodePointer root;
InternalVersionT oldestVersion; InternalVersionT oldestVersion;
int64_t oldestVersionFullPrecision; int64_t oldestVersionFullPrecision;
int64_t oldestExtantVersion; int64_t oldestExtantVersion;
@@ -3456,7 +3531,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
} }
}; };
Node *&getInTree(Node *n, ConflictSet::Impl *impl) { TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *impl) {
return n->parent == nullptr ? impl->root return n->parent == nullptr ? impl->root
: getChildExists(n->parent, n->parentsIndex); : getChildExists(n->parent, n->parentsIndex);
} }