weaselab/conflict-set
2
0
Fork 0
Code Issues 4 Packages Releases 13 Activity

Introduce TaggedNodePointer (not actually a tagged pointer yet)

Browse Source
This commit is contained in:
Andrew Noyes
2024-10-10 10:09:52 -07:00
parent c3190c11ac
commit 3739ccaaf2
1 changed files with 111 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
ConflictSet.cpp
View File

@@ -203,6 +203,61 @@ enum Type : int8_t {
template <class T> struct BoundedFreeListAllocator;
struct TaggedNodePointer {
TaggedNodePointer() = default;
operator struct Node *() { return (struct Node *)withoutType(); }
operator struct Node0 *() {
assert(getType() == Type_Node0);
return (struct Node0 *)withoutType();
}
operator struct Node3 *() {
assert(getType() == Type_Node3);
return (struct Node3 *)withoutType();
}
operator struct Node16 *() {
assert(getType() == Type_Node16);
return (struct Node16 *)withoutType();
}
operator struct Node48 *() {
assert(getType() == Type_Node48);
return (struct Node48 *)withoutType();
}
operator struct Node256 *() {
assert(getType() == Type_Node256);
return (struct Node256 *)withoutType();
}
/*implicit*/ TaggedNodePointer(std::nullptr_t) : p(0) {}
/*implicit*/ TaggedNodePointer(Node0 *x)
: TaggedNodePointer((struct Node *)x, Type_Node0) {}
/*implicit*/ TaggedNodePointer(Node3 *x)
: TaggedNodePointer((struct Node *)x, Type_Node3) {}
/*implicit*/ TaggedNodePointer(Node16 *x)
: TaggedNodePointer((struct Node *)x, Type_Node16) {}
/*implicit*/ TaggedNodePointer(Node48 *x)
: TaggedNodePointer((struct Node *)x, Type_Node48) {}
/*implicit*/ TaggedNodePointer(Node256 *x)
: TaggedNodePointer((struct Node *)x, Type_Node256) {}
bool operator!=(std::nullptr_t) { return p != 0; }
bool operator==(std::nullptr_t) { return p == 0; }
bool operator==(const TaggedNodePointer &) const = default;
bool operator==(Node *n) const { return (uintptr_t)n == withoutType(); }
Node *operator->() { return (Node *)withoutType(); }
Type getType();
TaggedNodePointer(const TaggedNodePointer &) = default;
TaggedNodePointer &operator=(const TaggedNodePointer &) = default;
/*implicit*/ TaggedNodePointer(Node *n);
private:
TaggedNodePointer(struct Node *p, Type) : p((uintptr_t)p) {
assert((this->p & 7) == 0);
assume(p != 0);
}
uintptr_t withoutType() const { return p; }
uintptr_t p;
};
struct Node {
/* begin section that's copied to the next node */
@@ -228,6 +283,14 @@ private:
int32_t partialKeyCapacity;
};
TaggedNodePointer::TaggedNodePointer(Node *n)
: TaggedNodePointer(n, n->getType()) {}
Type TaggedNodePointer::getType() {
assert(p != 0);
return ((Node *)p)->getType();
}
constexpr int kNodeCopyBegin = offsetof(Node, entry);
constexpr int kNodeCopySize =
offsetof(Node, parentsIndex) + sizeof(Node::parentsIndex) - kNodeCopyBegin;
@@ -251,7 +314,7 @@ struct Node3 : Node {
constexpr static auto kMaxNodes = 3;
constexpr static auto kType = Type_Node3;
Node *children[kMaxNodes];
TaggedNodePointer children[kMaxNodes];
InternalVersionT childMaxVersion[kMaxNodes];
// Sorted
uint8_t index[kMaxNodes];
@@ -267,7 +330,7 @@ struct Node16 : Node {
constexpr static auto kType = Type_Node16;
constexpr static auto kMaxNodes = 16;
Node *children[kMaxNodes];
TaggedNodePointer children[kMaxNodes];
InternalVersionT childMaxVersion[kMaxNodes];
// Sorted
uint8_t index[kMaxNodes];
@@ -288,7 +351,7 @@ struct Node48 : Node {
constexpr static int kMaxOfMaxTotalPages = kMaxNodes / kMaxOfMaxPageSize;
BitSet bitSet;
Node *children[kMaxNodes];
TaggedNodePointer children[kMaxNodes];
InternalVersionT childMaxVersion[kMaxNodes];
InternalVersionT maxOfMax[kMaxOfMaxTotalPages];
uint8_t reverseIndex[kMaxNodes];
@@ -310,7 +373,7 @@ struct Node256 : Node {
constexpr static int kMaxOfMaxTotalPages = kMaxNodes / kMaxOfMaxPageSize;
BitSet bitSet;
Node *children[kMaxNodes];
TaggedNodePointer children[kMaxNodes];
InternalVersionT childMaxVersion[kMaxNodes];
InternalVersionT maxOfMax[kMaxOfMaxTotalPages];
@@ -888,26 +951,26 @@ int getNodeIndexExists(Node16 *self, uint8_t index) {
}
// Precondition - an entry for index must exist in the node
Node *&getChildExists(Node3 *self, uint8_t index) {
TaggedNodePointer &getChildExists(Node3 *self, uint8_t index) {
return self->children[getNodeIndexExists(self, index)];
}
// Precondition - an entry for index must exist in the node
Node *&getChildExists(Node16 *self, uint8_t index) {
TaggedNodePointer &getChildExists(Node16 *self, uint8_t index) {
return self->children[getNodeIndexExists(self, index)];
}
// Precondition - an entry for index must exist in the node
Node *&getChildExists(Node48 *self, uint8_t index) {
TaggedNodePointer &getChildExists(Node48 *self, uint8_t index) {
assert(self->bitSet.test(index));
return self->children[self->index[index]];
}
// Precondition - an entry for index must exist in the node
Node *&getChildExists(Node256 *self, uint8_t index) {
TaggedNodePointer &getChildExists(Node256 *self, uint8_t index) {
assert(self->bitSet.test(index));
return self->children[index];
}
// Precondition - an entry for index must exist in the node
Node *&getChildExists(Node *self, uint8_t index) {
TaggedNodePointer &getChildExists(Node *self, uint8_t index) {
switch (self->getType()) {
case Type_Node0: // GCOVR_EXCL_LINE
__builtin_unreachable(); // GCOVR_EXCL_LINE
@@ -1038,7 +1101,7 @@ void setMaxVersion(Node *n, InternalVersionT newMax) {
}
}
Node *&getInTree(Node *n, ConflictSet::Impl *);
TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *);
Node *getChild(Node0 *, uint8_t) { return nullptr; }
Node *getChild(Node3 *self, uint8_t index) {
@@ -1245,14 +1308,15 @@ Node *getFirstChildExists(Node *self) {
}
}
void consumePartialKeyFull(Node *&self, std::span<const uint8_t> &key,
void consumePartialKeyFull(TaggedNodePointer &self,
std::span<const uint8_t> &key,
InternalVersionT writeVersion, WriteContext *tls) {
// Handle an existing partial key
int commonLen = std::min<int>(self->partialKeyLen, key.size());
int partialKeyIndex =
longestCommonPrefix(self->partialKey(), key.data(), commonLen);
if (partialKeyIndex < self->partialKeyLen) {
auto *old = self;
Node *old = self;
// Since root cannot have a partial key
assert(old->parent != nullptr);
InternalVersionT oldMaxVersion = exchangeMaxVersion(old, writeVersion);
@@ -1290,7 +1354,7 @@ void consumePartialKeyFull(Node *&self, std::span<const uint8_t> &key,
// Consume any partial key of `self`, and update `self` and
// `key` such that `self` is along the search path of `key`
inline __attribute__((always_inline)) void
consumePartialKey(Node *&self, std::span<const uint8_t> &key,
consumePartialKey(TaggedNodePointer &self, std::span<const uint8_t> &key,
InternalVersionT writeVersion, WriteContext *tls) {
if (self->partialKeyLen > 0) {
consumePartialKeyFull(self, key, writeVersion, tls);
@@ -1301,8 +1365,10 @@ consumePartialKey(Node *&self, std::span<const uint8_t> &key,
// such that the search path of the result + key is the same as the search path
// of self + key before the call. Creates a node if necessary. Updates
// `maxVersion` for result.
Node *&getOrCreateChild(Node *&self, std::span<const uint8_t> &key,
InternalVersionT newMaxVersion, WriteContext *tls) {
TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self,
std::span<const uint8_t> &key,
InternalVersionT newMaxVersion,
WriteContext *tls) {
int index = key.front();
key = key.subspan(1, key.size() - 1);
@@ -1660,10 +1726,11 @@ void rezero(Node *n, InternalVersionT z) {
}
#endif
void mergeWithChild(Node *&self, WriteContext *tls, ConflictSet::Impl *impl,
Node *&dontInvalidate, Node3 *self3) {
void mergeWithChild(TaggedNodePointer &self, WriteContext *tls,
ConflictSet::Impl *impl, Node *&dontInvalidate,
Node3 *self3) {
assert(!self3->entryPresent);
auto *child = self3->children[0];
Node *child = self3->children[0];
int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen;
if (minCapacity > child->getCapacity()) {
@@ -2255,7 +2322,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
if (!mask) {
return true;
}
auto *child = self->children[std::countr_zero(mask)];
Node *child = self->children[std::countr_zero(mask)];
const bool firstRangeOk =
!child->entryPresent || child->entry.rangeVersion <= readVersion;
uint32_t compared = 0;
@@ -2330,7 +2397,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
if (!mask) {
return true;
}
auto *child = self->children[std::countr_zero(mask)];
Node *child = self->children[std::countr_zero(mask)];
const bool firstRangeOk =
!child->entryPresent || child->entry.rangeVersion <= readVersion;
@@ -2375,7 +2442,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
{
int c = self->bitSet.firstSetGeq(begin + 1);
if (c >= 0 && c < end) {
auto *child = self->children[self->index[c]];
Node *child = self->children[self->index[c]];
if (child->entryPresent && child->entry.rangeVersion > readVersion) {
return false;
}
@@ -2409,7 +2476,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
{
int c = self->bitSet.firstSetGeq(begin + 1);
if (c >= 0 && c < end) {
auto *child = self->children[c];
Node *child = self->children[c];
if (child->entryPresent && child->entry.rangeVersion > readVersion) {
return false;
}
@@ -2835,8 +2902,10 @@ checkMaxBetweenExclusiveImpl<true>(Node *n, int begin, int end,
// of the result will have `maxVersion` set to `writeVersion` as a
// postcondition. Nodes along the search path may be invalidated. Callers must
// ensure that the max version of the self argument is updated.
[[nodiscard]] Node **insert(Node **self, std::span<const uint8_t> key,
InternalVersionT writeVersion, WriteContext *tls) {
[[nodiscard]] TaggedNodePointer *insert(TaggedNodePointer *self,
std::span<const uint8_t> key,
InternalVersionT writeVersion,
WriteContext *tls) {
for (; key.size() != 0; ++tls->accum.insert_iterations) {
self = &getOrCreateChild(*self, key, writeVersion, tls);
@@ -2864,17 +2933,23 @@ void eraseTree(Node *root, WriteContext *tls) {
} break;
case Type_Node3: {
auto *n3 = static_cast<Node3 *>(n);
toFree.append(std::span<Node *>(n3->children, n3->numChildren));
for (int i = 0; i < n3->numChildren; ++i) {
toFree.push_back(n3->children[i]);
}
tls->release(n3);
} break;
case Type_Node16: {
auto *n16 = static_cast<Node16 *>(n);
toFree.append(std::span<Node *>(n16->children, n16->numChildren));
for (int i = 0; i < n16->numChildren; ++i) {
toFree.push_back(n16->children[i]);
}
tls->release(n16);
} break;
case Type_Node48: {
auto *n48 = static_cast<Node48 *>(n);
toFree.append(std::span<Node *>(n48->children, n48->numChildren));
for (int i = 0; i < n48->numChildren; ++i) {
toFree.push_back(n48->children[i]);
}
tls->release(n48);
} break;
case Type_Node256: {
@@ -2890,10 +2965,10 @@ void eraseTree(Node *root, WriteContext *tls) {
}
}
void addPointWrite(Node *&root, std::span<const uint8_t> key,
void addPointWrite(TaggedNodePointer &root, std::span<const uint8_t> key,
InternalVersionT writeVersion, WriteContext *tls) {
++tls->accum.point_writes;
auto *n = *insert(&root, key, writeVersion, tls);
auto n = *insert(&root, key, writeVersion, tls);
if (!n->entryPresent) {
++tls->accum.entries_inserted;
auto *p = nextLogical(n);
@@ -3011,7 +3086,7 @@ void fixupMaxVersion(Node *node, WriteContext *tls) {
setMaxVersion(node, max);
}
void addWriteRange(Node *&root, std::span<const uint8_t> begin,
void addWriteRange(TaggedNodePointer &root, std::span<const uint8_t> begin,
std::span<const uint8_t> end, InternalVersionT writeVersion,
WriteContext *tls, ConflictSet::Impl *impl) {
@@ -3024,12 +3099,12 @@ void addWriteRange(Node *&root, std::span<const uint8_t> begin,
++tls->accum.range_writes;
const bool beginIsPrefix = lcp == int(begin.size());
Node **useAsRoot = insert(&root, begin.subspan(0, lcp), writeVersion, tls);
auto useAsRoot = insert(&root, begin.subspan(0, lcp), writeVersion, tls);
begin = begin.subspan(lcp, begin.size() - lcp);
end = end.subspan(lcp, end.size() - lcp);
auto *beginNode = *insert(useAsRoot, begin, writeVersion, tls);
Node *beginNode = *insert(useAsRoot, begin, writeVersion, tls);
addKey(beginNode);
if (!beginNode->entryPresent) {
++tls->accum.entries_inserted;
@@ -3040,7 +3115,7 @@ void addWriteRange(Node *&root, std::span<const uint8_t> begin,
}
beginNode->entry.pointVersion = writeVersion;
auto *endNode = *insert(useAsRoot, end, writeVersion, tls);
Node *endNode = *insert(useAsRoot, end, writeVersion, tls);
addKey(endNode);
if (!endNode->entryPresent) {
++tls->accum.entries_inserted;
@@ -3060,7 +3135,7 @@ void addWriteRange(Node *&root, std::span<const uint8_t> begin,
assert(!beginNode->endOfRange);
assert(!endNode->endOfRange);
endNode->endOfRange = true;
auto *iter = beginNode;
Node *iter = beginNode;
for (iter = nextLogical(iter); !iter->endOfRange;
iter = erase(iter, tls, impl, /*logical*/ true)) {
assert(!iter->endOfRange);
@@ -3375,7 +3450,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
std::span<const uint8_t> removalKey;
int64_t keyUpdates;
Node *root;
TaggedNodePointer root;
InternalVersionT oldestVersion;
int64_t oldestVersionFullPrecision;
int64_t oldestExtantVersion;
@@ -3456,7 +3531,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
}
};
Node *&getInTree(Node *n, ConflictSet::Impl *impl) {
TaggedNodePointer &getInTree(Node *n, ConflictSet::Impl *impl) {
return n->parent == nullptr ? impl->root
: getChildExists(n->parent, n->parentsIndex);
}