|
|
|
@@ -236,11 +236,6 @@ constexpr int kNodeCopyBegin = offsetof(Node, entry);
|
|
|
|
|
constexpr int kNodeCopySize =
|
|
|
|
|
offsetof(Node, parentsIndex) + sizeof(Node::parentsIndex) - kNodeCopyBegin;
|
|
|
|
|
|
|
|
|
|
struct Child {
|
|
|
|
|
int64_t childMaxVersion;
|
|
|
|
|
Node *child;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
// copyChildrenAndKeyFrom is responsible for copying all
|
|
|
|
|
// public members of Node, copying the partial key, logically copying the
|
|
|
|
|
// children (converting representation if necessary), and updating all the
|
|
|
|
@@ -262,7 +257,8 @@ struct Node3 : Node {
|
|
|
|
|
constexpr static auto kType = Type_Node3;
|
|
|
|
|
// Sorted
|
|
|
|
|
uint8_t index[kMaxNodes];
|
|
|
|
|
Child children[kMaxNodes];
|
|
|
|
|
Node *children[kMaxNodes];
|
|
|
|
|
int64_t childMaxVersion[kMaxNodes];
|
|
|
|
|
uint8_t *partialKey() { return (uint8_t *)(this + 1); }
|
|
|
|
|
|
|
|
|
|
void copyChildrenAndKeyFrom(const Node0 &other);
|
|
|
|
@@ -277,7 +273,8 @@ struct Node16 : Node {
|
|
|
|
|
constexpr static auto kMaxNodes = 16;
|
|
|
|
|
// Sorted
|
|
|
|
|
uint8_t index[kMaxNodes];
|
|
|
|
|
Child children[kMaxNodes];
|
|
|
|
|
Node *children[kMaxNodes];
|
|
|
|
|
int64_t childMaxVersion[kMaxNodes];
|
|
|
|
|
uint8_t *partialKey() { return (uint8_t *)(this + 1); }
|
|
|
|
|
|
|
|
|
|
void copyChildrenAndKeyFrom(const Node3 &other);
|
|
|
|
@@ -293,7 +290,14 @@ struct Node48 : Node {
|
|
|
|
|
BitSet bitSet;
|
|
|
|
|
int8_t nextFree;
|
|
|
|
|
int8_t index[256];
|
|
|
|
|
Child children[kMaxNodes];
|
|
|
|
|
Node *children[kMaxNodes];
|
|
|
|
|
int64_t childMaxVersion[kMaxNodes];
|
|
|
|
|
uint8_t reverseIndex[kMaxNodes];
|
|
|
|
|
constexpr static int kMaxOfMaxPageSize = 16;
|
|
|
|
|
constexpr static int kMaxOfMaxShift =
|
|
|
|
|
std::countr_zero(uint32_t(kMaxOfMaxPageSize));
|
|
|
|
|
constexpr static int kMaxOfMaxTotalPages = kMaxNodes / kMaxOfMaxPageSize;
|
|
|
|
|
int64_t maxOfMax[kMaxOfMaxTotalPages];
|
|
|
|
|
|
|
|
|
|
uint8_t *partialKey() { return (uint8_t *)(this + 1); }
|
|
|
|
|
|
|
|
|
@@ -307,7 +311,13 @@ struct Node48 : Node {
|
|
|
|
|
struct Node256 : Node {
|
|
|
|
|
constexpr static auto kType = Type_Node256;
|
|
|
|
|
BitSet bitSet;
|
|
|
|
|
Child children[256];
|
|
|
|
|
Node *children[256];
|
|
|
|
|
int64_t childMaxVersion[256];
|
|
|
|
|
constexpr static int kMaxOfMaxPageSize = 8;
|
|
|
|
|
constexpr static int kMaxOfMaxShift =
|
|
|
|
|
std::countr_zero(uint32_t(kMaxOfMaxPageSize));
|
|
|
|
|
constexpr static int kMaxOfMaxTotalPages = 256 / kMaxOfMaxPageSize;
|
|
|
|
|
int64_t maxOfMax[kMaxOfMaxTotalPages];
|
|
|
|
|
uint8_t *partialKey() { return (uint8_t *)(this + 1); }
|
|
|
|
|
void copyChildrenAndKeyFrom(const Node48 &other);
|
|
|
|
|
void copyChildrenAndKeyFrom(const Node256 &other);
|
|
|
|
@@ -339,8 +349,8 @@ inline void Node3::copyChildrenAndKeyFrom(const Node3 &other) {
|
|
|
|
|
memcpy(index, other.index, sizeof(*this) - sizeof(Node));
|
|
|
|
|
memcpy(partialKey(), &other + 1, partialKeyLen);
|
|
|
|
|
for (int i = 0; i < numChildren; ++i) {
|
|
|
|
|
assert(children[i].child->parent == &other);
|
|
|
|
|
children[i].child->parent = this;
|
|
|
|
|
assert(children[i]->parent == &other);
|
|
|
|
|
children[i]->parent = this;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -348,11 +358,13 @@ inline void Node3::copyChildrenAndKeyFrom(const Node16 &other) {
|
|
|
|
|
memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
|
|
|
|
|
kNodeCopySize);
|
|
|
|
|
memcpy(index, other.index, kMaxNodes);
|
|
|
|
|
memcpy(children, other.children, kMaxNodes * sizeof(Child));
|
|
|
|
|
memcpy(children, other.children, kMaxNodes * sizeof(children[0])); // NOLINT
|
|
|
|
|
memcpy(childMaxVersion, other.childMaxVersion,
|
|
|
|
|
kMaxNodes * sizeof(childMaxVersion[0]));
|
|
|
|
|
memcpy(partialKey(), &other + 1, partialKeyLen);
|
|
|
|
|
for (int i = 0; i < numChildren; ++i) {
|
|
|
|
|
assert(children[i].child->parent == &other);
|
|
|
|
|
children[i].child->parent = this;
|
|
|
|
|
assert(children[i]->parent == &other);
|
|
|
|
|
children[i]->parent = this;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -360,12 +372,15 @@ inline void Node16::copyChildrenAndKeyFrom(const Node3 &other) {
|
|
|
|
|
memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
|
|
|
|
|
kNodeCopySize);
|
|
|
|
|
memcpy(index, other.index, Node3::kMaxNodes);
|
|
|
|
|
memcpy(children, other.children, Node3::kMaxNodes * sizeof(Child));
|
|
|
|
|
memcpy(children, other.children,
|
|
|
|
|
Node3::kMaxNodes * sizeof(children[0])); // NOLINT
|
|
|
|
|
memcpy(childMaxVersion, other.childMaxVersion,
|
|
|
|
|
Node3::kMaxNodes * sizeof(childMaxVersion[0]));
|
|
|
|
|
memcpy(partialKey(), &other + 1, partialKeyLen);
|
|
|
|
|
assert(numChildren == Node3::kMaxNodes);
|
|
|
|
|
for (int i = 0; i < Node3::kMaxNodes; ++i) {
|
|
|
|
|
assert(children[i].child->parent == &other);
|
|
|
|
|
children[i].child->parent = this;
|
|
|
|
|
assert(children[i]->parent == &other);
|
|
|
|
|
children[i]->parent = this;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -375,8 +390,9 @@ inline void Node16::copyChildrenAndKeyFrom(const Node16 &other) {
|
|
|
|
|
memcpy(index, other.index, sizeof(index));
|
|
|
|
|
for (int i = 0; i < numChildren; ++i) {
|
|
|
|
|
children[i] = other.children[i];
|
|
|
|
|
assert(children[i].child->parent == &other);
|
|
|
|
|
children[i].child->parent = this;
|
|
|
|
|
childMaxVersion[i] = other.childMaxVersion[i];
|
|
|
|
|
assert(children[i]->parent == &other);
|
|
|
|
|
children[i]->parent = this;
|
|
|
|
|
}
|
|
|
|
|
memcpy(partialKey(), &other + 1, partialKeyLen);
|
|
|
|
|
}
|
|
|
|
@@ -392,8 +408,9 @@ inline void Node16::copyChildrenAndKeyFrom(const Node48 &other) {
|
|
|
|
|
assume(i < Node16::kMaxNodes);
|
|
|
|
|
index[i] = c;
|
|
|
|
|
children[i] = other.children[other.index[c]];
|
|
|
|
|
assert(children[i].child->parent == &other);
|
|
|
|
|
children[i].child->parent = this;
|
|
|
|
|
childMaxVersion[i] = other.childMaxVersion[other.index[c]];
|
|
|
|
|
assert(children[i]->parent == &other);
|
|
|
|
|
children[i]->parent = this;
|
|
|
|
|
++i;
|
|
|
|
|
},
|
|
|
|
|
0, 256);
|
|
|
|
@@ -405,6 +422,8 @@ inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) {
|
|
|
|
|
kNodeCopySize);
|
|
|
|
|
assert(numChildren == Node16::kMaxNodes);
|
|
|
|
|
memset(index, -1, sizeof(index));
|
|
|
|
|
memset(children, 0, sizeof(children));
|
|
|
|
|
memset(childMaxVersion, 0, sizeof(childMaxVersion));
|
|
|
|
|
memcpy(partialKey(), &other + 1, partialKeyLen);
|
|
|
|
|
bitSet.init();
|
|
|
|
|
nextFree = Node16::kMaxNodes;
|
|
|
|
@@ -413,8 +432,12 @@ inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) {
|
|
|
|
|
bitSet.set(x);
|
|
|
|
|
index[x] = i;
|
|
|
|
|
children[i] = other.children[i];
|
|
|
|
|
assert(children[i].child->parent == &other);
|
|
|
|
|
children[i].child->parent = this;
|
|
|
|
|
childMaxVersion[i] = other.childMaxVersion[i];
|
|
|
|
|
assert(children[i]->parent == &other);
|
|
|
|
|
children[i]->parent = this;
|
|
|
|
|
reverseIndex[i] = x;
|
|
|
|
|
maxOfMax[i >> Node48::kMaxOfMaxShift] =
|
|
|
|
|
std::max(maxOfMax[i >> Node48::kMaxOfMaxShift], childMaxVersion[i]);
|
|
|
|
|
++i;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@@ -422,13 +445,19 @@ inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) {
|
|
|
|
|
inline void Node48::copyChildrenAndKeyFrom(const Node48 &other) {
|
|
|
|
|
memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
|
|
|
|
|
kNodeCopySize);
|
|
|
|
|
memcpy(&bitSet, &other.bitSet,
|
|
|
|
|
sizeof(*this) - sizeof(Node) - sizeof(children));
|
|
|
|
|
bitSet = other.bitSet;
|
|
|
|
|
nextFree = other.nextFree;
|
|
|
|
|
memcpy(index, other.index, sizeof(index));
|
|
|
|
|
memset(children, 0, sizeof(children));
|
|
|
|
|
memset(childMaxVersion, 0, sizeof(childMaxVersion));
|
|
|
|
|
for (int i = 0; i < numChildren; ++i) {
|
|
|
|
|
children[i] = other.children[i];
|
|
|
|
|
assert(children[i].child->parent == &other);
|
|
|
|
|
children[i].child->parent = this;
|
|
|
|
|
childMaxVersion[i] = other.childMaxVersion[i];
|
|
|
|
|
assert(children[i]->parent == &other);
|
|
|
|
|
children[i]->parent = this;
|
|
|
|
|
}
|
|
|
|
|
memcpy(reverseIndex, other.reverseIndex, sizeof(reverseIndex));
|
|
|
|
|
memcpy(maxOfMax, other.maxOfMax, sizeof(maxOfMax));
|
|
|
|
|
memcpy(partialKey(), &other + 1, partialKeyLen);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -436,6 +465,8 @@ inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) {
|
|
|
|
|
memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
|
|
|
|
|
kNodeCopySize);
|
|
|
|
|
memset(index, -1, sizeof(index));
|
|
|
|
|
memset(children, 0, sizeof(children));
|
|
|
|
|
memset(childMaxVersion, 0, sizeof(childMaxVersion));
|
|
|
|
|
nextFree = other.numChildren;
|
|
|
|
|
bitSet = other.bitSet;
|
|
|
|
|
int i = 0;
|
|
|
|
@@ -446,8 +477,12 @@ inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) {
|
|
|
|
|
assume(i < Node48::kMaxNodes);
|
|
|
|
|
index[c] = i;
|
|
|
|
|
children[i] = other.children[c];
|
|
|
|
|
assert(children[i].child->parent == &other);
|
|
|
|
|
children[i].child->parent = this;
|
|
|
|
|
childMaxVersion[i] = other.childMaxVersion[c];
|
|
|
|
|
assert(children[i]->parent == &other);
|
|
|
|
|
children[i]->parent = this;
|
|
|
|
|
reverseIndex[i] = c;
|
|
|
|
|
maxOfMax[i >> Node48::kMaxOfMaxShift] =
|
|
|
|
|
std::max(maxOfMax[i >> Node48::kMaxOfMaxShift], childMaxVersion[i]);
|
|
|
|
|
++i;
|
|
|
|
|
},
|
|
|
|
|
0, 256);
|
|
|
|
@@ -457,13 +492,18 @@ inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) {
|
|
|
|
|
inline void Node256::copyChildrenAndKeyFrom(const Node48 &other) {
|
|
|
|
|
memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
|
|
|
|
|
kNodeCopySize);
|
|
|
|
|
memset(children, 0, sizeof(children));
|
|
|
|
|
bitSet = other.bitSet;
|
|
|
|
|
memset(children, 0, sizeof(children));
|
|
|
|
|
memset(childMaxVersion, 0, sizeof(childMaxVersion));
|
|
|
|
|
memset(maxOfMax, 0, sizeof(maxOfMax));
|
|
|
|
|
bitSet.forEachInRange(
|
|
|
|
|
[&](int c) {
|
|
|
|
|
children[c] = other.children[other.index[c]];
|
|
|
|
|
assert(children[c].child->parent == &other);
|
|
|
|
|
children[c].child->parent = this;
|
|
|
|
|
childMaxVersion[c] = other.childMaxVersion[other.index[c]];
|
|
|
|
|
assert(children[c]->parent == &other);
|
|
|
|
|
children[c]->parent = this;
|
|
|
|
|
maxOfMax[c >> Node256::kMaxOfMaxShift] = std::max(
|
|
|
|
|
maxOfMax[c >> Node256::kMaxOfMaxShift], childMaxVersion[c]);
|
|
|
|
|
},
|
|
|
|
|
0, 256);
|
|
|
|
|
memcpy(partialKey(), &other + 1, partialKeyLen);
|
|
|
|
@@ -473,14 +513,17 @@ inline void Node256::copyChildrenAndKeyFrom(const Node256 &other) {
|
|
|
|
|
memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin,
|
|
|
|
|
kNodeCopySize);
|
|
|
|
|
memset(children, 0, sizeof(children));
|
|
|
|
|
memset(childMaxVersion, 0, sizeof(childMaxVersion));
|
|
|
|
|
bitSet = other.bitSet;
|
|
|
|
|
bitSet.forEachInRange(
|
|
|
|
|
[&](int c) {
|
|
|
|
|
children[c] = other.children[c];
|
|
|
|
|
assert(children[c].child->parent == &other);
|
|
|
|
|
children[c].child->parent = this;
|
|
|
|
|
childMaxVersion[c] = other.childMaxVersion[c];
|
|
|
|
|
assert(children[c]->parent == &other);
|
|
|
|
|
children[c]->parent = this;
|
|
|
|
|
},
|
|
|
|
|
0, 256);
|
|
|
|
|
memcpy(maxOfMax, other.maxOfMax, sizeof(maxOfMax));
|
|
|
|
|
memcpy(partialKey(), &other + 1, partialKeyLen);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -532,7 +575,7 @@ template <class T> struct BoundedFreeListAllocator {
|
|
|
|
|
static_assert(std::derived_from<T, Node>);
|
|
|
|
|
static_assert(std::is_trivial_v<T>);
|
|
|
|
|
|
|
|
|
|
T *allocate(int partialKeyCapacity) {
|
|
|
|
|
T *allocate_helper(int partialKeyCapacity) {
|
|
|
|
|
if (freeList != nullptr) {
|
|
|
|
|
T *n = (T *)freeList;
|
|
|
|
|
VALGRIND_MAKE_MEM_DEFINED(freeList, sizeof(freeList));
|
|
|
|
@@ -560,6 +603,18 @@ template <class T> struct BoundedFreeListAllocator {
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
T *allocate(int partialKeyCapacity) {
|
|
|
|
|
T *result = allocate_helper(partialKeyCapacity);
|
|
|
|
|
if constexpr (!std::is_same_v<T, Node0>) {
|
|
|
|
|
memset(result->children, 0, sizeof(result->children));
|
|
|
|
|
memset(result->childMaxVersion, 0, sizeof(result->childMaxVersion));
|
|
|
|
|
}
|
|
|
|
|
if constexpr (std::is_same_v<T, Node48> || std::is_same_v<T, Node256>) {
|
|
|
|
|
memset(result->maxOfMax, 0, sizeof(result->maxOfMax));
|
|
|
|
|
}
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void release(T *p) {
|
|
|
|
|
if (freeListBytes >= kFreeListMaxMemory) {
|
|
|
|
|
removeNode(p);
|
|
|
|
@@ -711,29 +766,30 @@ Node *&getChildExists(Node *self, uint8_t index) {
|
|
|
|
|
__builtin_unreachable(); // GCOVR_EXCL_LINE
|
|
|
|
|
case Type_Node3: {
|
|
|
|
|
auto *self3 = static_cast<Node3 *>(self);
|
|
|
|
|
return self3->children[getNodeIndex(self3, index)].child;
|
|
|
|
|
return self3->children[getNodeIndex(self3, index)];
|
|
|
|
|
}
|
|
|
|
|
case Type_Node16: {
|
|
|
|
|
auto *self16 = static_cast<Node16 *>(self);
|
|
|
|
|
return self16->children[getNodeIndex(self16, index)].child;
|
|
|
|
|
return self16->children[getNodeIndex(self16, index)];
|
|
|
|
|
}
|
|
|
|
|
case Type_Node48: {
|
|
|
|
|
auto *self48 = static_cast<Node48 *>(self);
|
|
|
|
|
assert(self48->bitSet.test(index));
|
|
|
|
|
return self48->children[self48->index[index]].child;
|
|
|
|
|
return self48->children[self48->index[index]];
|
|
|
|
|
}
|
|
|
|
|
case Type_Node256: {
|
|
|
|
|
auto *self256 = static_cast<Node256 *>(self);
|
|
|
|
|
assert(self256->bitSet.test(index));
|
|
|
|
|
return self256->children[index].child;
|
|
|
|
|
return self256->children[index];
|
|
|
|
|
}
|
|
|
|
|
default: // GCOVR_EXCL_LINE
|
|
|
|
|
__builtin_unreachable(); // GCOVR_EXCL_LINE
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Precondition - an entry for index must exist in the node
|
|
|
|
|
int64_t &maxVersion(Node *n, ConflictSet::Impl *);
|
|
|
|
|
int64_t maxVersion(Node *n, ConflictSet::Impl *);
|
|
|
|
|
|
|
|
|
|
void setMaxVersion(Node *n, ConflictSet::Impl *, int64_t maxVersion);
|
|
|
|
|
|
|
|
|
|
Node *&getInTree(Node *n, ConflictSet::Impl *);
|
|
|
|
|
|
|
|
|
@@ -744,21 +800,21 @@ Node *getChild(Node *self, uint8_t index) {
|
|
|
|
|
case Type_Node3: {
|
|
|
|
|
auto *self3 = static_cast<Node3 *>(self);
|
|
|
|
|
int i = getNodeIndex(self3, index);
|
|
|
|
|
return i < 0 ? nullptr : self3->children[i].child;
|
|
|
|
|
return i < 0 ? nullptr : self3->children[i];
|
|
|
|
|
}
|
|
|
|
|
case Type_Node16: {
|
|
|
|
|
auto *self16 = static_cast<Node16 *>(self);
|
|
|
|
|
int i = getNodeIndex(self16, index);
|
|
|
|
|
return i < 0 ? nullptr : self16->children[i].child;
|
|
|
|
|
return i < 0 ? nullptr : self16->children[i];
|
|
|
|
|
}
|
|
|
|
|
case Type_Node48: {
|
|
|
|
|
auto *self48 = static_cast<Node48 *>(self);
|
|
|
|
|
int i = self48->index[index];
|
|
|
|
|
return i < 0 ? nullptr : self48->children[i].child;
|
|
|
|
|
return i < 0 ? nullptr : self48->children[i];
|
|
|
|
|
}
|
|
|
|
|
case Type_Node256: {
|
|
|
|
|
auto *self256 = static_cast<Node256 *>(self);
|
|
|
|
|
return self256->children[index].child;
|
|
|
|
|
return self256->children[index];
|
|
|
|
|
}
|
|
|
|
|
default: // GCOVR_EXCL_LINE
|
|
|
|
|
__builtin_unreachable(); // GCOVR_EXCL_LINE
|
|
|
|
@@ -874,26 +930,26 @@ Node *&getOrCreateChild(Node *&self, uint8_t index,
|
|
|
|
|
auto *self3 = static_cast<Node3 *>(self);
|
|
|
|
|
int i = getNodeIndex(self3, index);
|
|
|
|
|
if (i >= 0) {
|
|
|
|
|
return self3->children[i].child;
|
|
|
|
|
return self3->children[i];
|
|
|
|
|
}
|
|
|
|
|
} break;
|
|
|
|
|
case Type_Node16: {
|
|
|
|
|
auto *self16 = static_cast<Node16 *>(self);
|
|
|
|
|
int i = getNodeIndex(self16, index);
|
|
|
|
|
if (i >= 0) {
|
|
|
|
|
return self16->children[i].child;
|
|
|
|
|
return self16->children[i];
|
|
|
|
|
}
|
|
|
|
|
} break;
|
|
|
|
|
case Type_Node48: {
|
|
|
|
|
auto *self48 = static_cast<Node48 *>(self);
|
|
|
|
|
int secondIndex = self48->index[index];
|
|
|
|
|
if (secondIndex >= 0) {
|
|
|
|
|
return self48->children[secondIndex].child;
|
|
|
|
|
return self48->children[secondIndex];
|
|
|
|
|
}
|
|
|
|
|
} break;
|
|
|
|
|
case Type_Node256: {
|
|
|
|
|
auto *self256 = static_cast<Node256 *>(self);
|
|
|
|
|
if (auto &result = self256->children[index].child; result != nullptr) {
|
|
|
|
|
if (auto &result = self256->children[index]; result != nullptr) {
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
} break;
|
|
|
|
@@ -931,12 +987,16 @@ Node *&getOrCreateChild(Node *&self, uint8_t index,
|
|
|
|
|
memmove(self3->index + i + 1, self3->index + i,
|
|
|
|
|
self->numChildren - (i + 1));
|
|
|
|
|
memmove(self3->children + i + 1, self3->children + i,
|
|
|
|
|
(self->numChildren - (i + 1)) * sizeof(Child));
|
|
|
|
|
(self->numChildren - (i + 1)) *
|
|
|
|
|
sizeof(self3->children[0])); // NOLINT
|
|
|
|
|
memmove(self3->childMaxVersion + i + 1, self3->childMaxVersion + i,
|
|
|
|
|
(self->numChildren - (i + 1)) *
|
|
|
|
|
sizeof(self3->childMaxVersion[0]));
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
self3->index[i] = index;
|
|
|
|
|
auto &result = self3->children[i].child;
|
|
|
|
|
auto &result = self3->children[i];
|
|
|
|
|
result = nullptr;
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
@@ -968,7 +1028,10 @@ Node *&getOrCreateChild(Node *&self, uint8_t index,
|
|
|
|
|
memmove(self16->index + i + 1, self16->index + i,
|
|
|
|
|
self->numChildren - (i + 1));
|
|
|
|
|
memmove(self16->children + i + 1, self16->children + i,
|
|
|
|
|
(self->numChildren - (i + 1)) * sizeof(Child));
|
|
|
|
|
(self->numChildren - (i + 1)) * sizeof(self16->children[0]));
|
|
|
|
|
memmove(self16->childMaxVersion + i + 1, self16->childMaxVersion + i,
|
|
|
|
|
(self->numChildren - (i + 1)) *
|
|
|
|
|
sizeof(self16->childMaxVersion[0]));
|
|
|
|
|
}
|
|
|
|
|
#elif defined(HAS_ARM_NEON)
|
|
|
|
|
uint8x16_t indices;
|
|
|
|
@@ -988,7 +1051,11 @@ Node *&getOrCreateChild(Node *&self, uint8_t index,
|
|
|
|
|
memmove(self16->index + i + 1, self16->index + i,
|
|
|
|
|
self->numChildren - (i + 1));
|
|
|
|
|
memmove(self16->children + i + 1, self16->children + i,
|
|
|
|
|
(self->numChildren - (i + 1)) * sizeof(Child));
|
|
|
|
|
(self->numChildren - (i + 1)) *
|
|
|
|
|
sizeof(self16->children[0])); // NOLINT
|
|
|
|
|
memmove(self16->childMaxVersion + i + 1, self16->childMaxVersion + i,
|
|
|
|
|
(self->numChildren - (i + 1)) *
|
|
|
|
|
sizeof(self16->childMaxVersion[0]));
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
int i = 0;
|
|
|
|
@@ -997,13 +1064,16 @@ Node *&getOrCreateChild(Node *&self, uint8_t index,
|
|
|
|
|
memmove(self16->index + i + 1, self16->index + i,
|
|
|
|
|
self->numChildren - (i + 1));
|
|
|
|
|
memmove(self16->children + i + 1, self16->children + i,
|
|
|
|
|
(self->numChildren - (i + 1)) * sizeof(Child));
|
|
|
|
|
(self->numChildren - (i + 1)) * sizeof(self16->children[0]));
|
|
|
|
|
memmove(self16->childMaxVersion + i + 1, self16->childMaxVersion + i,
|
|
|
|
|
(self->numChildren - (i + 1)) *
|
|
|
|
|
sizeof(self16->childMaxVersion[0]));
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
self16->index[i] = index;
|
|
|
|
|
auto &result = self16->children[i].child;
|
|
|
|
|
auto &result = self16->children[i];
|
|
|
|
|
result = nullptr;
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
@@ -1025,7 +1095,8 @@ Node *&getOrCreateChild(Node *&self, uint8_t index,
|
|
|
|
|
assert(self48->nextFree < 48);
|
|
|
|
|
int nextFree = self48->nextFree++;
|
|
|
|
|
self48->index[index] = nextFree;
|
|
|
|
|
auto &result = self48->children[nextFree].child;
|
|
|
|
|
self48->reverseIndex[nextFree] = index;
|
|
|
|
|
auto &result = self48->children[nextFree];
|
|
|
|
|
result = nullptr;
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
@@ -1035,7 +1106,7 @@ Node *&getOrCreateChild(Node *&self, uint8_t index,
|
|
|
|
|
auto *self256 = static_cast<Node256 *>(self);
|
|
|
|
|
++self->numChildren;
|
|
|
|
|
self256->bitSet.set(index);
|
|
|
|
|
return self256->children[index].child;
|
|
|
|
|
return self256->children[index];
|
|
|
|
|
}
|
|
|
|
|
default: // GCOVR_EXCL_LINE
|
|
|
|
|
__builtin_unreachable(); // GCOVR_EXCL_LINE
|
|
|
|
@@ -1178,7 +1249,7 @@ void maybeDownsize(Node *self, NodeAllocators *allocators,
|
|
|
|
|
getInTree(self, impl) = newSelf;
|
|
|
|
|
allocators->node3.release(self3);
|
|
|
|
|
} else if (self->numChildren == 1 && !self->entryPresent) {
|
|
|
|
|
auto *child = self3->children[0].child;
|
|
|
|
|
auto *child = self3->children[0];
|
|
|
|
|
int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen;
|
|
|
|
|
|
|
|
|
|
if (minCapacity > child->getCapacity()) {
|
|
|
|
@@ -1210,7 +1281,7 @@ void maybeDownsize(Node *self, NodeAllocators *allocators,
|
|
|
|
|
|
|
|
|
|
// Max versions are stored in the parent, so we need to update it now
|
|
|
|
|
// that we have a new parent.
|
|
|
|
|
maxVersion(child, impl) = childMaxVersion;
|
|
|
|
|
setMaxVersion(child, impl, childMaxVersion);
|
|
|
|
|
|
|
|
|
|
getInTree(self, impl) = child;
|
|
|
|
|
allocators->node3.release(self3);
|
|
|
|
@@ -1291,7 +1362,11 @@ Node *erase(Node *self, NodeAllocators *allocators, ConflictSet::Impl *impl,
|
|
|
|
|
sizeof(parent3->index[0]) *
|
|
|
|
|
(parent->numChildren - (nodeIndex + 1)));
|
|
|
|
|
memmove(parent3->children + nodeIndex, parent3->children + nodeIndex + 1,
|
|
|
|
|
sizeof(parent3->children[0]) *
|
|
|
|
|
sizeof(parent3->children[0]) * // NOLINT
|
|
|
|
|
(parent->numChildren - (nodeIndex + 1)));
|
|
|
|
|
memmove(parent3->childMaxVersion + nodeIndex,
|
|
|
|
|
parent3->childMaxVersion + nodeIndex + 1,
|
|
|
|
|
sizeof(parent3->childMaxVersion[0]) *
|
|
|
|
|
(parent->numChildren - (nodeIndex + 1)));
|
|
|
|
|
|
|
|
|
|
--parent->numChildren;
|
|
|
|
@@ -1305,7 +1380,11 @@ Node *erase(Node *self, NodeAllocators *allocators, ConflictSet::Impl *impl,
|
|
|
|
|
sizeof(parent16->index[0]) *
|
|
|
|
|
(parent->numChildren - (nodeIndex + 1)));
|
|
|
|
|
memmove(parent16->children + nodeIndex, parent16->children + nodeIndex + 1,
|
|
|
|
|
sizeof(parent16->children[0]) *
|
|
|
|
|
sizeof(parent16->children[0]) * // NOLINT
|
|
|
|
|
(parent->numChildren - (nodeIndex + 1)));
|
|
|
|
|
memmove(parent16->childMaxVersion + nodeIndex,
|
|
|
|
|
parent16->childMaxVersion + nodeIndex + 1,
|
|
|
|
|
sizeof(parent16->childMaxVersion[0]) *
|
|
|
|
|
(parent->numChildren - (nodeIndex + 1)));
|
|
|
|
|
|
|
|
|
|
--parent->numChildren;
|
|
|
|
@@ -1324,9 +1403,18 @@ Node *erase(Node *self, NodeAllocators *allocators, ConflictSet::Impl *impl,
|
|
|
|
|
if (toRemoveChildrenIndex != lastChildrenIndex) {
|
|
|
|
|
parent48->children[toRemoveChildrenIndex] =
|
|
|
|
|
parent48->children[lastChildrenIndex];
|
|
|
|
|
parent48->index[parent48->children[toRemoveChildrenIndex]
|
|
|
|
|
.child->parentsIndex] = toRemoveChildrenIndex;
|
|
|
|
|
parent48->childMaxVersion[toRemoveChildrenIndex] =
|
|
|
|
|
parent48->childMaxVersion[lastChildrenIndex];
|
|
|
|
|
parent48->maxOfMax[toRemoveChildrenIndex >> Node48::kMaxOfMaxShift] =
|
|
|
|
|
std::max(parent48->maxOfMax[toRemoveChildrenIndex >>
|
|
|
|
|
Node48::kMaxOfMaxShift],
|
|
|
|
|
parent48->childMaxVersion[toRemoveChildrenIndex]);
|
|
|
|
|
auto parentIndex =
|
|
|
|
|
parent48->children[toRemoveChildrenIndex]->parentsIndex;
|
|
|
|
|
parent48->index[parentIndex] = toRemoveChildrenIndex;
|
|
|
|
|
parent48->reverseIndex[toRemoveChildrenIndex] = parentIndex;
|
|
|
|
|
}
|
|
|
|
|
parent48->childMaxVersion[lastChildrenIndex] = 0;
|
|
|
|
|
|
|
|
|
|
--parent->numChildren;
|
|
|
|
|
|
|
|
|
@@ -1336,7 +1424,7 @@ Node *erase(Node *self, NodeAllocators *allocators, ConflictSet::Impl *impl,
|
|
|
|
|
case Type_Node256: {
|
|
|
|
|
auto *parent256 = static_cast<Node256 *>(parent);
|
|
|
|
|
parent256->bitSet.reset(parentsIndex);
|
|
|
|
|
parent256->children[parentsIndex].child = nullptr;
|
|
|
|
|
parent256->children[parentsIndex] = nullptr;
|
|
|
|
|
|
|
|
|
|
--parent->numChildren;
|
|
|
|
|
|
|
|
|
@@ -1621,75 +1709,151 @@ downLeftSpine:
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Return the max version among all keys starting with the search path of n +
|
|
|
|
|
// [child], where child in (begin, end). Does not account for the range version
|
|
|
|
|
// of firstGt(searchpath(n) + [end - 1])
|
|
|
|
|
int64_t maxBetweenExclusive(Node *n, int begin, int end) {
|
|
|
|
|
// Returns true if all in-bounds vs are <= readVersion
|
|
|
|
|
bool scan16(const int64_t *vs, const uint8_t *is, int begin, int end,
|
|
|
|
|
int64_t readVersion) {
|
|
|
|
|
|
|
|
|
|
#ifdef HAS_ARM_NEON
|
|
|
|
|
|
|
|
|
|
assert(end - begin < 256);
|
|
|
|
|
uint8x16_t indices;
|
|
|
|
|
memcpy(&indices, is, 16);
|
|
|
|
|
// 0xff for each in bounds
|
|
|
|
|
auto results =
|
|
|
|
|
vcltq_u8(vsubq_u8(indices, vdupq_n_u8(begin)), vdupq_n_u8(end - begin));
|
|
|
|
|
// 0xf for each 0xff
|
|
|
|
|
uint64_t mask = vget_lane_u64(
|
|
|
|
|
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(results), 4)), 0);
|
|
|
|
|
|
|
|
|
|
uint64_t compared = 0;
|
|
|
|
|
for (int i = 0; i < 16; ++i) {
|
|
|
|
|
compared |= (uint64_t(vs[i] > readVersion) << (i << 2));
|
|
|
|
|
}
|
|
|
|
|
return !(compared & mask);
|
|
|
|
|
|
|
|
|
|
#else
|
|
|
|
|
|
|
|
|
|
const unsigned shiftUpperBound = end - begin;
|
|
|
|
|
const unsigned shiftAmount = begin;
|
|
|
|
|
auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
|
|
|
|
|
|
|
|
|
|
uint32_t compared = 0;
|
|
|
|
|
for (int i = 0; i < 16; ++i) {
|
|
|
|
|
compared |= ((vs[i] > readVersion) << i);
|
|
|
|
|
}
|
|
|
|
|
uint32_t mask = 0;
|
|
|
|
|
for (int i = 0; i < 16; ++i) {
|
|
|
|
|
mask |= inBounds(is[i]) << i;
|
|
|
|
|
}
|
|
|
|
|
return !(compared & mask);
|
|
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Return whether or not the max version among all keys starting with the search
|
|
|
|
|
// path of n + [child], where child in (begin, end) is <= readVersion. Does not
|
|
|
|
|
// account for the range version of firstGt(searchpath(n) + [end - 1])
|
|
|
|
|
bool checkMaxBetweenExclusive(Node *n, int begin, int end,
|
|
|
|
|
int64_t readVersion) {
|
|
|
|
|
assume(-1 <= begin);
|
|
|
|
|
assume(begin <= 256);
|
|
|
|
|
assume(-1 <= end);
|
|
|
|
|
assume(end <= 256);
|
|
|
|
|
assume(begin < end);
|
|
|
|
|
int64_t result = std::numeric_limits<int64_t>::lowest();
|
|
|
|
|
|
|
|
|
|
assert(!(begin == -1 && end == 256));
|
|
|
|
|
|
|
|
|
|
{
|
|
|
|
|
int c = getChildGeq(n, begin + 1);
|
|
|
|
|
if (c >= 0 && c < end) {
|
|
|
|
|
auto *child = getChildExists(n, c);
|
|
|
|
|
if (child->entryPresent) {
|
|
|
|
|
result = std::max(result, child->entry.rangeVersion);
|
|
|
|
|
if (!(child->entry.rangeVersion <= readVersion)) {
|
|
|
|
|
return false;
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
begin = c;
|
|
|
|
|
} else {
|
|
|
|
|
return result;
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// [begin, end) is now the half-open interval of children we're interested in.
|
|
|
|
|
|
|
|
|
|
const unsigned shiftUpperBound = end - begin;
|
|
|
|
|
const unsigned shiftAmount = begin;
|
|
|
|
|
auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
|
|
|
|
|
|
|
|
|
|
switch (n->getType()) {
|
|
|
|
|
case Type_Node0: // GCOVR_EXCL_LINE
|
|
|
|
|
// We would have returned above, after not finding a child
|
|
|
|
|
__builtin_unreachable(); // GCOVR_EXCL_LINE
|
|
|
|
|
case Type_Node3: {
|
|
|
|
|
auto *self = static_cast<Node3 *>(n);
|
|
|
|
|
for (int i = 0; i < self->numChildren && self->index[i] < end; ++i) {
|
|
|
|
|
if (begin <= self->index[i]) {
|
|
|
|
|
result = std::max(result, self->children[i].childMaxVersion);
|
|
|
|
|
}
|
|
|
|
|
bool result = true;
|
|
|
|
|
for (int i = 0; i < 3; ++i) {
|
|
|
|
|
result &= !((self->childMaxVersion[i] > readVersion) &
|
|
|
|
|
inBounds(self->index[i]));
|
|
|
|
|
}
|
|
|
|
|
return result;
|
|
|
|
|
} break;
|
|
|
|
|
case Type_Node16: {
|
|
|
|
|
auto *self = static_cast<Node16 *>(n);
|
|
|
|
|
for (int i = 0; i < self->numChildren && self->index[i] < end; ++i) {
|
|
|
|
|
if (begin <= self->index[i]) {
|
|
|
|
|
result = std::max(result, self->children[i].childMaxVersion);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return scan16(self->childMaxVersion, self->index, begin, end, readVersion);
|
|
|
|
|
|
|
|
|
|
} break;
|
|
|
|
|
case Type_Node48: {
|
|
|
|
|
auto *self = static_cast<Node48 *>(n);
|
|
|
|
|
self->bitSet.forEachInRange(
|
|
|
|
|
[&](int i) {
|
|
|
|
|
result =
|
|
|
|
|
std::max(result, self->children[self->index[i]].childMaxVersion);
|
|
|
|
|
},
|
|
|
|
|
begin, end);
|
|
|
|
|
break;
|
|
|
|
|
// Check all pages
|
|
|
|
|
static_assert(Node48::kMaxOfMaxPageSize == 16);
|
|
|
|
|
for (int i = 0; i < Node48::kMaxOfMaxTotalPages; ++i) {
|
|
|
|
|
if (self->maxOfMax[i] > readVersion) {
|
|
|
|
|
if (!scan16(self->childMaxVersion + (i << Node48::kMaxOfMaxShift),
|
|
|
|
|
self->reverseIndex + (i << Node48::kMaxOfMaxShift), begin,
|
|
|
|
|
end, readVersion)) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
case Type_Node256: {
|
|
|
|
|
auto *self = static_cast<Node256 *>(n);
|
|
|
|
|
self->bitSet.forEachInRange(
|
|
|
|
|
[&](int i) {
|
|
|
|
|
result = std::max(result, self->children[i].childMaxVersion);
|
|
|
|
|
},
|
|
|
|
|
begin, end);
|
|
|
|
|
break;
|
|
|
|
|
// Check the first page
|
|
|
|
|
if (self->maxOfMax[begin >> Node256::kMaxOfMaxShift] > readVersion) {
|
|
|
|
|
bool result = true;
|
|
|
|
|
for (int i = 0; i < Node256::kMaxOfMaxPageSize; ++i) {
|
|
|
|
|
int j = (begin & ~(Node256::kMaxOfMaxPageSize - 1)) + i;
|
|
|
|
|
result &= !((self->childMaxVersion[j] > readVersion) & inBounds(j));
|
|
|
|
|
}
|
|
|
|
|
if (!result) {
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// Check the last page
|
|
|
|
|
if (end >= 1 &&
|
|
|
|
|
self->maxOfMax[(end - 1) >> Node256::kMaxOfMaxShift] > readVersion) {
|
|
|
|
|
bool result = true;
|
|
|
|
|
for (int i = 0; i < Node256::kMaxOfMaxPageSize; ++i) {
|
|
|
|
|
int j = ((end - 1) & ~(Node256::kMaxOfMaxPageSize - 1)) + i;
|
|
|
|
|
result &= !((self->childMaxVersion[j] > readVersion) & inBounds(j));
|
|
|
|
|
}
|
|
|
|
|
if (!result) {
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// Check inner pages
|
|
|
|
|
bool result = true;
|
|
|
|
|
for (int i = (begin >> Node256::kMaxOfMaxShift) + 1;
|
|
|
|
|
i < ((end - 1) >> Node256::kMaxOfMaxShift); ++i) {
|
|
|
|
|
result &= self->maxOfMax[i] <= readVersion;
|
|
|
|
|
}
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
default: // GCOVR_EXCL_LINE
|
|
|
|
|
__builtin_unreachable(); // GCOVR_EXCL_LINE
|
|
|
|
|
}
|
|
|
|
|
#if DEBUG_VERBOSE && !defined(NDEBUG)
|
|
|
|
|
fprintf(stderr, "At `%s', max version in (%02x, %02x) is %" PRId64 "\n",
|
|
|
|
|
getSearchPathPrintable(n).c_str(), begin, end, result);
|
|
|
|
|
#endif
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Vector<uint8_t> getSearchPath(Arena &arena, Node *n) {
|
|
|
|
@@ -1722,7 +1886,7 @@ bool checkRangeStartsWith(Node *n, std::span<const uint8_t> key, int begin,
|
|
|
|
|
#endif
|
|
|
|
|
auto remaining = key;
|
|
|
|
|
if (remaining.size() == 0) {
|
|
|
|
|
return maxBetweenExclusive(n, begin, end) <= readVersion;
|
|
|
|
|
return checkMaxBetweenExclusive(n, begin, end, readVersion);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
auto *child = getChild(n, remaining[0]);
|
|
|
|
@@ -1821,7 +1985,7 @@ struct CheckRangeLeftSide {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (searchPathLen >= prefixLen) {
|
|
|
|
|
if (maxBetweenExclusive(n, remaining[0], 256) > readVersion) {
|
|
|
|
|
if (!checkMaxBetweenExclusive(n, remaining[0], 256, readVersion)) {
|
|
|
|
|
ok = false;
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
@@ -1963,7 +2127,7 @@ struct CheckRangeRightSide {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (maxBetweenExclusive(n, -1, remaining[0]) > readVersion) {
|
|
|
|
|
if (!checkMaxBetweenExclusive(n, -1, remaining[0], readVersion)) {
|
|
|
|
|
ok = false;
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
@@ -2174,7 +2338,7 @@ template <bool kBegin>
|
|
|
|
|
allocators) = old;
|
|
|
|
|
old->parent = *self;
|
|
|
|
|
old->parentsIndex = old->partialKey()[partialKeyIndex];
|
|
|
|
|
maxVersion(old, impl) = oldMaxVersion;
|
|
|
|
|
setMaxVersion(old, impl, oldMaxVersion);
|
|
|
|
|
|
|
|
|
|
memmove(old->partialKey(), old->partialKey() + partialKeyIndex + 1,
|
|
|
|
|
old->partialKeyLen - (partialKeyIndex + 1));
|
|
|
|
@@ -2198,9 +2362,8 @@ template <bool kBegin>
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if constexpr (kBegin) {
|
|
|
|
|
auto &m = maxVersion(*self, impl);
|
|
|
|
|
assert(writeVersion >= m);
|
|
|
|
|
m = writeVersion;
|
|
|
|
|
assert(maxVersion(*self, impl) <= writeVersion);
|
|
|
|
|
setMaxVersion(*self, impl, writeVersion);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (key.size() == 0) {
|
|
|
|
@@ -2208,9 +2371,8 @@ template <bool kBegin>
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if constexpr (!kBegin) {
|
|
|
|
|
auto &m = maxVersion(*self, impl);
|
|
|
|
|
assert(writeVersion >= m);
|
|
|
|
|
m = writeVersion;
|
|
|
|
|
assert(maxVersion(*self, impl) <= writeVersion);
|
|
|
|
|
setMaxVersion(*self, impl, writeVersion);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
auto &child = getOrCreateChild(*self, key.front(), allocators);
|
|
|
|
@@ -2221,8 +2383,7 @@ template <bool kBegin>
|
|
|
|
|
child->partialKeyLen = 0;
|
|
|
|
|
child->parent = *self;
|
|
|
|
|
child->parentsIndex = key.front();
|
|
|
|
|
maxVersion(child, impl) =
|
|
|
|
|
kBegin ? writeVersion : std::numeric_limits<int64_t>::lowest();
|
|
|
|
|
setMaxVersion(child, impl, kBegin ? writeVersion : 0);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
self = &child;
|
|
|
|
@@ -2267,7 +2428,7 @@ void addPointWrite(Node *&root, int64_t oldestVersion,
|
|
|
|
|
n->entryPresent = true;
|
|
|
|
|
|
|
|
|
|
n->entry.pointVersion = writeVersion;
|
|
|
|
|
maxVersion(n, impl) = writeVersion;
|
|
|
|
|
setMaxVersion(n, impl, writeVersion);
|
|
|
|
|
n->entry.rangeVersion =
|
|
|
|
|
p != nullptr ? p->entry.rangeVersion : oldestVersion;
|
|
|
|
|
} else {
|
|
|
|
@@ -2308,9 +2469,8 @@ void addWriteRange(Node *&root, int64_t oldestVersion,
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
auto &m = maxVersion(n, impl);
|
|
|
|
|
assert(writeVersion >= m);
|
|
|
|
|
m = writeVersion;
|
|
|
|
|
assert(maxVersion(n, impl) <= writeVersion);
|
|
|
|
|
setMaxVersion(n, impl, writeVersion);
|
|
|
|
|
|
|
|
|
|
remaining = remaining.subspan(n->partialKeyLen + 1,
|
|
|
|
|
remaining.size() - (n->partialKeyLen + 1));
|
|
|
|
@@ -2337,11 +2497,10 @@ void addWriteRange(Node *&root, int64_t oldestVersion,
|
|
|
|
|
beginNode->entry.rangeVersion =
|
|
|
|
|
p != nullptr ? p->entry.rangeVersion : oldestVersion;
|
|
|
|
|
beginNode->entry.pointVersion = writeVersion;
|
|
|
|
|
maxVersion(beginNode, impl) = writeVersion;
|
|
|
|
|
assert(maxVersion(beginNode, impl) <= writeVersion);
|
|
|
|
|
setMaxVersion(beginNode, impl, writeVersion);
|
|
|
|
|
}
|
|
|
|
|
auto &m = maxVersion(beginNode, impl);
|
|
|
|
|
assert(writeVersion >= m);
|
|
|
|
|
m = writeVersion;
|
|
|
|
|
setMaxVersion(beginNode, impl, writeVersion);
|
|
|
|
|
assert(writeVersion >= beginNode->entry.pointVersion);
|
|
|
|
|
beginNode->entry.pointVersion = writeVersion;
|
|
|
|
|
|
|
|
|
@@ -2356,8 +2515,8 @@ void addWriteRange(Node *&root, int64_t oldestVersion,
|
|
|
|
|
auto *p = nextLogical(endNode);
|
|
|
|
|
endNode->entry.pointVersion =
|
|
|
|
|
p != nullptr ? p->entry.rangeVersion : oldestVersion;
|
|
|
|
|
auto &m = maxVersion(endNode, impl);
|
|
|
|
|
m = std::max(m, endNode->entry.pointVersion);
|
|
|
|
|
auto m = maxVersion(endNode, impl);
|
|
|
|
|
setMaxVersion(endNode, impl, std::max(m, endNode->entry.pointVersion));
|
|
|
|
|
}
|
|
|
|
|
endNode->entry.rangeVersion = writeVersion;
|
|
|
|
|
|
|
|
|
@@ -2557,8 +2716,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
|
|
|
|
|
int64_t totalBytes = 0;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
// Precondition - an entry for index must exist in the node
|
|
|
|
|
int64_t &maxVersion(Node *n, ConflictSet::Impl *impl) {
|
|
|
|
|
int64_t maxVersion(Node *n, ConflictSet::Impl *impl) {
|
|
|
|
|
int index = n->parentsIndex;
|
|
|
|
|
n = n->parent;
|
|
|
|
|
if (n == nullptr) {
|
|
|
|
@@ -2570,22 +2728,66 @@ int64_t &maxVersion(Node *n, ConflictSet::Impl *impl) {
|
|
|
|
|
case Type_Node3: {
|
|
|
|
|
auto *n3 = static_cast<Node3 *>(n);
|
|
|
|
|
int i = getNodeIndex(n3, index);
|
|
|
|
|
return n3->children[i].childMaxVersion;
|
|
|
|
|
return n3->childMaxVersion[i];
|
|
|
|
|
}
|
|
|
|
|
case Type_Node16: {
|
|
|
|
|
auto *n16 = static_cast<Node16 *>(n);
|
|
|
|
|
int i = getNodeIndex(n16, index);
|
|
|
|
|
return n16->children[i].childMaxVersion;
|
|
|
|
|
return n16->childMaxVersion[i];
|
|
|
|
|
}
|
|
|
|
|
case Type_Node48: {
|
|
|
|
|
auto *n48 = static_cast<Node48 *>(n);
|
|
|
|
|
assert(n48->bitSet.test(index));
|
|
|
|
|
return n48->children[n48->index[index]].childMaxVersion;
|
|
|
|
|
return n48->childMaxVersion[n48->index[index]];
|
|
|
|
|
}
|
|
|
|
|
case Type_Node256: {
|
|
|
|
|
auto *n256 = static_cast<Node256 *>(n);
|
|
|
|
|
assert(n256->bitSet.test(index));
|
|
|
|
|
return n256->children[index].childMaxVersion;
|
|
|
|
|
return n256->childMaxVersion[index];
|
|
|
|
|
}
|
|
|
|
|
default: // GCOVR_EXCL_LINE
|
|
|
|
|
__builtin_unreachable(); // GCOVR_EXCL_LINE
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void setMaxVersion(Node *n, ConflictSet::Impl *impl, int64_t newMax) {
|
|
|
|
|
int index = n->parentsIndex;
|
|
|
|
|
n = n->parent;
|
|
|
|
|
if (n == nullptr) {
|
|
|
|
|
impl->rootMaxVersion = newMax;
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
switch (n->getType()) {
|
|
|
|
|
case Type_Node0: // GCOVR_EXCL_LINE
|
|
|
|
|
__builtin_unreachable(); // GCOVR_EXCL_LINE
|
|
|
|
|
case Type_Node3: {
|
|
|
|
|
auto *n3 = static_cast<Node3 *>(n);
|
|
|
|
|
int i = getNodeIndex(n3, index);
|
|
|
|
|
n3->childMaxVersion[i] = newMax;
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
case Type_Node16: {
|
|
|
|
|
auto *n16 = static_cast<Node16 *>(n);
|
|
|
|
|
int i = getNodeIndex(n16, index);
|
|
|
|
|
n16->childMaxVersion[i] = newMax;
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
case Type_Node48: {
|
|
|
|
|
auto *n48 = static_cast<Node48 *>(n);
|
|
|
|
|
assert(n48->bitSet.test(index));
|
|
|
|
|
int i = n48->index[index];
|
|
|
|
|
n48->childMaxVersion[i] = newMax;
|
|
|
|
|
n48->maxOfMax[i >> Node48::kMaxOfMaxShift] =
|
|
|
|
|
std::max(n48->maxOfMax[i >> Node48::kMaxOfMaxShift], newMax);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
case Type_Node256: {
|
|
|
|
|
auto *n256 = static_cast<Node256 *>(n);
|
|
|
|
|
assert(n256->bitSet.test(index));
|
|
|
|
|
n256->childMaxVersion[index] = newMax;
|
|
|
|
|
n256->maxOfMax[index >> Node256::kMaxOfMaxShift] =
|
|
|
|
|
std::max(n256->maxOfMax[index >> Node256::kMaxOfMaxShift], newMax);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
default: // GCOVR_EXCL_LINE
|
|
|
|
|
__builtin_unreachable(); // GCOVR_EXCL_LINE
|
|
|
|
@@ -2847,7 +3049,7 @@ Iterator firstGeq(Node *n, std::string_view key) {
|
|
|
|
|
[[maybe_unused]] int64_t checkMaxVersion(Node *root, Node *node,
|
|
|
|
|
int64_t oldestVersion, bool &success,
|
|
|
|
|
ConflictSet::Impl *impl) {
|
|
|
|
|
int64_t expected = std::numeric_limits<int64_t>::lowest();
|
|
|
|
|
int64_t expected = 0;
|
|
|
|
|
if (node->entryPresent) {
|
|
|
|
|
expected = std::max(expected, node->entry.pointVersion);
|
|
|
|
|
}
|
|
|
|
|