weaselab/conflict-set
2
0
Fork 0
Code Issues 4 Packages Releases 13 Activity

Mix type dispatch and interleaving for down left spine

Browse Source
This commit is contained in:
Andrew Noyes
2024-10-17 13:13:16 -07:00
parent 5cf04e9718
commit 10c2f06199
1 changed files with 105 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

178
ConflictSet.cpp
View File

@@ -1193,9 +1193,9 @@ ChildAndMaxVersion getChildAndMaxVersion(Node *self, uint8_t index) {
}
}
Node *getChildGeq(Node0 *, int) { return nullptr; }
TaggedNodePointer getChildGeq(Node0 *, int) { return nullptr; }
Node *getChildGeq(Node3 *n, int child) {
TaggedNodePointer getChildGeq(Node3 *n, int child) {
assume(n->numChildren >= 1);
assume(n->numChildren <= 3);
for (int i = 0; i < n->numChildren; ++i) {
@@ -1206,7 +1206,7 @@ Node *getChildGeq(Node3 *n, int child) {
return nullptr;
}
Node *getChildGeq(Node16 *self, int child) {
TaggedNodePointer getChildGeq(Node16 *self, int child) {
if (child > 255) {
return nullptr;
}
@@ -1249,14 +1249,14 @@ Node *getChildGeq(Node16 *self, int child) {
#endif
}
Node *getChildGeq(Node48 *self, int child) {
TaggedNodePointer getChildGeq(Node48 *self, int child) {
int c = self->bitSet.firstSetGeq(child);
if (c < 0) {
return nullptr;
}
return self->children[self->index[c]];
}
Node *getChildGeq(Node256 *self, int child) {
TaggedNodePointer getChildGeq(Node256 *self, int child) {
int c = self->bitSet.firstSetGeq(child);
if (c < 0) {
return nullptr;
@@ -1264,7 +1264,7 @@ Node *getChildGeq(Node256 *self, int child) {
return self->children[c];
}
Node *getChildGeq(Node *self, int child) {
TaggedNodePointer getChildGeq(Node *self, int child) {
switch (self->getType()) {
case Type_Node0:
return getChildGeq(static_cast<Node0 *>(self), child);
@@ -1317,26 +1317,26 @@ Node *getFirstChild(Node *self) {
}
// Precondition: self has a child
Node *getFirstChildExists(Node3 *self) {
TaggedNodePointer getFirstChildExists(Node3 *self) {
assert(self->numChildren > 0);
return self->children[0];
}
// Precondition: self has a child
Node *getFirstChildExists(Node16 *self) {
TaggedNodePointer getFirstChildExists(Node16 *self) {
assert(self->numChildren > 0);
return self->children[0];
}
// Precondition: self has a child
Node *getFirstChildExists(Node48 *self) {
TaggedNodePointer getFirstChildExists(Node48 *self) {
return self->children[self->index[self->bitSet.firstSetGeq(0)]];
}
// Precondition: self has a child
Node *getFirstChildExists(Node256 *self) {
TaggedNodePointer getFirstChildExists(Node256 *self) {
return self->children[self->bitSet.firstSetGeq(0)];
}
// Precondition: self has a child
Node *getFirstChildExists(Node *self) {
TaggedNodePointer getFirstChildExists(Node *self) {
// Only require that the node-specific overloads are covered
// GCOVR_EXCL_START
switch (self->getType()) {
@@ -2021,7 +2021,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl,
return result;
}
Node *nextSibling(Node *node) {
TaggedNodePointer nextSibling(Node *node) {
for (;;) {
if (node->parent == nullptr) {
return nullptr;
@@ -3097,13 +3097,25 @@ PRESERVE_NONE void complete(CheckJob *job, CheckContext *context) {
MUSTTAIL return keepGoing(job, context);
}
template <class NodeT>
PRESERVE_NONE void down_left_spine(CheckJob *job, CheckContext *context);
static Continuation downLeftSpineTable[] = {
down_left_spine<Node0>, down_left_spine<Node3>, down_left_spine<Node16>,
down_left_spine<Node48>, down_left_spine<Node256>};
template <class NodeT>
PRESERVE_NONE void down_left_spine(CheckJob *job, CheckContext *context) {
if (job->n->entryPresent) {
job->setResult(job->n->entry.rangeVersion <= job->readVersion);
assert(job->n->getType() == NodeT::kType);
NodeT *n = static_cast<NodeT *>(job->n);
if (n->entryPresent) {
job->setResult(n->entry.rangeVersion <= job->readVersion);
MUSTTAIL return complete(job, context);
}
job->n = getFirstChildExists(job->n);
auto child = getFirstChildExists(n);
job->n = child;
__builtin_prefetch(job->n);
job->continuation = downLeftSpineTable[child.getType()];
MUSTTAIL return keepGoing(job, context);
}
@@ -3136,7 +3148,7 @@ void begin(CheckJob *job, CheckContext *context) {
auto c = getChildGeq(job->n, job->begin[0]);
if (c != nullptr) {
job->n = c;
job->continuation = down_left_spine;
job->continuation = downLeftSpineTable[c.getType()];
__builtin_prefetch(job->n);
MUSTTAIL return keepGoing(job, context);
} else {
@@ -3163,15 +3175,15 @@ template <class NodeT> void iter(CheckJob *job, CheckContext *context) {
if (i < commonLen) [[unlikely]] {
auto c = n->partialKey()[i] <=> job->begin[i];
if (c > 0) {
job->continuation = down_left_spine;
MUSTTAIL return down_left_spine(job, context);
MUSTTAIL return down_left_spine<NodeT>(job, context);
} else {
job->n = nextSibling(n);
auto s = nextSibling(n);
job->n = s;
if (job->n == nullptr) {
job->setResult(true);
MUSTTAIL return complete(job, context);
}
job->continuation = down_left_spine;
job->continuation = downLeftSpineTable[s.getType()];
__builtin_prefetch(job->n);
MUSTTAIL return keepGoing(job, context);
}
@@ -3182,8 +3194,7 @@ template <class NodeT> void iter(CheckJob *job, CheckContext *context) {
} else if (n->partialKeyLen > int(job->begin.size())) [[unlikely]] {
// n is the first physical node greater than remaining, and there's no
// eq node
job->continuation = down_left_spine;
MUSTTAIL return down_left_spine(job, context);
MUSTTAIL return down_left_spine<NodeT>(job, context);
}
}
@@ -3200,8 +3211,9 @@ template <class NodeT> void iter(CheckJob *job, CheckContext *context) {
job->setResult(n->entry.pointVersion <= job->readVersion);
MUSTTAIL return complete(job, context);
}
job->n = getFirstChildExists(n);
job->continuation = down_left_spine;
auto c = getFirstChildExists(n);
job->n = c;
job->continuation = downLeftSpineTable[c.getType()];
__builtin_prefetch(job->n);
MUSTTAIL return keepGoing(job, context);
}
@@ -3213,16 +3225,17 @@ template <class NodeT> void iter(CheckJob *job, CheckContext *context) {
auto c = getChildGeq(n, job->begin[0]);
if (c != nullptr) {
job->n = c;
job->continuation = down_left_spine;
job->continuation = downLeftSpineTable[c.getType()];
__builtin_prefetch(job->n);
MUSTTAIL return keepGoing(job, context);
} else {
job->n = nextSibling(job->n);
auto c = nextSibling(job->n);
job->n = c;
if (job->n == nullptr) {
job->setResult(true);
MUSTTAIL return complete(job, context);
}
job->continuation = down_left_spine;
job->continuation = downLeftSpineTable[c->getType()];
__builtin_prefetch(job->n);
MUSTTAIL return keepGoing(job, context);
}
@@ -3260,7 +3273,7 @@ void begin(CheckJob *job, CheckContext *context) {
auto c = getChildGeq(job->n, job->begin[0]);
if (c != nullptr) {
job->n = c;
job->continuation = down_left_spine;
job->continuation = downLeftSpineTable[c.getType()];
__builtin_prefetch(job->n);
MUSTTAIL return keepGoing(job, context);
} else {
@@ -3287,15 +3300,15 @@ template <class NodeT> void iter(CheckJob *job, CheckContext *context) {
if (i < commonLen) [[unlikely]] {
auto c = n->partialKey()[i] <=> job->begin[i];
if (c > 0) {
job->continuation = down_left_spine;
MUSTTAIL return down_left_spine(job, context);
MUSTTAIL return down_left_spine<NodeT>(job, context);
} else {
job->n = nextSibling(n);
auto c = nextSibling(n);
job->n = c;
if (job->n == nullptr) {
job->setResult(true);
MUSTTAIL return complete(job, context);
}
job->continuation = down_left_spine;
job->continuation = downLeftSpineTable[c.getType()];
__builtin_prefetch(job->n);
MUSTTAIL return keepGoing(job, context);
}
@@ -3311,8 +3324,7 @@ template <class NodeT> void iter(CheckJob *job, CheckContext *context) {
job->setResult(false);
MUSTTAIL return complete(job, context);
}
job->continuation = down_left_spine;
MUSTTAIL return down_left_spine(job, context);
MUSTTAIL return down_left_spine<NodeT>(job, context);
}
}
@@ -3336,16 +3348,17 @@ template <class NodeT> void iter(CheckJob *job, CheckContext *context) {
auto c = getChildGeq(n, job->begin[0]);
if (c != nullptr) {
job->n = c;
job->continuation = down_left_spine;
job->continuation = downLeftSpineTable[c.getType()];
__builtin_prefetch(job->n);
MUSTTAIL return keepGoing(job, context);
} else {
job->n = nextSibling(job->n);
auto c = nextSibling(job->n);
job->n = c;
if (job->n == nullptr) {
job->setResult(true);
MUSTTAIL return complete(job, context);
}
job->continuation = down_left_spine;
job->continuation = downLeftSpineTable[c.getType()];
__builtin_prefetch(job->n);
MUSTTAIL return keepGoing(job, context);
}
@@ -3380,8 +3393,14 @@ static Continuation doneCommonPrefixIterTable[] = {
template <class NodeT>
PRESERVE_NONE void left_side_iter(CheckJob *, CheckContext *);
template <class NodeT>
PRESERVE_NONE void left_side_down_left_spine(CheckJob *, CheckContext *);
static Continuation leftSideDownLeftSpineTable[] = {
left_side_down_left_spine<Node0>, left_side_down_left_spine<Node3>,
left_side_down_left_spine<Node16>, left_side_down_left_spine<Node48>,
left_side_down_left_spine<Node256>};
PRESERVE_NONE void done_left_side_iter(CheckJob *, CheckContext *);
static Continuation leftSideIterTable[] = {
@@ -3525,16 +3544,17 @@ PRESERVE_NONE void done_common_prefix_iter(CheckJob *job,
auto c = getChildGeq(n, job->remaining[0]);
if (c != nullptr) {
job->n = c;
job->continuation = down_left_spine;
MUSTTAIL return job->continuation(job, context);
job->continuation = downLeftSpineTable[c.getType()];
__builtin_prefetch(job->n);
MUSTTAIL return keepGoing(job, context);
} else {
job->n = nextSibling(job->n);
auto c = nextSibling(job->n);
job->n = c;
if (job->n == nullptr) {
job->setResult(true);
MUSTTAIL return complete(job, context);
}
job->continuation = down_left_spine;
MUSTTAIL return job->continuation(job, context);
MUSTTAIL return downLeftSpineTable[c.getType()](job, context);
}
}
@@ -3563,16 +3583,17 @@ PRESERVE_NONE void done_common_prefix_iter(CheckJob *job,
auto c = getChildGeq(n, job->remaining[0]);
if (c != nullptr) {
job->n = c;
job->continuation = left_side_down_left_spine;
job->continuation = leftSideDownLeftSpineTable[c.getType()];
__builtin_prefetch(job->n);
MUSTTAIL return keepGoing(job, context);
} else {
job->n = nextSibling(job->n);
auto c = nextSibling(job->n);
job->n = c;
if (job->n == nullptr) {
job->continuation = done_left_side_iter;
MUSTTAIL return job->continuation(job, context);
}
job->continuation = left_side_down_left_spine;
job->continuation = leftSideDownLeftSpineTable[c.getType()];
__builtin_prefetch(job->n);
MUSTTAIL return keepGoing(job, context);
}
@@ -3602,8 +3623,7 @@ PRESERVE_NONE void left_side_iter(CheckJob *job, CheckContext *context) {
if (c > 0) {
if (n->parent == job->commonPrefixNode) {
if (i < job->lcp) {
job->continuation = left_side_down_left_spine;
MUSTTAIL return job->continuation(job, context);
MUSTTAIL return left_side_down_left_spine<NodeT>(job, context);
}
}
if (n->entryPresent && n->entry.rangeVersion > job->readVersion) {
@@ -3617,12 +3637,14 @@ PRESERVE_NONE void left_side_iter(CheckJob *job, CheckContext *context) {
job->continuation = done_left_side_iter;
MUSTTAIL return job->continuation(job, context);
} else {
job->n = nextSibling(n);
auto c = nextSibling(n);
job->n = c;
if (job->n == nullptr) {
job->continuation = done_left_side_iter;
MUSTTAIL return job->continuation(job, context);
}
job->continuation = left_side_down_left_spine;
job->continuation = leftSideDownLeftSpineTable[c.getType()];
__builtin_prefetch(job->n);
MUSTTAIL return job->continuation(job, context);
}
}
@@ -3673,12 +3695,13 @@ PRESERVE_NONE void left_side_iter(CheckJob *job, CheckContext *context) {
job->continuation = done_left_side_iter;
MUSTTAIL return job->continuation(job, context);
} else {
job->n = nextSibling(job->n);
auto c = nextSibling(job->n);
job->n = c;
if (job->n == nullptr) {
job->continuation = done_left_side_iter;
MUSTTAIL return job->continuation(job, context);
}
job->continuation = left_side_down_left_spine;
job->continuation = leftSideDownLeftSpineTable[c.getType()];
__builtin_prefetch(job->n);
MUSTTAIL return keepGoing(job, context);
}
@@ -3701,16 +3724,17 @@ PRESERVE_NONE void done_left_side_iter(CheckJob *job, CheckContext *context) {
auto c = getChildGeq(job->n, job->remaining[0]);
if (c != nullptr) {
job->n = c;
job->continuation = down_left_spine;
MUSTTAIL return job->continuation(job, context);
job->continuation = downLeftSpineTable[c.getType()];
MUSTTAIL return keepGoing(job, context);
} else {
job->n = nextSibling(job->n);
auto c = nextSibling(job->n);
job->n = c;
if (job->n == nullptr) {
job->setResult(true);
MUSTTAIL return complete(job, context);
}
job->continuation = down_left_spine;
MUSTTAIL return job->continuation(job, context);
job->continuation = downLeftSpineTable[c.getType()];
MUSTTAIL return keepGoing(job, context);
}
}
@@ -3720,16 +3744,22 @@ PRESERVE_NONE void done_left_side_iter(CheckJob *job, CheckContext *context) {
MUSTTAIL return keepGoing(job, context);
}
template <class NodeT>
void left_side_down_left_spine(CheckJob *job, CheckContext *context) {
if (job->n->entryPresent) {
if (job->n->entry.rangeVersion > job->readVersion) {
assert(job->n->getType() == NodeT::kType);
NodeT *n = static_cast<NodeT *>(job->n);
if (n->entryPresent) {
if (n->entry.rangeVersion > job->readVersion) {
job->setResult(false);
MUSTTAIL return complete(job, context);
}
job->continuation = done_left_side_iter;
MUSTTAIL return job->continuation(job, context);
}
job->n = getFirstChildExists(job->n);
auto c = getFirstChildExists(n);
job->n = c;
job->continuation = leftSideDownLeftSpineTable[c.getType()];
__builtin_prefetch(job->n);
MUSTTAIL return keepGoing(job, context);
}
@@ -3750,8 +3780,7 @@ PRESERVE_NONE void right_side_iter(CheckJob *job, CheckContext *context) {
if (i < commonLen) {
auto c = n->partialKey()[i] <=> job->remaining[i];
if (c > 0) {
job->continuation = down_left_spine;
MUSTTAIL return job->continuation(job, context);
MUSTTAIL return down_left_spine<NodeT>(job, context);
} else {
if ((n->parent != job->commonPrefixNode || i >= job->lcp) &&
n->entryPresent && n->entry.rangeVersion > job->readVersion) {
@@ -3763,13 +3792,15 @@ PRESERVE_NONE void right_side_iter(CheckJob *job, CheckContext *context) {
job->setResult(false);
MUSTTAIL return complete(job, context);
}
job->n = nextSibling(job->n);
auto c = nextSibling(job->n);
job->n = c;
if (job->n == nullptr) {
job->setResult(true);
MUSTTAIL return complete(job, context);
}
job->continuation = down_left_spine;
MUSTTAIL return job->continuation(job, context);
job->continuation = downLeftSpineTable[c.getType()];
__builtin_prefetch(job->n);
MUSTTAIL return keepGoing(job, context);
}
}
if (commonLen == n->partialKeyLen) {
@@ -3777,16 +3808,14 @@ PRESERVE_NONE void right_side_iter(CheckJob *job, CheckContext *context) {
job->remaining =
job->remaining.subspan(commonLen, job->remaining.size() - commonLen);
} else if (n->partialKeyLen > int(job->remaining.size())) {
job->continuation = down_left_spine;
MUSTTAIL return job->continuation(job, context);
MUSTTAIL return down_left_spine<NodeT>(job, context);
}
}
++context->readContext.range_read_iterations_accum;
if (job->remaining.size() == 0) {
job->continuation = down_left_spine;
MUSTTAIL return job->continuation(job, context);
MUSTTAIL return down_left_spine<NodeT>(job, context);
}
if (n->entryPresent && n->entry.pointVersion > job->readVersion) {
@@ -3811,16 +3840,19 @@ PRESERVE_NONE void right_side_iter(CheckJob *job, CheckContext *context) {
auto c = getChildGeq(n, job->remaining[0]);
if (c != nullptr) {
job->n = c;
job->continuation = down_left_spine;
MUSTTAIL return job->continuation(job, context);
job->continuation = downLeftSpineTable[c.getType()];
__builtin_prefetch(job->n);
MUSTTAIL return keepGoing(job, context);
} else {
job->n = nextSibling(job->n);
auto c = nextSibling(job->n);
job->n = c;
if (job->n == nullptr) {
job->setResult(true);
MUSTTAIL return complete(job, context);
}
job->continuation = down_left_spine;
MUSTTAIL return job->continuation(job, context);
job->continuation = downLeftSpineTable[c.getType()];
__builtin_prefetch(job->n);
MUSTTAIL return keepGoing(job, context);
}
}