diff --git a/ConflictSet.cpp b/ConflictSet.cpp index e47a6d5..b7b79c0 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -1193,9 +1193,9 @@ ChildAndMaxVersion getChildAndMaxVersion(Node *self, uint8_t index) { } } -Node *getChildGeq(Node0 *, int) { return nullptr; } +TaggedNodePointer getChildGeq(Node0 *, int) { return nullptr; } -Node *getChildGeq(Node3 *n, int child) { +TaggedNodePointer getChildGeq(Node3 *n, int child) { assume(n->numChildren >= 1); assume(n->numChildren <= 3); for (int i = 0; i < n->numChildren; ++i) { @@ -1206,7 +1206,7 @@ Node *getChildGeq(Node3 *n, int child) { return nullptr; } -Node *getChildGeq(Node16 *self, int child) { +TaggedNodePointer getChildGeq(Node16 *self, int child) { if (child > 255) { return nullptr; } @@ -1249,14 +1249,14 @@ Node *getChildGeq(Node16 *self, int child) { #endif } -Node *getChildGeq(Node48 *self, int child) { +TaggedNodePointer getChildGeq(Node48 *self, int child) { int c = self->bitSet.firstSetGeq(child); if (c < 0) { return nullptr; } return self->children[self->index[c]]; } -Node *getChildGeq(Node256 *self, int child) { +TaggedNodePointer getChildGeq(Node256 *self, int child) { int c = self->bitSet.firstSetGeq(child); if (c < 0) { return nullptr; @@ -1264,7 +1264,7 @@ Node *getChildGeq(Node256 *self, int child) { return self->children[c]; } -Node *getChildGeq(Node *self, int child) { +TaggedNodePointer getChildGeq(Node *self, int child) { switch (self->getType()) { case Type_Node0: return getChildGeq(static_cast(self), child); @@ -1317,26 +1317,26 @@ Node *getFirstChild(Node *self) { } // Precondition: self has a child -Node *getFirstChildExists(Node3 *self) { +TaggedNodePointer getFirstChildExists(Node3 *self) { assert(self->numChildren > 0); return self->children[0]; } // Precondition: self has a child -Node *getFirstChildExists(Node16 *self) { +TaggedNodePointer getFirstChildExists(Node16 *self) { assert(self->numChildren > 0); return self->children[0]; } // Precondition: self has a child -Node *getFirstChildExists(Node48 *self) { +TaggedNodePointer getFirstChildExists(Node48 *self) { return self->children[self->index[self->bitSet.firstSetGeq(0)]]; } // Precondition: self has a child -Node *getFirstChildExists(Node256 *self) { +TaggedNodePointer getFirstChildExists(Node256 *self) { return self->children[self->bitSet.firstSetGeq(0)]; } // Precondition: self has a child -Node *getFirstChildExists(Node *self) { +TaggedNodePointer getFirstChildExists(Node *self) { // Only require that the node-specific overloads are covered // GCOVR_EXCL_START switch (self->getType()) { @@ -2021,7 +2021,7 @@ Node *erase(Node *self, WriteContext *writeContext, ConflictSet::Impl *impl, return result; } -Node *nextSibling(Node *node) { +TaggedNodePointer nextSibling(Node *node) { for (;;) { if (node->parent == nullptr) { return nullptr; @@ -3097,13 +3097,25 @@ PRESERVE_NONE void complete(CheckJob *job, CheckContext *context) { MUSTTAIL return keepGoing(job, context); } +template +PRESERVE_NONE void down_left_spine(CheckJob *job, CheckContext *context); + +static Continuation downLeftSpineTable[] = { + down_left_spine, down_left_spine, down_left_spine, + down_left_spine, down_left_spine}; + +template PRESERVE_NONE void down_left_spine(CheckJob *job, CheckContext *context) { - if (job->n->entryPresent) { - job->setResult(job->n->entry.rangeVersion <= job->readVersion); + assert(job->n->getType() == NodeT::kType); + NodeT *n = static_cast(job->n); + if (n->entryPresent) { + job->setResult(n->entry.rangeVersion <= job->readVersion); MUSTTAIL return complete(job, context); } - job->n = getFirstChildExists(job->n); + auto child = getFirstChildExists(n); + job->n = child; __builtin_prefetch(job->n); + job->continuation = downLeftSpineTable[child.getType()]; MUSTTAIL return keepGoing(job, context); } @@ -3136,7 +3148,7 @@ void begin(CheckJob *job, CheckContext *context) { auto c = getChildGeq(job->n, job->begin[0]); if (c != nullptr) { job->n = c; - job->continuation = down_left_spine; + job->continuation = downLeftSpineTable[c.getType()]; __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } else { @@ -3163,15 +3175,15 @@ template void iter(CheckJob *job, CheckContext *context) { if (i < commonLen) [[unlikely]] { auto c = n->partialKey()[i] <=> job->begin[i]; if (c > 0) { - job->continuation = down_left_spine; - MUSTTAIL return down_left_spine(job, context); + MUSTTAIL return down_left_spine(job, context); } else { - job->n = nextSibling(n); + auto s = nextSibling(n); + job->n = s; if (job->n == nullptr) { job->setResult(true); MUSTTAIL return complete(job, context); } - job->continuation = down_left_spine; + job->continuation = downLeftSpineTable[s.getType()]; __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } @@ -3182,8 +3194,7 @@ template void iter(CheckJob *job, CheckContext *context) { } else if (n->partialKeyLen > int(job->begin.size())) [[unlikely]] { // n is the first physical node greater than remaining, and there's no // eq node - job->continuation = down_left_spine; - MUSTTAIL return down_left_spine(job, context); + MUSTTAIL return down_left_spine(job, context); } } @@ -3200,8 +3211,9 @@ template void iter(CheckJob *job, CheckContext *context) { job->setResult(n->entry.pointVersion <= job->readVersion); MUSTTAIL return complete(job, context); } - job->n = getFirstChildExists(n); - job->continuation = down_left_spine; + auto c = getFirstChildExists(n); + job->n = c; + job->continuation = downLeftSpineTable[c.getType()]; __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } @@ -3213,16 +3225,17 @@ template void iter(CheckJob *job, CheckContext *context) { auto c = getChildGeq(n, job->begin[0]); if (c != nullptr) { job->n = c; - job->continuation = down_left_spine; + job->continuation = downLeftSpineTable[c.getType()]; __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } else { - job->n = nextSibling(job->n); + auto c = nextSibling(job->n); + job->n = c; if (job->n == nullptr) { job->setResult(true); MUSTTAIL return complete(job, context); } - job->continuation = down_left_spine; + job->continuation = downLeftSpineTable[c->getType()]; __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } @@ -3260,7 +3273,7 @@ void begin(CheckJob *job, CheckContext *context) { auto c = getChildGeq(job->n, job->begin[0]); if (c != nullptr) { job->n = c; - job->continuation = down_left_spine; + job->continuation = downLeftSpineTable[c.getType()]; __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } else { @@ -3287,15 +3300,15 @@ template void iter(CheckJob *job, CheckContext *context) { if (i < commonLen) [[unlikely]] { auto c = n->partialKey()[i] <=> job->begin[i]; if (c > 0) { - job->continuation = down_left_spine; - MUSTTAIL return down_left_spine(job, context); + MUSTTAIL return down_left_spine(job, context); } else { - job->n = nextSibling(n); + auto c = nextSibling(n); + job->n = c; if (job->n == nullptr) { job->setResult(true); MUSTTAIL return complete(job, context); } - job->continuation = down_left_spine; + job->continuation = downLeftSpineTable[c.getType()]; __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } @@ -3311,8 +3324,7 @@ template void iter(CheckJob *job, CheckContext *context) { job->setResult(false); MUSTTAIL return complete(job, context); } - job->continuation = down_left_spine; - MUSTTAIL return down_left_spine(job, context); + MUSTTAIL return down_left_spine(job, context); } } @@ -3336,16 +3348,17 @@ template void iter(CheckJob *job, CheckContext *context) { auto c = getChildGeq(n, job->begin[0]); if (c != nullptr) { job->n = c; - job->continuation = down_left_spine; + job->continuation = downLeftSpineTable[c.getType()]; __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } else { - job->n = nextSibling(job->n); + auto c = nextSibling(job->n); + job->n = c; if (job->n == nullptr) { job->setResult(true); MUSTTAIL return complete(job, context); } - job->continuation = down_left_spine; + job->continuation = downLeftSpineTable[c.getType()]; __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } @@ -3380,8 +3393,14 @@ static Continuation doneCommonPrefixIterTable[] = { template PRESERVE_NONE void left_side_iter(CheckJob *, CheckContext *); +template PRESERVE_NONE void left_side_down_left_spine(CheckJob *, CheckContext *); +static Continuation leftSideDownLeftSpineTable[] = { + left_side_down_left_spine, left_side_down_left_spine, + left_side_down_left_spine, left_side_down_left_spine, + left_side_down_left_spine}; + PRESERVE_NONE void done_left_side_iter(CheckJob *, CheckContext *); static Continuation leftSideIterTable[] = { @@ -3525,16 +3544,17 @@ PRESERVE_NONE void done_common_prefix_iter(CheckJob *job, auto c = getChildGeq(n, job->remaining[0]); if (c != nullptr) { job->n = c; - job->continuation = down_left_spine; - MUSTTAIL return job->continuation(job, context); + job->continuation = downLeftSpineTable[c.getType()]; + __builtin_prefetch(job->n); + MUSTTAIL return keepGoing(job, context); } else { - job->n = nextSibling(job->n); + auto c = nextSibling(job->n); + job->n = c; if (job->n == nullptr) { job->setResult(true); MUSTTAIL return complete(job, context); } - job->continuation = down_left_spine; - MUSTTAIL return job->continuation(job, context); + MUSTTAIL return downLeftSpineTable[c.getType()](job, context); } } @@ -3563,16 +3583,17 @@ PRESERVE_NONE void done_common_prefix_iter(CheckJob *job, auto c = getChildGeq(n, job->remaining[0]); if (c != nullptr) { job->n = c; - job->continuation = left_side_down_left_spine; + job->continuation = leftSideDownLeftSpineTable[c.getType()]; __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } else { - job->n = nextSibling(job->n); + auto c = nextSibling(job->n); + job->n = c; if (job->n == nullptr) { job->continuation = done_left_side_iter; MUSTTAIL return job->continuation(job, context); } - job->continuation = left_side_down_left_spine; + job->continuation = leftSideDownLeftSpineTable[c.getType()]; __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } @@ -3602,8 +3623,7 @@ PRESERVE_NONE void left_side_iter(CheckJob *job, CheckContext *context) { if (c > 0) { if (n->parent == job->commonPrefixNode) { if (i < job->lcp) { - job->continuation = left_side_down_left_spine; - MUSTTAIL return job->continuation(job, context); + MUSTTAIL return left_side_down_left_spine(job, context); } } if (n->entryPresent && n->entry.rangeVersion > job->readVersion) { @@ -3617,12 +3637,14 @@ PRESERVE_NONE void left_side_iter(CheckJob *job, CheckContext *context) { job->continuation = done_left_side_iter; MUSTTAIL return job->continuation(job, context); } else { - job->n = nextSibling(n); + auto c = nextSibling(n); + job->n = c; if (job->n == nullptr) { job->continuation = done_left_side_iter; MUSTTAIL return job->continuation(job, context); } - job->continuation = left_side_down_left_spine; + job->continuation = leftSideDownLeftSpineTable[c.getType()]; + __builtin_prefetch(job->n); MUSTTAIL return job->continuation(job, context); } } @@ -3673,12 +3695,13 @@ PRESERVE_NONE void left_side_iter(CheckJob *job, CheckContext *context) { job->continuation = done_left_side_iter; MUSTTAIL return job->continuation(job, context); } else { - job->n = nextSibling(job->n); + auto c = nextSibling(job->n); + job->n = c; if (job->n == nullptr) { job->continuation = done_left_side_iter; MUSTTAIL return job->continuation(job, context); } - job->continuation = left_side_down_left_spine; + job->continuation = leftSideDownLeftSpineTable[c.getType()]; __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } @@ -3701,16 +3724,17 @@ PRESERVE_NONE void done_left_side_iter(CheckJob *job, CheckContext *context) { auto c = getChildGeq(job->n, job->remaining[0]); if (c != nullptr) { job->n = c; - job->continuation = down_left_spine; - MUSTTAIL return job->continuation(job, context); + job->continuation = downLeftSpineTable[c.getType()]; + MUSTTAIL return keepGoing(job, context); } else { - job->n = nextSibling(job->n); + auto c = nextSibling(job->n); + job->n = c; if (job->n == nullptr) { job->setResult(true); MUSTTAIL return complete(job, context); } - job->continuation = down_left_spine; - MUSTTAIL return job->continuation(job, context); + job->continuation = downLeftSpineTable[c.getType()]; + MUSTTAIL return keepGoing(job, context); } } @@ -3720,16 +3744,22 @@ PRESERVE_NONE void done_left_side_iter(CheckJob *job, CheckContext *context) { MUSTTAIL return keepGoing(job, context); } +template void left_side_down_left_spine(CheckJob *job, CheckContext *context) { - if (job->n->entryPresent) { - if (job->n->entry.rangeVersion > job->readVersion) { + assert(job->n->getType() == NodeT::kType); + NodeT *n = static_cast(job->n); + + if (n->entryPresent) { + if (n->entry.rangeVersion > job->readVersion) { job->setResult(false); MUSTTAIL return complete(job, context); } job->continuation = done_left_side_iter; MUSTTAIL return job->continuation(job, context); } - job->n = getFirstChildExists(job->n); + auto c = getFirstChildExists(n); + job->n = c; + job->continuation = leftSideDownLeftSpineTable[c.getType()]; __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } @@ -3750,8 +3780,7 @@ PRESERVE_NONE void right_side_iter(CheckJob *job, CheckContext *context) { if (i < commonLen) { auto c = n->partialKey()[i] <=> job->remaining[i]; if (c > 0) { - job->continuation = down_left_spine; - MUSTTAIL return job->continuation(job, context); + MUSTTAIL return down_left_spine(job, context); } else { if ((n->parent != job->commonPrefixNode || i >= job->lcp) && n->entryPresent && n->entry.rangeVersion > job->readVersion) { @@ -3763,13 +3792,15 @@ PRESERVE_NONE void right_side_iter(CheckJob *job, CheckContext *context) { job->setResult(false); MUSTTAIL return complete(job, context); } - job->n = nextSibling(job->n); + auto c = nextSibling(job->n); + job->n = c; if (job->n == nullptr) { job->setResult(true); MUSTTAIL return complete(job, context); } - job->continuation = down_left_spine; - MUSTTAIL return job->continuation(job, context); + job->continuation = downLeftSpineTable[c.getType()]; + __builtin_prefetch(job->n); + MUSTTAIL return keepGoing(job, context); } } if (commonLen == n->partialKeyLen) { @@ -3777,16 +3808,14 @@ PRESERVE_NONE void right_side_iter(CheckJob *job, CheckContext *context) { job->remaining = job->remaining.subspan(commonLen, job->remaining.size() - commonLen); } else if (n->partialKeyLen > int(job->remaining.size())) { - job->continuation = down_left_spine; - MUSTTAIL return job->continuation(job, context); + MUSTTAIL return down_left_spine(job, context); } } ++context->readContext.range_read_iterations_accum; if (job->remaining.size() == 0) { - job->continuation = down_left_spine; - MUSTTAIL return job->continuation(job, context); + MUSTTAIL return down_left_spine(job, context); } if (n->entryPresent && n->entry.pointVersion > job->readVersion) { @@ -3811,16 +3840,19 @@ PRESERVE_NONE void right_side_iter(CheckJob *job, CheckContext *context) { auto c = getChildGeq(n, job->remaining[0]); if (c != nullptr) { job->n = c; - job->continuation = down_left_spine; - MUSTTAIL return job->continuation(job, context); + job->continuation = downLeftSpineTable[c.getType()]; + __builtin_prefetch(job->n); + MUSTTAIL return keepGoing(job, context); } else { - job->n = nextSibling(job->n); + auto c = nextSibling(job->n); + job->n = c; if (job->n == nullptr) { job->setResult(true); MUSTTAIL return complete(job, context); } - job->continuation = down_left_spine; - MUSTTAIL return job->continuation(job, context); + job->continuation = downLeftSpineTable[c.getType()]; + __builtin_prefetch(job->n); + MUSTTAIL return keepGoing(job, context); } }