diff --git a/ConflictSet.cpp b/ConflictSet.cpp index a98f0e6..758d356 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -3103,11 +3103,12 @@ Node *firstGeqPhysical(Node *n, const std::span key) { #define PRESERVE_NONE #endif -typedef PRESERVE_NONE void (*Continuation)(struct CheckJob *, - struct CheckContext *); +namespace check { + +typedef PRESERVE_NONE void (*Continuation)(struct Job *, struct Context *); // State relevant to an individual query -struct CheckJob { +struct Job { void setResult(bool ok) { *result = ok ? ConflictSet::Commit : ConflictSet::Conflict; } @@ -3126,12 +3127,12 @@ struct CheckJob { InternalVersionT readVersion; ConflictSet::Result *result; Continuation continuation; - CheckJob *prev; - CheckJob *next; + Job *prev; + Job *next; }; // State relevant to every query -struct CheckContext { +struct Context { int count; int64_t oldestVersionFullPrecision; Node *root; @@ -3141,12 +3142,12 @@ struct CheckContext { ReadContext readContext; }; -PRESERVE_NONE void keepGoing(CheckJob *job, CheckContext *context) { +PRESERVE_NONE void keepGoing(Job *job, Context *context) { job = job->next; MUSTTAIL return job->continuation(job, context); } -PRESERVE_NONE void complete(CheckJob *job, CheckContext *context) { +PRESERVE_NONE void complete(Job *job, Context *context) { if (context->started == context->count) { if (job->prev == job) { return; @@ -3163,14 +3164,14 @@ PRESERVE_NONE void complete(CheckJob *job, CheckContext *context) { } template -PRESERVE_NONE void down_left_spine(CheckJob *job, CheckContext *context); +PRESERVE_NONE void down_left_spine(Job *job, Context *context); static Continuation downLeftSpineTable[] = { down_left_spine, down_left_spine, down_left_spine, down_left_spine, down_left_spine}; template -PRESERVE_NONE void down_left_spine(CheckJob *job, CheckContext *context) { +PRESERVE_NONE void down_left_spine(Job *job, Context *context) { assert(job->n->getType() == NodeT::kType); NodeT *n = static_cast(job->n); if (n->entryPresent) { @@ -3184,16 +3185,16 @@ PRESERVE_NONE void down_left_spine(CheckJob *job, CheckContext *context) { MUSTTAIL return keepGoing(job, context); } -namespace check_point_read_state_machine { +namespace point_read_state_machine { -PRESERVE_NONE void begin(CheckJob *, CheckContext *); +PRESERVE_NONE void begin(Job *, Context *); -template PRESERVE_NONE void iter(CheckJob *, CheckContext *); +template PRESERVE_NONE void iter(Job *, Context *); static Continuation iterTable[] = {iter, iter, iter, iter, iter}; -void begin(CheckJob *job, CheckContext *context) { +void begin(Job *job, Context *context) { ++context->readContext.point_read_accum; #if DEBUG_VERBOSE && !defined(NDEBUG) fprintf(stderr, "Check point read: %s\n", printable(key).c_str()); @@ -3228,7 +3229,7 @@ void begin(CheckJob *job, CheckContext *context) { MUSTTAIL return keepGoing(job, context); } -template void iter(CheckJob *job, CheckContext *context) { +template void iter(Job *job, Context *context) { assert(NodeT::kType == job->n->getType()); NodeT *n = static_cast(job->n); @@ -3311,18 +3312,18 @@ template void iter(CheckJob *job, CheckContext *context) { MUSTTAIL return keepGoing(job, context); } -} // namespace check_point_read_state_machine +} // namespace point_read_state_machine -namespace check_prefix_read_state_machine { +namespace prefix_read_state_machine { -PRESERVE_NONE void begin(CheckJob *, CheckContext *); +PRESERVE_NONE void begin(Job *, Context *); -template PRESERVE_NONE void iter(CheckJob *, CheckContext *); +template PRESERVE_NONE void iter(Job *, Context *); static Continuation iterTable[] = {iter, iter, iter, iter, iter}; -void begin(CheckJob *job, CheckContext *context) { +void begin(Job *job, Context *context) { ++context->readContext.prefix_read_accum; #if DEBUG_VERBOSE && !defined(NDEBUG) fprintf(stderr, "Check prefix read: %s\n", printable(key).c_str()); @@ -3353,7 +3354,7 @@ void begin(CheckJob *job, CheckContext *context) { MUSTTAIL return keepGoing(job, context); } -template void iter(CheckJob *job, CheckContext *context) { +template void iter(Job *job, Context *context) { assert(NodeT::kType == job->n->getType()); NodeT *n = static_cast(job->n); @@ -3434,16 +3435,15 @@ template void iter(CheckJob *job, CheckContext *context) { MUSTTAIL return keepGoing(job, context); } -} // namespace check_prefix_read_state_machine +} // namespace prefix_read_state_machine -namespace check_range_read_state_machine { -PRESERVE_NONE void begin(CheckJob *, CheckContext *); +namespace range_read_state_machine { +PRESERVE_NONE void begin(Job *, Context *); + +template PRESERVE_NONE void common_prefix_iter(Job *, Context *); template -PRESERVE_NONE void common_prefix_iter(CheckJob *, CheckContext *); - -template -PRESERVE_NONE void done_common_prefix_iter(CheckJob *, CheckContext *); +PRESERVE_NONE void done_common_prefix_iter(Job *, Context *); static Continuation commonPrefixIterTable[] = { common_prefix_iter, common_prefix_iter, @@ -3455,43 +3455,41 @@ static Continuation doneCommonPrefixIterTable[] = { done_common_prefix_iter, done_common_prefix_iter, done_common_prefix_iter}; -template -PRESERVE_NONE void left_side_iter(CheckJob *, CheckContext *); +template PRESERVE_NONE void left_side_iter(Job *, Context *); template -PRESERVE_NONE void left_side_down_left_spine(CheckJob *, CheckContext *); +PRESERVE_NONE void left_side_down_left_spine(Job *, Context *); static Continuation leftSideDownLeftSpineTable[] = { left_side_down_left_spine, left_side_down_left_spine, left_side_down_left_spine, left_side_down_left_spine, left_side_down_left_spine}; -PRESERVE_NONE void done_left_side_iter(CheckJob *, CheckContext *); +PRESERVE_NONE void done_left_side_iter(Job *, Context *); static Continuation leftSideIterTable[] = { left_side_iter, left_side_iter, left_side_iter, left_side_iter, left_side_iter}; -template -PRESERVE_NONE void right_side_iter(CheckJob *, CheckContext *); +template PRESERVE_NONE void right_side_iter(Job *, Context *); static Continuation rightSideIterTable[] = { right_side_iter, right_side_iter, right_side_iter, right_side_iter, right_side_iter}; -PRESERVE_NONE void begin(CheckJob *job, CheckContext *context) { +PRESERVE_NONE void begin(Job *job, Context *context) { job->lcp = longestCommonPrefix(job->begin.data(), job->end.data(), std::min(job->begin.size(), job->end.size())); if (job->lcp == int(job->begin.size()) && job->end.size() == job->begin.size() + 1 && job->end.back() == 0) { // Call directly since we have nothing to prefetch - MUSTTAIL return check_point_read_state_machine::begin(job, context); + MUSTTAIL return check::point_read_state_machine::begin(job, context); } if (job->lcp == int(job->begin.size() - 1) && job->end.size() == job->begin.size() && int(job->begin.back()) + 1 == int(job->end.back())) { // Call directly since we have nothing to prefetch - MUSTTAIL return check_prefix_read_state_machine::begin(job, context); + MUSTTAIL return check::prefix_read_state_machine::begin(job, context); } ++context->readContext.range_read_accum; @@ -3514,8 +3512,7 @@ PRESERVE_NONE void begin(CheckJob *job, CheckContext *context) { } // Advance down common prefix, but stay on a physical path in the tree -template -void common_prefix_iter(CheckJob *job, CheckContext *context) { +template void common_prefix_iter(Job *job, Context *context) { assert(NodeT::kType == job->child->getType()); NodeT *child = static_cast(job->child); @@ -3559,8 +3556,7 @@ void common_prefix_iter(CheckJob *job, CheckContext *context) { } template -PRESERVE_NONE void done_common_prefix_iter(CheckJob *job, - CheckContext *context) { +PRESERVE_NONE void done_common_prefix_iter(Job *job, Context *context) { assert(NodeT::kType == job->n->getType()); NodeT *n = static_cast(job->n); @@ -3666,7 +3662,7 @@ PRESERVE_NONE void done_common_prefix_iter(CheckJob *job, // Return true if the max version among all keys that start with key[:prefixLen] // that are >= key is <= readVersion template -PRESERVE_NONE void left_side_iter(CheckJob *job, CheckContext *context) { +PRESERVE_NONE void left_side_iter(Job *job, Context *context) { assert(NodeT::kType == job->n->getType()); NodeT *n = static_cast(job->n); @@ -3765,7 +3761,7 @@ PRESERVE_NONE void left_side_iter(CheckJob *job, CheckContext *context) { MUSTTAIL return keepGoing(job, context); } -PRESERVE_NONE void done_left_side_iter(CheckJob *job, CheckContext *context) { +PRESERVE_NONE void done_left_side_iter(Job *job, Context *context) { job->n = job->commonPrefixNode; job->remaining = job->end; @@ -3797,7 +3793,7 @@ PRESERVE_NONE void done_left_side_iter(CheckJob *job, CheckContext *context) { } template -void left_side_down_left_spine(CheckJob *job, CheckContext *context) { +void left_side_down_left_spine(Job *job, Context *context) { assert(job->n->getType() == NodeT::kType); NodeT *n = static_cast(job->n); @@ -3818,7 +3814,7 @@ void left_side_down_left_spine(CheckJob *job, CheckContext *context) { // Return true if the max version among all keys that start with key[:prefixLen] // that are < key is <= readVersion template -PRESERVE_NONE void right_side_iter(CheckJob *job, CheckContext *context) { +PRESERVE_NONE void right_side_iter(Job *job, Context *context) { assert(NodeT::kType == job->n->getType()); NodeT *n = static_cast(job->n); @@ -3913,11 +3909,10 @@ PRESERVE_NONE void right_side_iter(CheckJob *job, CheckContext *context) { MUSTTAIL return keepGoing(job, context); } -} // namespace check_range_read_state_machine +} // namespace range_read_state_machine -void CheckJob::init(const ConflictSet::ReadRange *read, - ConflictSet::Result *result, Node *root, - int64_t oldestVersionFullPrecision) { +void Job::init(const ConflictSet::ReadRange *read, ConflictSet::Result *result, + Node *root, int64_t oldestVersionFullPrecision) { auto begin = std::span(read->begin.p, read->begin.len); auto end = std::span(read->end.p, read->end.len); if (read->readVersion < oldestVersionFullPrecision) [[unlikely]] { @@ -3928,16 +3923,17 @@ void CheckJob::init(const ConflictSet::ReadRange *read, this->n = root; this->readVersion = InternalVersionT(read->readVersion); this->result = result; - continuation = check_point_read_state_machine::begin; + continuation = check::point_read_state_machine::begin; } else { this->begin = begin; this->end = end; this->n = root; this->readVersion = InternalVersionT(read->readVersion); this->result = result; - continuation = check_range_read_state_machine::begin; + continuation = check::range_read_state_machine::begin; } } +} // namespace check // Sequential implementations namespace { @@ -4352,7 +4348,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { // We still have the sequential implementation for compilers that don't // support preserve_none and musttail void useSequential(const ReadRange *reads, Result *result, int count, - CheckContext &context) { + check::Context &context) { for (int i = 0; i < count; ++i) { if (reads[i].readVersion < oldestVersionFullPrecision) [[unlikely]] { result[i] = TooOld; @@ -4384,7 +4380,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { } int64_t check_byte_accum = 0; - CheckContext context; + check::Context context; context.readContext.impl = this; #if __has_attribute(preserve_none) && __has_attribute(musttail) @@ -4392,7 +4388,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { useSequential(reads, result, count, context); } else { constexpr int kConcurrent = 16; - CheckJob inProgress[kConcurrent]; + check::Job inProgress[kConcurrent]; context.count = count; context.oldestVersionFullPrecision = oldestVersionFullPrecision; context.root = root; @@ -4420,7 +4416,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { #ifndef NDEBUG Arena arena; auto *results2 = new (arena) Result[count]; - CheckContext context2; + check::Context context2; context2.readContext.impl = this; useSequential(reads, results2, count, context2); assert(memcmp(result, results2, count) == 0);