From 8757d2387c823d88fdc7ab596c36fe0274294646 Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Wed, 20 Nov 2024 12:07:32 -0800 Subject: [PATCH] Call prefetch within TaggedNodePointer::getType Instead of at every call site --- ConflictSet.cpp | 46 ++++++++-------------------------------------- 1 file changed, 8 insertions(+), 38 deletions(-) diff --git a/ConflictSet.cpp b/ConflictSet.cpp index 1cbe4d4..5f56f5d 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -262,6 +262,11 @@ struct TaggedNodePointer { TaggedNodePointer &operator=(const TaggedNodePointer &) = default; /*implicit*/ TaggedNodePointer(Node *n); + void prefetch() { + // __builtin_prefetch is ok even if argument isn't addressable + __builtin_prefetch((void *)withoutType()); + } + private: TaggedNodePointer(struct Node *p, Type t) : p((uintptr_t)p) { assert((this->p & 7) == 0); @@ -313,6 +318,7 @@ TaggedNodePointer::TaggedNodePointer(Node *n) Type TaggedNodePointer::getType() { assert(p != 0); + prefetch(); return Type(p & uintptr_t(7)); } @@ -1416,12 +1422,12 @@ TaggedNodePointer getChildGeq(Node *self, int child) { TaggedNodePointer getFirstChild(Node0 *) { return nullptr; } TaggedNodePointer getFirstChild(Node3 *self) { // Improves scan performance - __builtin_prefetch(self->children[1]); + self->children[1].prefetch(); return self->children[0]; } TaggedNodePointer getFirstChild(Node16 *self) { // Improves scan performance - __builtin_prefetch(self->children[1]); + self->children[1].prefetch(); return self->children[0]; } TaggedNodePointer getFirstChild(Node48 *self) { @@ -3247,7 +3253,6 @@ PRESERVE_NONE void down_left_spine(Job *job, Context *context) { } auto child = getFirstChild(n); job->n = child; - __builtin_prefetch(job->n); job->continuation = downLeftSpineTable[child.getType()]; MUSTTAIL return keepGoing(job, context); } @@ -3278,7 +3283,6 @@ void begin(Job *job, Context *context) { if (c != nullptr) { job->n = c; job->continuation = downLeftSpineTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } else { // The root never has a next sibling @@ -3288,7 +3292,6 @@ void begin(Job *job, Context *context) { } job->continuation = iterTable[taggedChild.getType()]; job->n = child; - __builtin_prefetch(child); MUSTTAIL return keepGoing(job, context); } @@ -3313,7 +3316,6 @@ template void iter(Job *job, Context *context) { MUSTTAIL return complete(job, context); } job->continuation = downLeftSpineTable[s.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } } @@ -3343,7 +3345,6 @@ template void iter(Job *job, Context *context) { auto c = getFirstChild(n); job->n = c; job->continuation = downLeftSpineTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } @@ -3355,7 +3356,6 @@ template void iter(Job *job, Context *context) { if (c != nullptr) { job->n = c; job->continuation = downLeftSpineTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } else { auto c = nextSibling(job->n); @@ -3365,13 +3365,11 @@ template void iter(Job *job, Context *context) { MUSTTAIL return complete(job, context); } job->continuation = downLeftSpineTable[c->getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } } job->continuation = iterTable[taggedChild.getType()]; job->n = child; - __builtin_prefetch(child); MUSTTAIL return keepGoing(job, context); } @@ -3400,7 +3398,6 @@ void begin(Job *job, Context *context) { if (c != nullptr) { job->n = c; job->continuation = downLeftSpineTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } else { // The root never has a next sibling @@ -3410,7 +3407,6 @@ void begin(Job *job, Context *context) { } job->continuation = iterTable[taggedChild.getType()]; job->n = child; - __builtin_prefetch(child); MUSTTAIL return keepGoing(job, context); } @@ -3435,7 +3431,6 @@ template void iter(Job *job, Context *context) { MUSTTAIL return complete(job, context); } job->continuation = downLeftSpineTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } } @@ -3475,7 +3470,6 @@ template void iter(Job *job, Context *context) { if (c != nullptr) { job->n = c; job->continuation = downLeftSpineTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } else { auto c = nextSibling(job->n); @@ -3485,13 +3479,11 @@ template void iter(Job *job, Context *context) { MUSTTAIL return complete(job, context); } job->continuation = downLeftSpineTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } } job->continuation = iterTable[taggedChild.getType()]; job->n = child; - __builtin_prefetch(child); MUSTTAIL return keepGoing(job, context); } @@ -3567,7 +3559,6 @@ PRESERVE_NONE void begin(Job *job, Context *context) { } job->continuation = commonPrefixIterTable[c.getType()]; - __builtin_prefetch(job->child); MUSTTAIL return keepGoing(job, context); } @@ -3611,7 +3602,6 @@ template void common_prefix_iter(Job *job, Context *context) { } job->continuation = commonPrefixIterTable[c.getType()]; - __builtin_prefetch(job->child); MUSTTAIL return keepGoing(job, context); } @@ -3660,7 +3650,6 @@ PRESERVE_NONE void done_common_prefix_iter(Job *job, Context *context) { if (c != nullptr) { job->n = c; job->continuation = downLeftSpineTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } else { auto c = nextSibling(job->n); @@ -3675,7 +3664,6 @@ PRESERVE_NONE void done_common_prefix_iter(Job *job, Context *context) { job->n = child; job->continuation = rightSideIterTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } @@ -3699,7 +3687,6 @@ PRESERVE_NONE void done_common_prefix_iter(Job *job, Context *context) { if (c != nullptr) { job->n = c; job->continuation = leftSideDownLeftSpineTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } else { auto c = nextSibling(job->n); @@ -3708,14 +3695,12 @@ PRESERVE_NONE void done_common_prefix_iter(Job *job, Context *context) { MUSTTAIL return done_left_side_iter(job, context); } job->continuation = leftSideDownLeftSpineTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } } job->n = child; job->continuation = leftSideIterTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } @@ -3756,7 +3741,6 @@ PRESERVE_NONE void left_side_iter(Job *job, Context *context) { MUSTTAIL return done_left_side_iter(job, context); } job->continuation = leftSideDownLeftSpineTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } } @@ -3810,14 +3794,12 @@ PRESERVE_NONE void left_side_iter(Job *job, Context *context) { MUSTTAIL return done_left_side_iter(job, context); } job->continuation = leftSideDownLeftSpineTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } } job->n = child; job->continuation = leftSideIterTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } @@ -3848,7 +3830,6 @@ PRESERVE_NONE void done_left_side_iter(Job *job, Context *context) { job->n = child; job->continuation = rightSideIterTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } @@ -3867,7 +3848,6 @@ void left_side_down_left_spine(Job *job, Context *context) { auto c = getFirstChild(n); job->n = c; job->continuation = leftSideDownLeftSpineTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } @@ -3906,7 +3886,6 @@ PRESERVE_NONE void right_side_iter(Job *job, Context *context) { MUSTTAIL return complete(job, context); } job->continuation = downLeftSpineTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } } @@ -3948,7 +3927,6 @@ PRESERVE_NONE void right_side_iter(Job *job, Context *context) { if (c != nullptr) { job->n = c; job->continuation = downLeftSpineTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } else { auto c = nextSibling(job->n); @@ -3958,14 +3936,12 @@ PRESERVE_NONE void right_side_iter(Job *job, Context *context) { MUSTTAIL return complete(job, context); } job->continuation = downLeftSpineTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } } job->n = child; job->continuation = rightSideIterTable[c.getType()]; - __builtin_prefetch(job->n); MUSTTAIL return keepGoing(job, context); } @@ -4187,7 +4163,6 @@ void pointIter(Job *job, Context *context) { } job->continuation = PointIterTable::table[job->child.getType()]; - __builtin_prefetch(job->child); MUSTTAIL return keepGoing(job, context); } @@ -4287,7 +4262,6 @@ void prefixIter(Job *job, Context *context) { goto gotoEndIter; } else { job->continuation = BeginIterTable::table[job->child.getType()]; - __builtin_prefetch(job->child); MUSTTAIL return keepGoing(job, context); } } @@ -4299,7 +4273,6 @@ void prefixIter(Job *job, Context *context) { } job->continuation = PrefixIterTable::table[job->child.getType()]; - __builtin_prefetch(job->child); MUSTTAIL return keepGoing(job, context); noNodeOnSearchPath: { @@ -4326,7 +4299,6 @@ gotoEndIter: MUSTTAIL return complete(job, context); } else { job->continuation = EndIterTable::table[job->child.getType()]; - __builtin_prefetch(job->child); MUSTTAIL return keepGoing(job, context); } } @@ -4374,7 +4346,6 @@ void beginIter(Job *job, Context *context) { } job->continuation = BeginIterTable::table[job->child.getType()]; - __builtin_prefetch(job->child); MUSTTAIL return keepGoing(job, context); gotoEndIter: @@ -4441,7 +4412,6 @@ void endIter(Job *job, Context *context) { } job->continuation = EndIterTable::table[job->child.getType()]; - __builtin_prefetch(job->child); MUSTTAIL return keepGoing(job, context); }