Call prefetch within TaggedNodePointer::getType

Instead of at every call site

ConflictSet.cpp

@@ -262,6 +262,11 @@ struct TaggedNodePointer {
   TaggedNodePointer &operator=(const TaggedNodePointer &) = default;
   /*implicit*/ TaggedNodePointer(Node *n);
 
+  void prefetch() {
+    // __builtin_prefetch is ok even if argument isn't addressable
+    __builtin_prefetch((void *)withoutType());
+  }
+
 private:
   TaggedNodePointer(struct Node *p, Type t) : p((uintptr_t)p) {
     assert((this->p & 7) == 0);
@@ -313,6 +318,7 @@ TaggedNodePointer::TaggedNodePointer(Node *n)
 
 Type TaggedNodePointer::getType() {
   assert(p != 0);
+  prefetch();
   return Type(p & uintptr_t(7));
 }
 
@@ -1416,12 +1422,12 @@ TaggedNodePointer getChildGeq(Node *self, int child) {
 TaggedNodePointer getFirstChild(Node0 *) { return nullptr; }
 TaggedNodePointer getFirstChild(Node3 *self) {
   // Improves scan performance
-  __builtin_prefetch(self->children[1]);
+  self->children[1].prefetch();
   return self->children[0];
 }
 TaggedNodePointer getFirstChild(Node16 *self) {
   // Improves scan performance
-  __builtin_prefetch(self->children[1]);
+  self->children[1].prefetch();
   return self->children[0];
 }
 TaggedNodePointer getFirstChild(Node48 *self) {
@@ -3247,7 +3253,6 @@ PRESERVE_NONE void down_left_spine(Job *job, Context *context) {
   }
   auto child = getFirstChild(n);
   job->n = child;
-  __builtin_prefetch(job->n);
   job->continuation = downLeftSpineTable[child.getType()];
   MUSTTAIL return keepGoing(job, context);
 }
@@ -3278,7 +3283,6 @@ void begin(Job *job, Context *context) {
     if (c != nullptr) {
       job->n = c;
       job->continuation = downLeftSpineTable[c.getType()];
-      __builtin_prefetch(job->n);
       MUSTTAIL return keepGoing(job, context);
     } else {
       // The root never has a next sibling
@@ -3288,7 +3292,6 @@ void begin(Job *job, Context *context) {
   }
   job->continuation = iterTable[taggedChild.getType()];
   job->n = child;
-  __builtin_prefetch(child);
   MUSTTAIL return keepGoing(job, context);
 }
 
@@ -3313,7 +3316,6 @@ template <class NodeT> void iter(Job *job, Context *context) {
           MUSTTAIL return complete(job, context);
         }
         job->continuation = downLeftSpineTable[s.getType()];
-        __builtin_prefetch(job->n);
         MUSTTAIL return keepGoing(job, context);
       }
     }
@@ -3343,7 +3345,6 @@ template <class NodeT> void iter(Job *job, Context *context) {
     auto c = getFirstChild(n);
     job->n = c;
     job->continuation = downLeftSpineTable[c.getType()];
-    __builtin_prefetch(job->n);
     MUSTTAIL return keepGoing(job, context);
   }
 
@@ -3355,7 +3356,6 @@ template <class NodeT> void iter(Job *job, Context *context) {
     if (c != nullptr) {
       job->n = c;
       job->continuation = downLeftSpineTable[c.getType()];
-      __builtin_prefetch(job->n);
       MUSTTAIL return keepGoing(job, context);
     } else {
       auto c = nextSibling(job->n);
@@ -3365,13 +3365,11 @@ template <class NodeT> void iter(Job *job, Context *context) {
         MUSTTAIL return complete(job, context);
       }
       job->continuation = downLeftSpineTable[c->getType()];
-      __builtin_prefetch(job->n);
       MUSTTAIL return keepGoing(job, context);
     }
   }
   job->continuation = iterTable[taggedChild.getType()];
   job->n = child;
-  __builtin_prefetch(child);
   MUSTTAIL return keepGoing(job, context);
 }
 
@@ -3400,7 +3398,6 @@ void begin(Job *job, Context *context) {
     if (c != nullptr) {
       job->n = c;
       job->continuation = downLeftSpineTable[c.getType()];
-      __builtin_prefetch(job->n);
       MUSTTAIL return keepGoing(job, context);
     } else {
       // The root never has a next sibling
@@ -3410,7 +3407,6 @@ void begin(Job *job, Context *context) {
   }
   job->continuation = iterTable[taggedChild.getType()];
   job->n = child;
-  __builtin_prefetch(child);
   MUSTTAIL return keepGoing(job, context);
 }
 
@@ -3435,7 +3431,6 @@ template <class NodeT> void iter(Job *job, Context *context) {
           MUSTTAIL return complete(job, context);
         }
         job->continuation = downLeftSpineTable[c.getType()];
-        __builtin_prefetch(job->n);
         MUSTTAIL return keepGoing(job, context);
       }
     }
@@ -3475,7 +3470,6 @@ template <class NodeT> void iter(Job *job, Context *context) {
     if (c != nullptr) {
       job->n = c;
       job->continuation = downLeftSpineTable[c.getType()];
-      __builtin_prefetch(job->n);
       MUSTTAIL return keepGoing(job, context);
     } else {
       auto c = nextSibling(job->n);
@@ -3485,13 +3479,11 @@ template <class NodeT> void iter(Job *job, Context *context) {
         MUSTTAIL return complete(job, context);
       }
       job->continuation = downLeftSpineTable[c.getType()];
-      __builtin_prefetch(job->n);
       MUSTTAIL return keepGoing(job, context);
     }
   }
   job->continuation = iterTable[taggedChild.getType()];
   job->n = child;
-  __builtin_prefetch(child);
   MUSTTAIL return keepGoing(job, context);
 }
 
@@ -3567,7 +3559,6 @@ PRESERVE_NONE void begin(Job *job, Context *context) {
   }
 
   job->continuation = commonPrefixIterTable[c.getType()];
-  __builtin_prefetch(job->child);
   MUSTTAIL return keepGoing(job, context);
 }
 
@@ -3611,7 +3602,6 @@ template <class NodeT> void common_prefix_iter(Job *job, Context *context) {
   }
 
   job->continuation = commonPrefixIterTable[c.getType()];
-  __builtin_prefetch(job->child);
   MUSTTAIL return keepGoing(job, context);
 }
 
@@ -3660,7 +3650,6 @@ PRESERVE_NONE void done_common_prefix_iter(Job *job, Context *context) {
       if (c != nullptr) {
         job->n = c;
         job->continuation = downLeftSpineTable[c.getType()];
-        __builtin_prefetch(job->n);
         MUSTTAIL return keepGoing(job, context);
       } else {
         auto c = nextSibling(job->n);
@@ -3675,7 +3664,6 @@ PRESERVE_NONE void done_common_prefix_iter(Job *job, Context *context) {
 
     job->n = child;
     job->continuation = rightSideIterTable[c.getType()];
-    __builtin_prefetch(job->n);
     MUSTTAIL return keepGoing(job, context);
   }
 
@@ -3699,7 +3687,6 @@ PRESERVE_NONE void done_common_prefix_iter(Job *job, Context *context) {
     if (c != nullptr) {
       job->n = c;
       job->continuation = leftSideDownLeftSpineTable[c.getType()];
-      __builtin_prefetch(job->n);
       MUSTTAIL return keepGoing(job, context);
     } else {
       auto c = nextSibling(job->n);
@@ -3708,14 +3695,12 @@ PRESERVE_NONE void done_common_prefix_iter(Job *job, Context *context) {
         MUSTTAIL return done_left_side_iter(job, context);
       }
       job->continuation = leftSideDownLeftSpineTable[c.getType()];
-      __builtin_prefetch(job->n);
       MUSTTAIL return keepGoing(job, context);
     }
   }
 
   job->n = child;
   job->continuation = leftSideIterTable[c.getType()];
-  __builtin_prefetch(job->n);
   MUSTTAIL return keepGoing(job, context);
 }
 
@@ -3756,7 +3741,6 @@ PRESERVE_NONE void left_side_iter(Job *job, Context *context) {
           MUSTTAIL return done_left_side_iter(job, context);
         }
         job->continuation = leftSideDownLeftSpineTable[c.getType()];
-        __builtin_prefetch(job->n);
         MUSTTAIL return keepGoing(job, context);
       }
     }
@@ -3810,14 +3794,12 @@ PRESERVE_NONE void left_side_iter(Job *job, Context *context) {
         MUSTTAIL return done_left_side_iter(job, context);
       }
       job->continuation = leftSideDownLeftSpineTable[c.getType()];
-      __builtin_prefetch(job->n);
       MUSTTAIL return keepGoing(job, context);
     }
   }
 
   job->n = child;
   job->continuation = leftSideIterTable[c.getType()];
-  __builtin_prefetch(job->n);
   MUSTTAIL return keepGoing(job, context);
 }
 
@@ -3848,7 +3830,6 @@ PRESERVE_NONE void done_left_side_iter(Job *job, Context *context) {
 
   job->n = child;
   job->continuation = rightSideIterTable[c.getType()];
-  __builtin_prefetch(job->n);
   MUSTTAIL return keepGoing(job, context);
 }
 
@@ -3867,7 +3848,6 @@ void left_side_down_left_spine(Job *job, Context *context) {
   auto c = getFirstChild(n);
   job->n = c;
   job->continuation = leftSideDownLeftSpineTable[c.getType()];
-  __builtin_prefetch(job->n);
   MUSTTAIL return keepGoing(job, context);
 }
 
@@ -3906,7 +3886,6 @@ PRESERVE_NONE void right_side_iter(Job *job, Context *context) {
           MUSTTAIL return complete(job, context);
         }
         job->continuation = downLeftSpineTable[c.getType()];
-        __builtin_prefetch(job->n);
         MUSTTAIL return keepGoing(job, context);
       }
     }
@@ -3948,7 +3927,6 @@ PRESERVE_NONE void right_side_iter(Job *job, Context *context) {
     if (c != nullptr) {
       job->n = c;
       job->continuation = downLeftSpineTable[c.getType()];
-      __builtin_prefetch(job->n);
       MUSTTAIL return keepGoing(job, context);
     } else {
       auto c = nextSibling(job->n);
@@ -3958,14 +3936,12 @@ PRESERVE_NONE void right_side_iter(Job *job, Context *context) {
         MUSTTAIL return complete(job, context);
       }
       job->continuation = downLeftSpineTable[c.getType()];
-      __builtin_prefetch(job->n);
       MUSTTAIL return keepGoing(job, context);
     }
   }
 
   job->n = child;
   job->continuation = rightSideIterTable[c.getType()];
-  __builtin_prefetch(job->n);
   MUSTTAIL return keepGoing(job, context);
 }
 
@@ -4187,7 +4163,6 @@ void pointIter(Job *job, Context *context) {
   }
 
   job->continuation = PointIterTable<NodeTTo>::table[job->child.getType()];
-  __builtin_prefetch(job->child);
   MUSTTAIL return keepGoing(job, context);
 }
 
@@ -4287,7 +4262,6 @@ void prefixIter(Job *job, Context *context) {
       goto gotoEndIter;
     } else {
       job->continuation = BeginIterTable<NodeTTo>::table[job->child.getType()];
-      __builtin_prefetch(job->child);
       MUSTTAIL return keepGoing(job, context);
     }
   }
@@ -4299,7 +4273,6 @@ void prefixIter(Job *job, Context *context) {
   }
 
   job->continuation = PrefixIterTable<NodeTTo>::table[job->child.getType()];
-  __builtin_prefetch(job->child);
   MUSTTAIL return keepGoing(job, context);
 
 noNodeOnSearchPath: {
@@ -4326,7 +4299,6 @@ gotoEndIter:
     MUSTTAIL return complete(job, context);
   } else {
     job->continuation = EndIterTable<NodeTTo>::table[job->child.getType()];
-    __builtin_prefetch(job->child);
     MUSTTAIL return keepGoing(job, context);
   }
 }
@@ -4374,7 +4346,6 @@ void beginIter(Job *job, Context *context) {
   }
 
   job->continuation = BeginIterTable<NodeTTo>::table[job->child.getType()];
-  __builtin_prefetch(job->child);
   MUSTTAIL return keepGoing(job, context);
 
 gotoEndIter:
@@ -4441,7 +4412,6 @@ void endIter(Job *job, Context *context) {
   }
 
   job->continuation = EndIterTable<NodeTTo>::table[job->child.getType()];
-  __builtin_prefetch(job->child);
   MUSTTAIL return keepGoing(job, context);
 }