diff --git a/ConflictSet.cpp b/ConflictSet.cpp index 5e6ae8c..16b3000 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -2114,8 +2114,9 @@ scan16(const InternalVersionT *vs, int begin, int end, // GCOVR_EXCL_LINE // path of n + [child], where child in (begin, end) is <= readVersion. Does not // account for the range version of firstGt(searchpath(n) + [end - 1]) template -bool checkMaxBetweenExclusive(Node *n, int begin, int end, - InternalVersionT readVersion, ReadContext *tls) { +bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end, + InternalVersionT readVersion, + ReadContext *tls) { ++tls->range_read_node_scan_accum; assume(-1 <= begin); assume(begin <= 256); @@ -2363,6 +2364,22 @@ bool checkMaxBetweenExclusive(Node *n, int begin, int end, } } +#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__) +// This gets covered in local development +// GCOVR_EXCL_START +__attribute__((target("avx512f"))) bool +checkMaxBetweenExclusive(Node *n, int begin, int end, + InternalVersionT readVersion, ReadContext *tls) { + return checkMaxBetweenExclusiveImpl(n, begin, end, readVersion, tls); +} +// GCOVR_EXCL_STOP +__attribute__((target("default"))) +#endif +bool checkMaxBetweenExclusive(Node *n, int begin, int end, + InternalVersionT readVersion, ReadContext *tls) { + return checkMaxBetweenExclusiveImpl(n, begin, end, readVersion, tls); +} + Vector getSearchPath(Arena &arena, Node *n) { assert(n != nullptr); auto result = vector(arena); @@ -2385,7 +2402,6 @@ Vector getSearchPath(Arena &arena, Node *n) { // // Precondition: transitively, no child of n has a search path that's a longer // prefix of key than n -template bool checkRangeStartsWith(Node *n, std::span key, int begin, int end, InternalVersionT readVersion, ReadContext *tls) { @@ -2395,7 +2411,7 @@ bool checkRangeStartsWith(Node *n, std::span key, int begin, auto remaining = key; auto *impl = tls->impl; if (remaining.size() == 0) { - return checkMaxBetweenExclusive(n, begin, end, readVersion, tls); + return checkMaxBetweenExclusive(n, begin, end, readVersion, tls); } auto *child = getChild(n, remaining[0]); @@ -2454,7 +2470,7 @@ downLeftSpine: namespace { // Return true if the max version among all keys that start with key[:prefixLen] // that are >= key is <= readVersion -template struct CheckRangeLeftSide { +struct CheckRangeLeftSide { CheckRangeLeftSide(Node *n, std::span key, int prefixLen, InternalVersionT readVersion, ReadContext *tls) : n(n), remaining(key), prefixLen(prefixLen), readVersion(readVersion), @@ -2487,8 +2503,7 @@ template struct CheckRangeLeftSide { } if (searchPathLen >= prefixLen) { - if (!checkMaxBetweenExclusive(n, remaining[0], 256, readVersion, - tls)) { + if (!checkMaxBetweenExclusive(n, remaining[0], 256, readVersion, tls)) { ok = false; return true; } @@ -2570,7 +2585,7 @@ template struct CheckRangeLeftSide { // Return true if the max version among all keys that start with key[:prefixLen] // that are < key is <= readVersion -template struct CheckRangeRightSide { +struct CheckRangeRightSide { CheckRangeRightSide(Node *n, std::span key, int prefixLen, InternalVersionT readVersion, ReadContext *tls) : n(n), key(key), remaining(key), prefixLen(prefixLen), @@ -2613,8 +2628,7 @@ template struct CheckRangeRightSide { return true; } - if (!checkMaxBetweenExclusive(n, -1, remaining[0], readVersion, - tls)) { + if (!checkMaxBetweenExclusive(n, -1, remaining[0], readVersion, tls)) { ok = false; return true; } @@ -2701,10 +2715,9 @@ template struct CheckRangeRightSide { }; } // namespace -template -bool checkRangeReadImpl(Node *n, std::span begin, - std::span end, - InternalVersionT readVersion, ReadContext *tls) { +bool checkRangeRead(Node *n, std::span begin, + std::span end, InternalVersionT readVersion, + ReadContext *tls) { int lcp = longestCommonPrefix(begin.data(), end.data(), std::min(begin.size(), end.size())); if (lcp == int(begin.size()) && end.size() == begin.size() + 1 && @@ -2746,22 +2759,19 @@ bool checkRangeReadImpl(Node *n, std::span begin, lcp -= consumed; if (lcp == int(begin.size())) { - CheckRangeRightSide checkRangeRightSide{n, end, lcp, readVersion, - tls}; + CheckRangeRightSide checkRangeRightSide{n, end, lcp, readVersion, tls}; while (!checkRangeRightSide.step()) ; return checkRangeRightSide.ok; } - if (!checkRangeStartsWith(n, begin.subspan(0, lcp), begin[lcp], - end[lcp], readVersion, tls)) { + if (!checkRangeStartsWith(n, begin.subspan(0, lcp), begin[lcp], end[lcp], + readVersion, tls)) { return false; } - CheckRangeLeftSide checkRangeLeftSide{n, begin, lcp + 1, readVersion, - tls}; - CheckRangeRightSide checkRangeRightSide{n, end, lcp + 1, readVersion, - tls}; + CheckRangeLeftSide checkRangeLeftSide{n, begin, lcp + 1, readVersion, tls}; + CheckRangeRightSide checkRangeRightSide{n, end, lcp + 1, readVersion, tls}; for (;;) { bool leftDone = checkRangeLeftSide.step(); @@ -2800,45 +2810,8 @@ template __attribute__((always_inline, target("avx512f"))) bool scan16(const InternalVersionT *vs, int begin, int end, InternalVersionT readVersion); template __attribute__((target("avx512f"))) bool -checkMaxBetweenExclusive(Node *n, int begin, int end, - InternalVersionT readVersion, ReadContext *); -template __attribute__((target("avx512f"))) bool -checkRangeStartsWith(Node *n, std::span key, int begin, - int end, InternalVersionT readVersion, - ReadContext *); -template __attribute__((target("avx512f"))) bool -CheckRangeLeftSide::step(); -template __attribute__((target("avx512f"))) bool -CheckRangeRightSide::step(); -template __attribute__((target("avx512f"))) bool -checkRangeReadImpl(Node *n, std::span begin, - std::span end, - InternalVersionT readVersion, ReadContext *); -#endif - -#if defined(__SANITIZE_THREAD__) || !defined(__x86_64__) -bool checkRangeRead(Node *n, std::span begin, - std::span end, InternalVersionT readVersion, - ReadContext *tls) { - return checkRangeReadImpl(n, begin, end, readVersion, tls); -} -#else -// Only one of these is ever exercised. I'm not worried about somehow not -// calling one of these though. -// GCOVR_EXCL_START -__attribute__((target("default"))) bool -checkRangeRead(Node *n, std::span begin, - std::span end, InternalVersionT readVersion, - ReadContext *tls) { - return checkRangeReadImpl(n, begin, end, readVersion, tls); -} -__attribute__((target("avx512f"))) bool -checkRangeRead(Node *n, std::span begin, - std::span end, InternalVersionT readVersion, - ReadContext *tls) { - return checkRangeReadImpl(n, begin, end, readVersion, tls); -} -// GCOVR_EXCL_STOP +checkMaxBetweenExclusiveImpl(Node *n, int begin, int end, + InternalVersionT readVersion, ReadContext *); #endif // Returns a pointer to the newly inserted node. Caller must set