2 Commits

Author SHA1 Message Date
8f03a105bb Use target avx512f,avx512bw
All checks were successful
Tests / Clang total: 1479, passed: 1479
Clang |Total|New|Outstanding|Fixed|Trend |:-:|:-:|:-:|:-:|:-: |0|0|0|0|:clap:
Tests / Debug total: 1477, passed: 1477
Tests / SIMD fallback total: 1479, passed: 1479
Tests / Release [gcc] total: 1479, passed: 1479
GNU C Compiler (gcc) |Total|New|Outstanding|Fixed|Trend |:-:|:-:|:-:|:-:|:-: |0|0|0|0|:clap:
Tests / Release [gcc,aarch64] total: 1102, passed: 1102
Tests / Coverage total: 1111, passed: 1111
Code Coverage #### Project Overview No changes detected, that affect the code coverage. * Line Coverage: 99.60% (1737/1744) * Branch Coverage: 64.99% (1498/2305) * Complexity Density: 0.00 * Lines of Code: 1744 #### Quality Gates Summary Output truncated.
weaselab/conflict-set/pipeline/head This commit looks good
Appears to fix gcc build
2024-08-02 21:47:23 -07:00
0e574856be Make checkMaxBetweenExclusive a multi-version function
This introduces more branches but reduces code size
2024-08-02 21:09:55 -07:00

View File

@@ -1676,7 +1676,7 @@ int firstNeqStride(const uint8_t *ap, const uint8_t *bp) {
// This gets covered in local development // This gets covered in local development
// GCOVR_EXCL_START // GCOVR_EXCL_START
#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__) #if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
__attribute__((target("avx512bw"))) int __attribute__((target("avx512f,avx512bw"))) int
longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) { longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
int i = 0; int i = 0;
int end = cl & ~63; int end = cl & ~63;
@@ -2114,8 +2114,9 @@ scan16(const InternalVersionT *vs, int begin, int end, // GCOVR_EXCL_LINE
// path of n + [child], where child in (begin, end) is <= readVersion. Does not // path of n + [child], where child in (begin, end) is <= readVersion. Does not
// account for the range version of firstGt(searchpath(n) + [end - 1]) // account for the range version of firstGt(searchpath(n) + [end - 1])
template <bool kAVX512> template <bool kAVX512>
bool checkMaxBetweenExclusive(Node *n, int begin, int end, bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) { InternalVersionT readVersion,
ReadContext *tls) {
++tls->range_read_node_scan_accum; ++tls->range_read_node_scan_accum;
assume(-1 <= begin); assume(-1 <= begin);
assume(begin <= 256); assume(begin <= 256);
@@ -2363,6 +2364,22 @@ bool checkMaxBetweenExclusive(Node *n, int begin, int end,
} }
} }
#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
// This gets covered in local development
// GCOVR_EXCL_START
__attribute__((target("avx512f"))) bool
checkMaxBetweenExclusive(Node *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<true>(n, begin, end, readVersion, tls);
}
// GCOVR_EXCL_STOP
__attribute__((target("default")))
#endif
bool checkMaxBetweenExclusive(Node *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
}
Vector<uint8_t> getSearchPath(Arena &arena, Node *n) { Vector<uint8_t> getSearchPath(Arena &arena, Node *n) {
assert(n != nullptr); assert(n != nullptr);
auto result = vector<uint8_t>(arena); auto result = vector<uint8_t>(arena);
@@ -2385,7 +2402,6 @@ Vector<uint8_t> getSearchPath(Arena &arena, Node *n) {
// //
// Precondition: transitively, no child of n has a search path that's a longer // Precondition: transitively, no child of n has a search path that's a longer
// prefix of key than n // prefix of key than n
template <bool kAVX512>
bool checkRangeStartsWith(Node *n, std::span<const uint8_t> key, int begin, bool checkRangeStartsWith(Node *n, std::span<const uint8_t> key, int begin,
int end, InternalVersionT readVersion, int end, InternalVersionT readVersion,
ReadContext *tls) { ReadContext *tls) {
@@ -2395,7 +2411,7 @@ bool checkRangeStartsWith(Node *n, std::span<const uint8_t> key, int begin,
auto remaining = key; auto remaining = key;
auto *impl = tls->impl; auto *impl = tls->impl;
if (remaining.size() == 0) { if (remaining.size() == 0) {
return checkMaxBetweenExclusive<kAVX512>(n, begin, end, readVersion, tls); return checkMaxBetweenExclusive(n, begin, end, readVersion, tls);
} }
auto *child = getChild(n, remaining[0]); auto *child = getChild(n, remaining[0]);
@@ -2454,7 +2470,7 @@ downLeftSpine:
namespace { namespace {
// Return true if the max version among all keys that start with key[:prefixLen] // Return true if the max version among all keys that start with key[:prefixLen]
// that are >= key is <= readVersion // that are >= key is <= readVersion
template <bool kAVX512> struct CheckRangeLeftSide { struct CheckRangeLeftSide {
CheckRangeLeftSide(Node *n, std::span<const uint8_t> key, int prefixLen, CheckRangeLeftSide(Node *n, std::span<const uint8_t> key, int prefixLen,
InternalVersionT readVersion, ReadContext *tls) InternalVersionT readVersion, ReadContext *tls)
: n(n), remaining(key), prefixLen(prefixLen), readVersion(readVersion), : n(n), remaining(key), prefixLen(prefixLen), readVersion(readVersion),
@@ -2487,8 +2503,7 @@ template <bool kAVX512> struct CheckRangeLeftSide {
} }
if (searchPathLen >= prefixLen) { if (searchPathLen >= prefixLen) {
if (!checkMaxBetweenExclusive<kAVX512>(n, remaining[0], 256, readVersion, if (!checkMaxBetweenExclusive(n, remaining[0], 256, readVersion, tls)) {
tls)) {
ok = false; ok = false;
return true; return true;
} }
@@ -2570,7 +2585,7 @@ template <bool kAVX512> struct CheckRangeLeftSide {
// Return true if the max version among all keys that start with key[:prefixLen] // Return true if the max version among all keys that start with key[:prefixLen]
// that are < key is <= readVersion // that are < key is <= readVersion
template <bool kAVX512> struct CheckRangeRightSide { struct CheckRangeRightSide {
CheckRangeRightSide(Node *n, std::span<const uint8_t> key, int prefixLen, CheckRangeRightSide(Node *n, std::span<const uint8_t> key, int prefixLen,
InternalVersionT readVersion, ReadContext *tls) InternalVersionT readVersion, ReadContext *tls)
: n(n), key(key), remaining(key), prefixLen(prefixLen), : n(n), key(key), remaining(key), prefixLen(prefixLen),
@@ -2613,8 +2628,7 @@ template <bool kAVX512> struct CheckRangeRightSide {
return true; return true;
} }
if (!checkMaxBetweenExclusive<kAVX512>(n, -1, remaining[0], readVersion, if (!checkMaxBetweenExclusive(n, -1, remaining[0], readVersion, tls)) {
tls)) {
ok = false; ok = false;
return true; return true;
} }
@@ -2701,10 +2715,9 @@ template <bool kAVX512> struct CheckRangeRightSide {
}; };
} // namespace } // namespace
template <bool kAVX512> bool checkRangeRead(Node *n, std::span<const uint8_t> begin,
bool checkRangeReadImpl(Node *n, std::span<const uint8_t> begin, std::span<const uint8_t> end, InternalVersionT readVersion,
std::span<const uint8_t> end, ReadContext *tls) {
InternalVersionT readVersion, ReadContext *tls) {
int lcp = longestCommonPrefix(begin.data(), end.data(), int lcp = longestCommonPrefix(begin.data(), end.data(),
std::min(begin.size(), end.size())); std::min(begin.size(), end.size()));
if (lcp == int(begin.size()) && end.size() == begin.size() + 1 && if (lcp == int(begin.size()) && end.size() == begin.size() + 1 &&
@@ -2746,22 +2759,19 @@ bool checkRangeReadImpl(Node *n, std::span<const uint8_t> begin,
lcp -= consumed; lcp -= consumed;
if (lcp == int(begin.size())) { if (lcp == int(begin.size())) {
CheckRangeRightSide<kAVX512> checkRangeRightSide{n, end, lcp, readVersion, CheckRangeRightSide checkRangeRightSide{n, end, lcp, readVersion, tls};
tls};
while (!checkRangeRightSide.step()) while (!checkRangeRightSide.step())
; ;
return checkRangeRightSide.ok; return checkRangeRightSide.ok;
} }
if (!checkRangeStartsWith<kAVX512>(n, begin.subspan(0, lcp), begin[lcp], if (!checkRangeStartsWith(n, begin.subspan(0, lcp), begin[lcp], end[lcp],
end[lcp], readVersion, tls)) { readVersion, tls)) {
return false; return false;
} }
CheckRangeLeftSide<kAVX512> checkRangeLeftSide{n, begin, lcp + 1, readVersion, CheckRangeLeftSide checkRangeLeftSide{n, begin, lcp + 1, readVersion, tls};
tls}; CheckRangeRightSide checkRangeRightSide{n, end, lcp + 1, readVersion, tls};
CheckRangeRightSide<kAVX512> checkRangeRightSide{n, end, lcp + 1, readVersion,
tls};
for (;;) { for (;;) {
bool leftDone = checkRangeLeftSide.step(); bool leftDone = checkRangeLeftSide.step();
@@ -2800,45 +2810,8 @@ template __attribute__((always_inline, target("avx512f"))) bool
scan16<true>(const InternalVersionT *vs, int begin, int end, scan16<true>(const InternalVersionT *vs, int begin, int end,
InternalVersionT readVersion); InternalVersionT readVersion);
template __attribute__((target("avx512f"))) bool template __attribute__((target("avx512f"))) bool
checkMaxBetweenExclusive<true>(Node *n, int begin, int end, checkMaxBetweenExclusiveImpl<true>(Node *n, int begin, int end,
InternalVersionT readVersion, ReadContext *); InternalVersionT readVersion, ReadContext *);
template __attribute__((target("avx512f"))) bool
checkRangeStartsWith<true>(Node *n, std::span<const uint8_t> key, int begin,
int end, InternalVersionT readVersion,
ReadContext *);
template __attribute__((target("avx512f"))) bool
CheckRangeLeftSide<true>::step();
template __attribute__((target("avx512f"))) bool
CheckRangeRightSide<true>::step();
template __attribute__((target("avx512f"))) bool
checkRangeReadImpl<true>(Node *n, std::span<const uint8_t> begin,
std::span<const uint8_t> end,
InternalVersionT readVersion, ReadContext *);
#endif
#if defined(__SANITIZE_THREAD__) || !defined(__x86_64__)
bool checkRangeRead(Node *n, std::span<const uint8_t> begin,
std::span<const uint8_t> end, InternalVersionT readVersion,
ReadContext *tls) {
return checkRangeReadImpl<false>(n, begin, end, readVersion, tls);
}
#else
// Only one of these is ever exercised. I'm not worried about somehow not
// calling one of these though.
// GCOVR_EXCL_START
__attribute__((target("default"))) bool
checkRangeRead(Node *n, std::span<const uint8_t> begin,
std::span<const uint8_t> end, InternalVersionT readVersion,
ReadContext *tls) {
return checkRangeReadImpl<false>(n, begin, end, readVersion, tls);
}
__attribute__((target("avx512f"))) bool
checkRangeRead(Node *n, std::span<const uint8_t> begin,
std::span<const uint8_t> end, InternalVersionT readVersion,
ReadContext *tls) {
return checkRangeReadImpl<true>(n, begin, end, readVersion, tls);
}
// GCOVR_EXCL_STOP
#endif #endif
// Returns a pointer to the newly inserted node. Caller must set // Returns a pointer to the newly inserted node. Caller must set