Compare commits
2 Commits
493a6572ad
...
8f03a105bb
Author | SHA1 | Date | |
---|---|---|---|
8f03a105bb | |||
0e574856be |
@@ -1676,7 +1676,7 @@ int firstNeqStride(const uint8_t *ap, const uint8_t *bp) {
|
|||||||
// This gets covered in local development
|
// This gets covered in local development
|
||||||
// GCOVR_EXCL_START
|
// GCOVR_EXCL_START
|
||||||
#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
|
#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
|
||||||
__attribute__((target("avx512bw"))) int
|
__attribute__((target("avx512f,avx512bw"))) int
|
||||||
longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
|
longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
|
||||||
int i = 0;
|
int i = 0;
|
||||||
int end = cl & ~63;
|
int end = cl & ~63;
|
||||||
@@ -2114,8 +2114,9 @@ scan16(const InternalVersionT *vs, int begin, int end, // GCOVR_EXCL_LINE
|
|||||||
// path of n + [child], where child in (begin, end) is <= readVersion. Does not
|
// path of n + [child], where child in (begin, end) is <= readVersion. Does not
|
||||||
// account for the range version of firstGt(searchpath(n) + [end - 1])
|
// account for the range version of firstGt(searchpath(n) + [end - 1])
|
||||||
template <bool kAVX512>
|
template <bool kAVX512>
|
||||||
bool checkMaxBetweenExclusive(Node *n, int begin, int end,
|
bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
|
||||||
InternalVersionT readVersion, ReadContext *tls) {
|
InternalVersionT readVersion,
|
||||||
|
ReadContext *tls) {
|
||||||
++tls->range_read_node_scan_accum;
|
++tls->range_read_node_scan_accum;
|
||||||
assume(-1 <= begin);
|
assume(-1 <= begin);
|
||||||
assume(begin <= 256);
|
assume(begin <= 256);
|
||||||
@@ -2363,6 +2364,22 @@ bool checkMaxBetweenExclusive(Node *n, int begin, int end,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
|
||||||
|
// This gets covered in local development
|
||||||
|
// GCOVR_EXCL_START
|
||||||
|
__attribute__((target("avx512f"))) bool
|
||||||
|
checkMaxBetweenExclusive(Node *n, int begin, int end,
|
||||||
|
InternalVersionT readVersion, ReadContext *tls) {
|
||||||
|
return checkMaxBetweenExclusiveImpl<true>(n, begin, end, readVersion, tls);
|
||||||
|
}
|
||||||
|
// GCOVR_EXCL_STOP
|
||||||
|
__attribute__((target("default")))
|
||||||
|
#endif
|
||||||
|
bool checkMaxBetweenExclusive(Node *n, int begin, int end,
|
||||||
|
InternalVersionT readVersion, ReadContext *tls) {
|
||||||
|
return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
|
||||||
|
}
|
||||||
|
|
||||||
Vector<uint8_t> getSearchPath(Arena &arena, Node *n) {
|
Vector<uint8_t> getSearchPath(Arena &arena, Node *n) {
|
||||||
assert(n != nullptr);
|
assert(n != nullptr);
|
||||||
auto result = vector<uint8_t>(arena);
|
auto result = vector<uint8_t>(arena);
|
||||||
@@ -2385,7 +2402,6 @@ Vector<uint8_t> getSearchPath(Arena &arena, Node *n) {
|
|||||||
//
|
//
|
||||||
// Precondition: transitively, no child of n has a search path that's a longer
|
// Precondition: transitively, no child of n has a search path that's a longer
|
||||||
// prefix of key than n
|
// prefix of key than n
|
||||||
template <bool kAVX512>
|
|
||||||
bool checkRangeStartsWith(Node *n, std::span<const uint8_t> key, int begin,
|
bool checkRangeStartsWith(Node *n, std::span<const uint8_t> key, int begin,
|
||||||
int end, InternalVersionT readVersion,
|
int end, InternalVersionT readVersion,
|
||||||
ReadContext *tls) {
|
ReadContext *tls) {
|
||||||
@@ -2395,7 +2411,7 @@ bool checkRangeStartsWith(Node *n, std::span<const uint8_t> key, int begin,
|
|||||||
auto remaining = key;
|
auto remaining = key;
|
||||||
auto *impl = tls->impl;
|
auto *impl = tls->impl;
|
||||||
if (remaining.size() == 0) {
|
if (remaining.size() == 0) {
|
||||||
return checkMaxBetweenExclusive<kAVX512>(n, begin, end, readVersion, tls);
|
return checkMaxBetweenExclusive(n, begin, end, readVersion, tls);
|
||||||
}
|
}
|
||||||
|
|
||||||
auto *child = getChild(n, remaining[0]);
|
auto *child = getChild(n, remaining[0]);
|
||||||
@@ -2454,7 +2470,7 @@ downLeftSpine:
|
|||||||
namespace {
|
namespace {
|
||||||
// Return true if the max version among all keys that start with key[:prefixLen]
|
// Return true if the max version among all keys that start with key[:prefixLen]
|
||||||
// that are >= key is <= readVersion
|
// that are >= key is <= readVersion
|
||||||
template <bool kAVX512> struct CheckRangeLeftSide {
|
struct CheckRangeLeftSide {
|
||||||
CheckRangeLeftSide(Node *n, std::span<const uint8_t> key, int prefixLen,
|
CheckRangeLeftSide(Node *n, std::span<const uint8_t> key, int prefixLen,
|
||||||
InternalVersionT readVersion, ReadContext *tls)
|
InternalVersionT readVersion, ReadContext *tls)
|
||||||
: n(n), remaining(key), prefixLen(prefixLen), readVersion(readVersion),
|
: n(n), remaining(key), prefixLen(prefixLen), readVersion(readVersion),
|
||||||
@@ -2487,8 +2503,7 @@ template <bool kAVX512> struct CheckRangeLeftSide {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (searchPathLen >= prefixLen) {
|
if (searchPathLen >= prefixLen) {
|
||||||
if (!checkMaxBetweenExclusive<kAVX512>(n, remaining[0], 256, readVersion,
|
if (!checkMaxBetweenExclusive(n, remaining[0], 256, readVersion, tls)) {
|
||||||
tls)) {
|
|
||||||
ok = false;
|
ok = false;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -2570,7 +2585,7 @@ template <bool kAVX512> struct CheckRangeLeftSide {
|
|||||||
|
|
||||||
// Return true if the max version among all keys that start with key[:prefixLen]
|
// Return true if the max version among all keys that start with key[:prefixLen]
|
||||||
// that are < key is <= readVersion
|
// that are < key is <= readVersion
|
||||||
template <bool kAVX512> struct CheckRangeRightSide {
|
struct CheckRangeRightSide {
|
||||||
CheckRangeRightSide(Node *n, std::span<const uint8_t> key, int prefixLen,
|
CheckRangeRightSide(Node *n, std::span<const uint8_t> key, int prefixLen,
|
||||||
InternalVersionT readVersion, ReadContext *tls)
|
InternalVersionT readVersion, ReadContext *tls)
|
||||||
: n(n), key(key), remaining(key), prefixLen(prefixLen),
|
: n(n), key(key), remaining(key), prefixLen(prefixLen),
|
||||||
@@ -2613,8 +2628,7 @@ template <bool kAVX512> struct CheckRangeRightSide {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!checkMaxBetweenExclusive<kAVX512>(n, -1, remaining[0], readVersion,
|
if (!checkMaxBetweenExclusive(n, -1, remaining[0], readVersion, tls)) {
|
||||||
tls)) {
|
|
||||||
ok = false;
|
ok = false;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -2701,10 +2715,9 @@ template <bool kAVX512> struct CheckRangeRightSide {
|
|||||||
};
|
};
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
template <bool kAVX512>
|
bool checkRangeRead(Node *n, std::span<const uint8_t> begin,
|
||||||
bool checkRangeReadImpl(Node *n, std::span<const uint8_t> begin,
|
std::span<const uint8_t> end, InternalVersionT readVersion,
|
||||||
std::span<const uint8_t> end,
|
ReadContext *tls) {
|
||||||
InternalVersionT readVersion, ReadContext *tls) {
|
|
||||||
int lcp = longestCommonPrefix(begin.data(), end.data(),
|
int lcp = longestCommonPrefix(begin.data(), end.data(),
|
||||||
std::min(begin.size(), end.size()));
|
std::min(begin.size(), end.size()));
|
||||||
if (lcp == int(begin.size()) && end.size() == begin.size() + 1 &&
|
if (lcp == int(begin.size()) && end.size() == begin.size() + 1 &&
|
||||||
@@ -2746,22 +2759,19 @@ bool checkRangeReadImpl(Node *n, std::span<const uint8_t> begin,
|
|||||||
lcp -= consumed;
|
lcp -= consumed;
|
||||||
|
|
||||||
if (lcp == int(begin.size())) {
|
if (lcp == int(begin.size())) {
|
||||||
CheckRangeRightSide<kAVX512> checkRangeRightSide{n, end, lcp, readVersion,
|
CheckRangeRightSide checkRangeRightSide{n, end, lcp, readVersion, tls};
|
||||||
tls};
|
|
||||||
while (!checkRangeRightSide.step())
|
while (!checkRangeRightSide.step())
|
||||||
;
|
;
|
||||||
return checkRangeRightSide.ok;
|
return checkRangeRightSide.ok;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!checkRangeStartsWith<kAVX512>(n, begin.subspan(0, lcp), begin[lcp],
|
if (!checkRangeStartsWith(n, begin.subspan(0, lcp), begin[lcp], end[lcp],
|
||||||
end[lcp], readVersion, tls)) {
|
readVersion, tls)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
CheckRangeLeftSide<kAVX512> checkRangeLeftSide{n, begin, lcp + 1, readVersion,
|
CheckRangeLeftSide checkRangeLeftSide{n, begin, lcp + 1, readVersion, tls};
|
||||||
tls};
|
CheckRangeRightSide checkRangeRightSide{n, end, lcp + 1, readVersion, tls};
|
||||||
CheckRangeRightSide<kAVX512> checkRangeRightSide{n, end, lcp + 1, readVersion,
|
|
||||||
tls};
|
|
||||||
|
|
||||||
for (;;) {
|
for (;;) {
|
||||||
bool leftDone = checkRangeLeftSide.step();
|
bool leftDone = checkRangeLeftSide.step();
|
||||||
@@ -2800,45 +2810,8 @@ template __attribute__((always_inline, target("avx512f"))) bool
|
|||||||
scan16<true>(const InternalVersionT *vs, int begin, int end,
|
scan16<true>(const InternalVersionT *vs, int begin, int end,
|
||||||
InternalVersionT readVersion);
|
InternalVersionT readVersion);
|
||||||
template __attribute__((target("avx512f"))) bool
|
template __attribute__((target("avx512f"))) bool
|
||||||
checkMaxBetweenExclusive<true>(Node *n, int begin, int end,
|
checkMaxBetweenExclusiveImpl<true>(Node *n, int begin, int end,
|
||||||
InternalVersionT readVersion, ReadContext *);
|
InternalVersionT readVersion, ReadContext *);
|
||||||
template __attribute__((target("avx512f"))) bool
|
|
||||||
checkRangeStartsWith<true>(Node *n, std::span<const uint8_t> key, int begin,
|
|
||||||
int end, InternalVersionT readVersion,
|
|
||||||
ReadContext *);
|
|
||||||
template __attribute__((target("avx512f"))) bool
|
|
||||||
CheckRangeLeftSide<true>::step();
|
|
||||||
template __attribute__((target("avx512f"))) bool
|
|
||||||
CheckRangeRightSide<true>::step();
|
|
||||||
template __attribute__((target("avx512f"))) bool
|
|
||||||
checkRangeReadImpl<true>(Node *n, std::span<const uint8_t> begin,
|
|
||||||
std::span<const uint8_t> end,
|
|
||||||
InternalVersionT readVersion, ReadContext *);
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#if defined(__SANITIZE_THREAD__) || !defined(__x86_64__)
|
|
||||||
bool checkRangeRead(Node *n, std::span<const uint8_t> begin,
|
|
||||||
std::span<const uint8_t> end, InternalVersionT readVersion,
|
|
||||||
ReadContext *tls) {
|
|
||||||
return checkRangeReadImpl<false>(n, begin, end, readVersion, tls);
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
// Only one of these is ever exercised. I'm not worried about somehow not
|
|
||||||
// calling one of these though.
|
|
||||||
// GCOVR_EXCL_START
|
|
||||||
__attribute__((target("default"))) bool
|
|
||||||
checkRangeRead(Node *n, std::span<const uint8_t> begin,
|
|
||||||
std::span<const uint8_t> end, InternalVersionT readVersion,
|
|
||||||
ReadContext *tls) {
|
|
||||||
return checkRangeReadImpl<false>(n, begin, end, readVersion, tls);
|
|
||||||
}
|
|
||||||
__attribute__((target("avx512f"))) bool
|
|
||||||
checkRangeRead(Node *n, std::span<const uint8_t> begin,
|
|
||||||
std::span<const uint8_t> end, InternalVersionT readVersion,
|
|
||||||
ReadContext *tls) {
|
|
||||||
return checkRangeReadImpl<true>(n, begin, end, readVersion, tls);
|
|
||||||
}
|
|
||||||
// GCOVR_EXCL_STOP
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
// Returns a pointer to the newly inserted node. Caller must set
|
// Returns a pointer to the newly inserted node. Caller must set
|
||||||
|
Reference in New Issue
Block a user