weaselab/conflict-set
2
0
Fork 0
Code Issues 4 Packages Releases 13 Activity

Specific node overloads for checkMaxBetweenExclusive

Browse Source
This commit is contained in:
Andrew Noyes
2024-10-14 16:54:44 -07:00
parent 824037bf32
commit 11e8717da8
1 changed files with 315 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

175
ConflictSet.cpp
View File

@@ -2147,7 +2147,14 @@ bool scan16(const InternalVersionT *vs, int begin, int end,
// path of n + [child], where child in (begin, end) is <= readVersion. Does not
// account for the range version of firstGt(searchpath(n) + [end - 1])
template <bool kAVX512>
bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
bool checkMaxBetweenExclusiveImpl(Node0 *, int, int, InternalVersionT,
ReadContext *tls) {
++tls->range_read_node_scan_accum;
return true;
}
template <bool kAVX512>
bool checkMaxBetweenExclusiveImpl(Node3 *n, int begin, int end,
InternalVersionT readVersion,
ReadContext *tls) {
++tls->range_read_node_scan_accum;
@@ -2156,22 +2163,15 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
assume(-1 <= end);
assume(end <= 256);
assume(begin < end);
assert(!(begin == -1 && end == 256));
switch (n->getType()) {
case Type_Node0:
return true;
case Type_Node3: {
auto *self = static_cast<Node3 *>(n);
++begin;
const unsigned shiftUpperBound = end - begin;
const unsigned shiftAmount = begin;
auto inBounds = [&](unsigned c) {
return c - shiftAmount < shiftUpperBound;
};
auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
uint32_t mask = 0;
for (int i = 0; i < Node3::kMaxNodes; ++i) {
@@ -2190,8 +2190,20 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
}
return !(compared & mask) && firstRangeOk;
}
case Type_Node16: {
}
template <bool kAVX512>
bool checkMaxBetweenExclusiveImpl(Node16 *n, int begin, int end,
InternalVersionT readVersion,
ReadContext *tls) {
++tls->range_read_node_scan_accum;
assume(-1 <= begin);
assume(begin <= 256);
assume(-1 <= end);
assume(end <= 256);
assume(begin < end);
assert(!(begin == -1 && end == 256));
auto *self = static_cast<Node16 *>(n);
++begin;
@@ -2231,8 +2243,8 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
uint16x4_t conflicting[4];
for (int i = 0; i < 4; ++i) {
conflicting[i] = vmovn_u32(
vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
conflicting[i] =
vmovn_u32(vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
}
auto combined =
vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
@@ -2249,8 +2261,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
memcpy(&indices, self->index, 16);
indices = _mm_sub_epi8(indices, _mm_set1_epi8(begin));
uint32_t mask =
0xffff &
~_mm_movemask_epi8(_mm_cmpeq_epi8(
0xffff & ~_mm_movemask_epi8(_mm_cmpeq_epi8(
indices, _mm_max_epu8(indices, _mm_set1_epi8(end - begin))));
mask &= (1 << self->numChildren) - 1;
if (!mask) {
@@ -2272,9 +2283,7 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
const unsigned shiftUpperBound = end - begin;
const unsigned shiftAmount = begin;
auto inBounds = [&](unsigned c) {
return c - shiftAmount < shiftUpperBound;
};
auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
uint32_t mask = 0;
for (int i = 0; i < 16; ++i) {
@@ -2294,8 +2303,20 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
return !(compared & mask) && firstRangeOk;
#endif
}
case Type_Node48: {
}
template <bool kAVX512>
bool checkMaxBetweenExclusiveImpl(Node48 *n, int begin, int end,
InternalVersionT readVersion,
ReadContext *tls) {
++tls->range_read_node_scan_accum;
assume(-1 <= begin);
assume(begin <= 256);
assume(-1 <= end);
assume(end <= 256);
assume(begin < end);
assert(!(begin == -1 && end == 256));
auto *self = static_cast<Node48 *>(n);
{
@@ -2327,8 +2348,20 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
}
}
return true;
}
case Type_Node256: {
}
template <bool kAVX512>
bool checkMaxBetweenExclusiveImpl(Node256 *n, int begin, int end,
InternalVersionT readVersion,
ReadContext *tls) {
++tls->range_read_node_scan_accum;
assume(-1 <= begin);
assume(begin <= 256);
assume(-1 <= end);
assume(end <= 256);
assume(begin < end);
assert(!(begin == -1 && end == 256));
static_assert(Node256::kMaxOfMaxTotalPages == 16);
auto *self = static_cast<Node256 *>(n);
@@ -2380,8 +2413,32 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
}
}
// Check inner pages
return scan16<kAVX512>(self->maxOfMax, firstPage + 1, lastPage,
readVersion);
return scan16<kAVX512>(self->maxOfMax, firstPage + 1, lastPage, readVersion);
}
template <bool kAVX512>
bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
InternalVersionT readVersion,
ReadContext *tls) {
switch (n->getType()) {
case Type_Node0:
return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node0 *>(n), begin,
end, readVersion, tls);
case Type_Node3: {
return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node3 *>(n), begin,
end, readVersion, tls);
}
case Type_Node16: {
return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node16 *>(n),
begin, end, readVersion, tls);
}
case Type_Node48: {
return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node48 *>(n),
begin, end, readVersion, tls);
}
case Type_Node256: {
return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node256 *>(n),
begin, end, readVersion, tls);
}
default: // GCOVR_EXCL_LINE
__builtin_unreachable(); // GCOVR_EXCL_LINE
@@ -2402,6 +2459,58 @@ bool checkMaxBetweenExclusive(Node *n, int begin, int end,
return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
}
bool checkMaxBetweenExclusive(Node0 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
}
bool checkMaxBetweenExclusive(Node3 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
}
#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
__attribute__((target("avx512f"))) bool
checkMaxBetweenExclusive(Node16 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<true>(n, begin, end, readVersion, tls);
}
__attribute__((target("default")))
#endif
bool checkMaxBetweenExclusive(Node16 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
}
#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
__attribute__((target("avx512f"))) bool
checkMaxBetweenExclusive(Node48 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<true>(n, begin, end, readVersion, tls);
}
__attribute__((target("default")))
#endif
bool checkMaxBetweenExclusive(Node48 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
}
#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
__attribute__((target("avx512f"))) bool
checkMaxBetweenExclusive(Node256 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<true>(n, begin, end, readVersion, tls);
}
__attribute__((target("default")))
#endif
bool checkMaxBetweenExclusive(Node256 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
}
Vector<uint8_t> getSearchPath(Arena &arena, Node *n) {
assert(n != nullptr);
auto result = vector<uint8_t>(arena);
@@ -2424,7 +2533,8 @@ Vector<uint8_t> getSearchPath(Arena &arena, Node *n) {
//
// Precondition: transitively, no child of n has a search path that's a longer
// prefix of key than n
bool checkRangeStartsWith(Node *n, std::span<const uint8_t> key, int begin,
template <class NodeT>
bool checkRangeStartsWith(NodeT *n, std::span<const uint8_t> key, int begin,
int end, InternalVersionT readVersion,
ReadContext *tls) {
#if DEBUG_VERBOSE && !defined(NDEBUG)
@@ -2501,6 +2611,15 @@ scan16<true>(const InternalVersionT *vs, int begin, int end,
template __attribute__((target("avx512f"))) bool
checkMaxBetweenExclusiveImpl<true>(Node *n, int begin, int end,
InternalVersionT readVersion, ReadContext *);
template __attribute__((target("avx512f"))) bool
checkMaxBetweenExclusiveImpl<true>(Node16 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *);
template __attribute__((target("avx512f"))) bool
checkMaxBetweenExclusiveImpl<true>(Node48 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *);
template __attribute__((target("avx512f"))) bool
checkMaxBetweenExclusiveImpl<true>(Node256 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *);
#endif
// Returns a pointer the pointer to the newly inserted node in the tree. Caller
@@ -3207,14 +3326,16 @@ PRESERVE_NONE void begin(CheckJob *job, CheckContext *context) {
job->remaining = job->begin.subspan(0, job->lcp);
if (job->remaining.size() == 0) {
MUSTTAIL return done_common_prefix_iter(job, context);
job->continuation = done_common_prefix_iter;
MUSTTAIL return job->continuation(job, context);
}
auto [c, maxV] = getChildAndMaxVersion(job->n, job->remaining[0]);
job->maxV = maxV;
job->child = c;
if (job->child == nullptr) {
MUSTTAIL return done_common_prefix_iter(job, context);
job->continuation = done_common_prefix_iter;
MUSTTAIL return job->continuation(job, context);
}
job->continuation = commonPrefixIterTable[c.getType()];