weaselab/conflict-set
2
0
Fork 0
Code Issues 4 Packages Releases 13 Activity

Specific node overloads for checkMaxBetweenExclusive

Browse Source
This commit is contained in:
Andrew Noyes
2024-10-14 16:54:44 -07:00
parent 824037bf32
commit 11e8717da8
1 changed files with 315 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

509
ConflictSet.cpp
View File

@@ -2147,7 +2147,14 @@ bool scan16(const InternalVersionT *vs, int begin, int end,
// path of n + [child], where child in (begin, end) is <= readVersion. Does not // path of n + [child], where child in (begin, end) is <= readVersion. Does not
// account for the range version of firstGt(searchpath(n) + [end - 1]) // account for the range version of firstGt(searchpath(n) + [end - 1])
template <bool kAVX512> template <bool kAVX512>
bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end, bool checkMaxBetweenExclusiveImpl(Node0 *, int, int, InternalVersionT,
ReadContext *tls) {
++tls->range_read_node_scan_accum;
return true;
}
template <bool kAVX512>
bool checkMaxBetweenExclusiveImpl(Node3 *n, int begin, int end,
InternalVersionT readVersion, InternalVersionT readVersion,
ReadContext *tls) { ReadContext *tls) {
++tls->range_read_node_scan_accum; ++tls->range_read_node_scan_accum;
@@ -2156,232 +2163,282 @@ bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
assume(-1 <= end); assume(-1 <= end);
assume(end <= 256); assume(end <= 256);
assume(begin < end); assume(begin < end);
assert(!(begin == -1 && end == 256)); assert(!(begin == -1 && end == 256));
switch (n->getType()) { auto *self = static_cast<Node3 *>(n);
case Type_Node0:
return true;
case Type_Node3: {
auto *self = static_cast<Node3 *>(n);
++begin; ++begin;
const unsigned shiftUpperBound = end - begin; const unsigned shiftUpperBound = end - begin;
const unsigned shiftAmount = begin; const unsigned shiftAmount = begin;
auto inBounds = [&](unsigned c) { auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
return c - shiftAmount < shiftUpperBound;
};
uint32_t mask = 0; uint32_t mask = 0;
for (int i = 0; i < Node3::kMaxNodes; ++i) { for (int i = 0; i < Node3::kMaxNodes; ++i) {
mask |= inBounds(self->index[i]) << i; mask |= inBounds(self->index[i]) << i;
} }
mask &= (1 << self->numChildren) - 1; mask &= (1 << self->numChildren) - 1;
if (!mask) { if (!mask) {
return true; return true;
} }
Node *child = self->children[std::countr_zero(mask)]; Node *child = self->children[std::countr_zero(mask)];
const bool firstRangeOk = const bool firstRangeOk =
!child->entryPresent || child->entry.rangeVersion <= readVersion; !child->entryPresent || child->entry.rangeVersion <= readVersion;
uint32_t compared = 0; uint32_t compared = 0;
for (int i = 0; i < Node3::kMaxNodes; ++i) { for (int i = 0; i < Node3::kMaxNodes; ++i) {
compared |= (self->childMaxVersion[i] > readVersion) << i; compared |= (self->childMaxVersion[i] > readVersion) << i;
}
return !(compared & mask) && firstRangeOk;
} }
case Type_Node16: {
auto *self = static_cast<Node16 *>(n);
++begin; return !(compared & mask) && firstRangeOk;
}
assert(begin <= end); template <bool kAVX512>
assert(end - begin < 256); bool checkMaxBetweenExclusiveImpl(Node16 *n, int begin, int end,
InternalVersionT readVersion,
ReadContext *tls) {
++tls->range_read_node_scan_accum;
assume(-1 <= begin);
assume(begin <= 256);
assume(-1 <= end);
assume(end <= 256);
assume(begin < end);
assert(!(begin == -1 && end == 256));
auto *self = static_cast<Node16 *>(n);
++begin;
assert(begin <= end);
assert(end - begin < 256);
#ifdef HAS_ARM_NEON #ifdef HAS_ARM_NEON
uint8x16_t indices; uint8x16_t indices;
memcpy(&indices, self->index, 16); memcpy(&indices, self->index, 16);
// 0xff for each in bounds // 0xff for each in bounds
auto results = auto results =
vcltq_u8(vsubq_u8(indices, vdupq_n_u8(begin)), vdupq_n_u8(end - begin)); vcltq_u8(vsubq_u8(indices, vdupq_n_u8(begin)), vdupq_n_u8(end - begin));
// 0xf for each 0xff // 0xf for each 0xff
uint64_t mask = vget_lane_u64( uint64_t mask = vget_lane_u64(
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(results), 4)), 0); vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(results), 4)), 0);
mask &= self->numChildren == 16 mask &= self->numChildren == 16
? uint64_t(-1) ? uint64_t(-1)
: (uint64_t(1) << (self->numChildren << 2)) - 1; : (uint64_t(1) << (self->numChildren << 2)) - 1;
if (!mask) { if (!mask) {
return true; return true;
} }
Node *child = self->children[std::countr_zero(mask) >> 2]; Node *child = self->children[std::countr_zero(mask) >> 2];
const bool firstRangeOk = const bool firstRangeOk =
!child->entryPresent || child->entry.rangeVersion <= readVersion; !child->entryPresent || child->entry.rangeVersion <= readVersion;
uint32x4_t w4[4]; uint32x4_t w4[4];
memcpy(w4, self->childMaxVersion, sizeof(w4)); memcpy(w4, self->childMaxVersion, sizeof(w4));
uint32_t rv; uint32_t rv;
memcpy(&rv, &readVersion, sizeof(rv)); memcpy(&rv, &readVersion, sizeof(rv));
const auto rvVec = vdupq_n_u32(rv); const auto rvVec = vdupq_n_u32(rv);
int32x4_t z; int32x4_t z;
memset(&z, 0, sizeof(z)); memset(&z, 0, sizeof(z));
uint16x4_t conflicting[4]; uint16x4_t conflicting[4];
for (int i = 0; i < 4; ++i) { for (int i = 0; i < 4; ++i) {
conflicting[i] = vmovn_u32( conflicting[i] =
vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z)); vmovn_u32(vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
} }
auto combined = auto combined =
vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])), vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
vmovn_u16(vcombine_u16(conflicting[2], conflicting[3]))); vmovn_u16(vcombine_u16(conflicting[2], conflicting[3])));
uint64_t compared = vget_lane_u64( uint64_t compared = vget_lane_u64(
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(combined), 4)), 0); vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(combined), 4)), 0);
return !(compared & mask) && firstRangeOk; return !(compared & mask) && firstRangeOk;
#elif defined(HAS_AVX) #elif defined(HAS_AVX)
__m128i indices; __m128i indices;
memcpy(&indices, self->index, 16); memcpy(&indices, self->index, 16);
indices = _mm_sub_epi8(indices, _mm_set1_epi8(begin)); indices = _mm_sub_epi8(indices, _mm_set1_epi8(begin));
uint32_t mask = uint32_t mask =
0xffff & 0xffff & ~_mm_movemask_epi8(_mm_cmpeq_epi8(
~_mm_movemask_epi8(_mm_cmpeq_epi8( indices, _mm_max_epu8(indices, _mm_set1_epi8(end - begin))));
indices, _mm_max_epu8(indices, _mm_set1_epi8(end - begin)))); mask &= (1 << self->numChildren) - 1;
mask &= (1 << self->numChildren) - 1; if (!mask) {
if (!mask) { return true;
return true; }
} Node *child = self->children[std::countr_zero(mask)];
Node *child = self->children[std::countr_zero(mask)]; const bool firstRangeOk =
const bool firstRangeOk = !child->entryPresent || child->entry.rangeVersion <= readVersion;
!child->entryPresent || child->entry.rangeVersion <= readVersion;
uint32_t compared = 0; uint32_t compared = 0;
if constexpr (kAVX512) { if constexpr (kAVX512) {
compared = compare16_avx512(self->childMaxVersion, readVersion); compared = compare16_avx512(self->childMaxVersion, readVersion);
} else { } else {
compared = compare16(self->childMaxVersion, readVersion); compared = compare16(self->childMaxVersion, readVersion);
} }
return !(compared & mask) && firstRangeOk; return !(compared & mask) && firstRangeOk;
#else #else
const unsigned shiftUpperBound = end - begin; const unsigned shiftUpperBound = end - begin;
const unsigned shiftAmount = begin; const unsigned shiftAmount = begin;
auto inBounds = [&](unsigned c) { auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
return c - shiftAmount < shiftUpperBound;
};
uint32_t mask = 0; uint32_t mask = 0;
for (int i = 0; i < 16; ++i) { for (int i = 0; i < 16; ++i) {
mask |= inBounds(self->index[i]) << i; mask |= inBounds(self->index[i]) << i;
}
mask &= (1 << self->numChildren) - 1;
if (!mask) {
return true;
}
Node *child = self->children[std::countr_zero(mask)];
const bool firstRangeOk =
!child->entryPresent || child->entry.rangeVersion <= readVersion;
uint32_t compared = 0;
for (int i = 0; i < 16; ++i) {
compared |= (self->childMaxVersion[i] > readVersion) << i;
}
return !(compared & mask) && firstRangeOk;
#endif
} }
case Type_Node48: { mask &= (1 << self->numChildren) - 1;
auto *self = static_cast<Node48 *>(n); if (!mask) {
{
int c = self->bitSet.firstSetGeq(begin + 1);
if (c >= 0 && c < end) {
Node *child = self->children[self->index[c]];
if (child->entryPresent && child->entry.rangeVersion > readVersion) {
return false;
}
begin = c;
} else {
return true;
}
// [begin, end) is now the half-open interval of children we're interested
// in.
assert(begin < end);
}
// Check all pages
static_assert(Node48::kMaxOfMaxPageSize == 16);
for (int i = 0; i < Node48::kMaxOfMaxTotalPages; ++i) {
if (self->maxOfMax[i] > readVersion) {
if (!scan16<kAVX512>(self->childMaxVersion +
(i << Node48::kMaxOfMaxShift),
self->reverseIndex + (i << Node48::kMaxOfMaxShift),
begin, end, readVersion)) {
return false;
}
}
}
return true; return true;
} }
case Type_Node256: { Node *child = self->children[std::countr_zero(mask)];
static_assert(Node256::kMaxOfMaxTotalPages == 16); const bool firstRangeOk =
auto *self = static_cast<Node256 *>(n); !child->entryPresent || child->entry.rangeVersion <= readVersion;
uint32_t compared = 0;
for (int i = 0; i < 16; ++i) {
compared |= (self->childMaxVersion[i] > readVersion) << i;
}
return !(compared & mask) && firstRangeOk;
{ #endif
int c = self->bitSet.firstSetGeq(begin + 1); }
if (c >= 0 && c < end) {
Node *child = self->children[c];
if (child->entryPresent && child->entry.rangeVersion > readVersion) {
return false;
}
begin = c;
} else {
return true;
}
// [begin, end) is now the half-open interval of children we're interested
// in.
assert(begin < end);
}
const int firstPage = begin >> Node256::kMaxOfMaxShift; template <bool kAVX512>
const int lastPage = (end - 1) >> Node256::kMaxOfMaxShift; bool checkMaxBetweenExclusiveImpl(Node48 *n, int begin, int end,
// Check the only page if there's only one InternalVersionT readVersion,
if (firstPage == lastPage) { ReadContext *tls) {
if (self->maxOfMax[firstPage] <= readVersion) { ++tls->range_read_node_scan_accum;
return true; assume(-1 <= begin);
assume(begin <= 256);
assume(-1 <= end);
assume(end <= 256);
assume(begin < end);
assert(!(begin == -1 && end == 256));
auto *self = static_cast<Node48 *>(n);
{
int c = self->bitSet.firstSetGeq(begin + 1);
if (c >= 0 && c < end) {
Node *child = self->children[self->index[c]];
if (child->entryPresent && child->entry.rangeVersion > readVersion) {
return false;
} }
const int intraPageBegin = begin & (Node256::kMaxOfMaxPageSize - 1); begin = c;
const int intraPageEnd = end - (lastPage << Node256::kMaxOfMaxShift); } else {
return scan16<kAVX512>(self->childMaxVersion + return true;
(firstPage << Node256::kMaxOfMaxShift),
intraPageBegin, intraPageEnd, readVersion);
} }
// Check the first page // [begin, end) is now the half-open interval of children we're interested
if (self->maxOfMax[firstPage] > readVersion) { // in.
const int intraPageBegin = begin & (Node256::kMaxOfMaxPageSize - 1); assert(begin < end);
}
// Check all pages
static_assert(Node48::kMaxOfMaxPageSize == 16);
for (int i = 0; i < Node48::kMaxOfMaxTotalPages; ++i) {
if (self->maxOfMax[i] > readVersion) {
if (!scan16<kAVX512>(self->childMaxVersion + if (!scan16<kAVX512>(self->childMaxVersion +
(i << Node48::kMaxOfMaxShift),
self->reverseIndex + (i << Node48::kMaxOfMaxShift),
begin, end, readVersion)) {
return false;
}
}
}
return true;
}
template <bool kAVX512>
bool checkMaxBetweenExclusiveImpl(Node256 *n, int begin, int end,
InternalVersionT readVersion,
ReadContext *tls) {
++tls->range_read_node_scan_accum;
assume(-1 <= begin);
assume(begin <= 256);
assume(-1 <= end);
assume(end <= 256);
assume(begin < end);
assert(!(begin == -1 && end == 256));
static_assert(Node256::kMaxOfMaxTotalPages == 16);
auto *self = static_cast<Node256 *>(n);
{
int c = self->bitSet.firstSetGeq(begin + 1);
if (c >= 0 && c < end) {
Node *child = self->children[c];
if (child->entryPresent && child->entry.rangeVersion > readVersion) {
return false;
}
begin = c;
} else {
return true;
}
// [begin, end) is now the half-open interval of children we're interested
// in.
assert(begin < end);
}
const int firstPage = begin >> Node256::kMaxOfMaxShift;
const int lastPage = (end - 1) >> Node256::kMaxOfMaxShift;
// Check the only page if there's only one
if (firstPage == lastPage) {
if (self->maxOfMax[firstPage] <= readVersion) {
return true;
}
const int intraPageBegin = begin & (Node256::kMaxOfMaxPageSize - 1);
const int intraPageEnd = end - (lastPage << Node256::kMaxOfMaxShift);
return scan16<kAVX512>(self->childMaxVersion +
(firstPage << Node256::kMaxOfMaxShift), (firstPage << Node256::kMaxOfMaxShift),
intraPageBegin, 16, readVersion)) { intraPageBegin, intraPageEnd, readVersion);
return false; }
} // Check the first page
if (self->maxOfMax[firstPage] > readVersion) {
const int intraPageBegin = begin & (Node256::kMaxOfMaxPageSize - 1);
if (!scan16<kAVX512>(self->childMaxVersion +
(firstPage << Node256::kMaxOfMaxShift),
intraPageBegin, 16, readVersion)) {
return false;
} }
// Check the last page }
if (self->maxOfMax[lastPage] > readVersion) { // Check the last page
const int intraPageEnd = end - (lastPage << Node256::kMaxOfMaxShift); if (self->maxOfMax[lastPage] > readVersion) {
if (!scan16<kAVX512>(self->childMaxVersion + const int intraPageEnd = end - (lastPage << Node256::kMaxOfMaxShift);
(lastPage << Node256::kMaxOfMaxShift), if (!scan16<kAVX512>(self->childMaxVersion +
0, intraPageEnd, readVersion)) { (lastPage << Node256::kMaxOfMaxShift),
return false; 0, intraPageEnd, readVersion)) {
} return false;
} }
// Check inner pages }
return scan16<kAVX512>(self->maxOfMax, firstPage + 1, lastPage, // Check inner pages
readVersion); return scan16<kAVX512>(self->maxOfMax, firstPage + 1, lastPage, readVersion);
}
template <bool kAVX512>
bool checkMaxBetweenExclusiveImpl(Node *n, int begin, int end,
InternalVersionT readVersion,
ReadContext *tls) {
switch (n->getType()) {
case Type_Node0:
return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node0 *>(n), begin,
end, readVersion, tls);
case Type_Node3: {
return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node3 *>(n), begin,
end, readVersion, tls);
}
case Type_Node16: {
return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node16 *>(n),
begin, end, readVersion, tls);
}
case Type_Node48: {
return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node48 *>(n),
begin, end, readVersion, tls);
}
case Type_Node256: {
return checkMaxBetweenExclusiveImpl<kAVX512>(static_cast<Node256 *>(n),
begin, end, readVersion, tls);
} }
default: // GCOVR_EXCL_LINE default: // GCOVR_EXCL_LINE
__builtin_unreachable(); // GCOVR_EXCL_LINE __builtin_unreachable(); // GCOVR_EXCL_LINE
@@ -2402,6 +2459,58 @@ bool checkMaxBetweenExclusive(Node *n, int begin, int end,
return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls); return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
} }
bool checkMaxBetweenExclusive(Node0 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
}
bool checkMaxBetweenExclusive(Node3 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
}
#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
__attribute__((target("avx512f"))) bool
checkMaxBetweenExclusive(Node16 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<true>(n, begin, end, readVersion, tls);
}
__attribute__((target("default")))
#endif
bool checkMaxBetweenExclusive(Node16 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
}
#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
__attribute__((target("avx512f"))) bool
checkMaxBetweenExclusive(Node48 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<true>(n, begin, end, readVersion, tls);
}
__attribute__((target("default")))
#endif
bool checkMaxBetweenExclusive(Node48 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
}
#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
__attribute__((target("avx512f"))) bool
checkMaxBetweenExclusive(Node256 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<true>(n, begin, end, readVersion, tls);
}
__attribute__((target("default")))
#endif
bool checkMaxBetweenExclusive(Node256 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *tls) {
return checkMaxBetweenExclusiveImpl<false>(n, begin, end, readVersion, tls);
}
Vector<uint8_t> getSearchPath(Arena &arena, Node *n) { Vector<uint8_t> getSearchPath(Arena &arena, Node *n) {
assert(n != nullptr); assert(n != nullptr);
auto result = vector<uint8_t>(arena); auto result = vector<uint8_t>(arena);
@@ -2424,7 +2533,8 @@ Vector<uint8_t> getSearchPath(Arena &arena, Node *n) {
// //
// Precondition: transitively, no child of n has a search path that's a longer // Precondition: transitively, no child of n has a search path that's a longer
// prefix of key than n // prefix of key than n
bool checkRangeStartsWith(Node *n, std::span<const uint8_t> key, int begin, template <class NodeT>
bool checkRangeStartsWith(NodeT *n, std::span<const uint8_t> key, int begin,
int end, InternalVersionT readVersion, int end, InternalVersionT readVersion,
ReadContext *tls) { ReadContext *tls) {
#if DEBUG_VERBOSE && !defined(NDEBUG) #if DEBUG_VERBOSE && !defined(NDEBUG)
@@ -2501,6 +2611,15 @@ scan16<true>(const InternalVersionT *vs, int begin, int end,
template __attribute__((target("avx512f"))) bool template __attribute__((target("avx512f"))) bool
checkMaxBetweenExclusiveImpl<true>(Node *n, int begin, int end, checkMaxBetweenExclusiveImpl<true>(Node *n, int begin, int end,
InternalVersionT readVersion, ReadContext *); InternalVersionT readVersion, ReadContext *);
template __attribute__((target("avx512f"))) bool
checkMaxBetweenExclusiveImpl<true>(Node16 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *);
template __attribute__((target("avx512f"))) bool
checkMaxBetweenExclusiveImpl<true>(Node48 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *);
template __attribute__((target("avx512f"))) bool
checkMaxBetweenExclusiveImpl<true>(Node256 *n, int begin, int end,
InternalVersionT readVersion, ReadContext *);
#endif #endif
// Returns a pointer the pointer to the newly inserted node in the tree. Caller // Returns a pointer the pointer to the newly inserted node in the tree. Caller
@@ -3207,14 +3326,16 @@ PRESERVE_NONE void begin(CheckJob *job, CheckContext *context) {
job->remaining = job->begin.subspan(0, job->lcp); job->remaining = job->begin.subspan(0, job->lcp);
if (job->remaining.size() == 0) { if (job->remaining.size() == 0) {
MUSTTAIL return done_common_prefix_iter(job, context); job->continuation = done_common_prefix_iter;
MUSTTAIL return job->continuation(job, context);
} }
auto [c, maxV] = getChildAndMaxVersion(job->n, job->remaining[0]); auto [c, maxV] = getChildAndMaxVersion(job->n, job->remaining[0]);
job->maxV = maxV; job->maxV = maxV;
job->child = c; job->child = c;
if (job->child == nullptr) { if (job->child == nullptr) {
MUSTTAIL return done_common_prefix_iter(job, context); job->continuation = done_common_prefix_iter;
MUSTTAIL return job->continuation(job, context);
} }
job->continuation = commonPrefixIterTable[c.getType()]; job->continuation = commonPrefixIterTable[c.getType()];