1212 lines
36 KiB
C++
1212 lines
36 KiB
C++
#include "ConflictSet.h"
|
|
#include "Internal.h"
|
|
|
|
#include <algorithm>
|
|
#include <bit>
|
|
#include <cassert>
|
|
#include <cstdint>
|
|
#include <cstring>
|
|
#include <limits>
|
|
#include <span>
|
|
#include <string>
|
|
#include <string_view>
|
|
#include <utility>
|
|
|
|
#ifdef HAS_AVX
|
|
#include <immintrin.h>
|
|
#elif defined(HAS_ARM_NEON)
|
|
#include <arm_neon.h>
|
|
#endif
|
|
|
|
// ==================== BEGIN IMPLEMENTATION ====================
|
|
|
|
struct Entry {
|
|
int64_t pointVersion;
|
|
int64_t rangeVersion;
|
|
};
|
|
|
|
enum class Type : int8_t {
|
|
Node4,
|
|
Node16,
|
|
Node48,
|
|
Node256,
|
|
Invalid,
|
|
};
|
|
struct Node {
|
|
/* begin section that's copied to the next node */
|
|
Node *parent = nullptr;
|
|
int64_t maxVersion = std::numeric_limits<int64_t>::lowest();
|
|
Entry entry;
|
|
constexpr static auto kCompressedKeyMaxLen = 18;
|
|
int16_t numChildren = 0;
|
|
bool entryPresent = false;
|
|
uint8_t parentsIndex = 0;
|
|
uint8_t compressedKey[kCompressedKeyMaxLen];
|
|
int8_t compressedKeyLen = 0;
|
|
/* end section that's copied to the next node */
|
|
|
|
Type type = Type::Invalid;
|
|
};
|
|
|
|
struct Node4 : Node {
|
|
// Sorted
|
|
uint8_t index[4] = {};
|
|
Node *children[4] = {};
|
|
Node4() { this->type = Type::Node4; }
|
|
};
|
|
|
|
Node *newNode() { return new (safe_malloc(sizeof(Node4))) Node4; }
|
|
|
|
struct Node16 : Node {
|
|
// Sorted
|
|
uint8_t index[16] = {};
|
|
Node *children[16] = {};
|
|
Node16() { this->type = Type::Node16; }
|
|
};
|
|
|
|
struct Node48 : Node {
|
|
int8_t nextFree = 0;
|
|
int8_t index[256];
|
|
Node *children[48] = {};
|
|
Node48() {
|
|
this->type = Type::Node48;
|
|
memset(index, -1, 256);
|
|
}
|
|
};
|
|
|
|
struct PointerSet {
|
|
bool test(int i) const {
|
|
assert(0 <= i);
|
|
assert(i < 256);
|
|
if (i < 128) {
|
|
return (lo >> i) & 1;
|
|
} else {
|
|
return (hi >> (i - 128)) & 1;
|
|
}
|
|
}
|
|
|
|
void set(int i) {
|
|
assert(0 <= i);
|
|
assert(i < 256);
|
|
if (i < 128) {
|
|
lo |= __uint128_t(1) << i;
|
|
} else {
|
|
hi |= __uint128_t(1) << (i - 128);
|
|
}
|
|
}
|
|
|
|
int firstSetGeq(int i) const {
|
|
if (i < 128) {
|
|
int a = std::countr_zero(lo >> i);
|
|
if (a < 128) {
|
|
assert(i + a < 128);
|
|
return i + a;
|
|
}
|
|
i = 128;
|
|
}
|
|
int b = std::countr_zero(hi >> (i - 128));
|
|
if (b < 128) {
|
|
assert(i + b < 256);
|
|
return i + b;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
int lastSetLeq(int i) const {
|
|
if (i >= 128) {
|
|
int a = std::countl_zero(hi << (255 - i));
|
|
if (a < 128) {
|
|
return i - a;
|
|
}
|
|
i = 127;
|
|
}
|
|
int b = std::countl_zero(lo << (127 - i));
|
|
if (b < 128) {
|
|
return i - b;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
private:
|
|
__uint128_t lo = 0;
|
|
__uint128_t hi = 0;
|
|
};
|
|
|
|
struct Node256 : Node {
|
|
PointerSet pointerSet;
|
|
Node *children[256] = {};
|
|
Node256() { this->type = Type::Node256; }
|
|
};
|
|
|
|
int getNodeIndex(Node4 *self, uint8_t index) {
|
|
for (int i = 0; i < self->numChildren; ++i) {
|
|
if (self->index[i] == index) {
|
|
return i;
|
|
}
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
int getNodeIndex(Node16 *self, uint8_t index) {
|
|
#ifdef HAS_AVX
|
|
// Based on https://www.the-paper-trail.org/post/art-paper-notes/
|
|
|
|
// key_vec is 16 repeated copies of the searched-for byte, one for every
|
|
// possible position in child_keys that needs to be searched.
|
|
__m128i key_vec = _mm_set1_epi8(index);
|
|
|
|
// Compare all child_keys to 'index' in parallel. Don't worry if some of the
|
|
// keys aren't valid, we'll mask the results to only consider the valid ones
|
|
// below.
|
|
__m128i indices;
|
|
memcpy(&indices, self->index, sizeof(self->index));
|
|
__m128i results = _mm_cmpeq_epi8(key_vec, indices);
|
|
|
|
// Build a mask to select only the first node->num_children values from the
|
|
// comparison (because the other values are meaningless)
|
|
int mask = (1 << self->numChildren) - 1;
|
|
|
|
// Change the results of the comparison into a bitfield, masking off any
|
|
// invalid comparisons.
|
|
int bitfield = _mm_movemask_epi8(results) & mask;
|
|
|
|
// No match if there are no '1's in the bitfield.
|
|
if (bitfield == 0)
|
|
return -1;
|
|
|
|
// Find the index of the first '1' in the bitfield by counting the leading
|
|
// zeros.
|
|
return std::countr_zero(bitfield);
|
|
#elif defined(HAS_ARM_NEON)
|
|
// Based on
|
|
// https://community.arm.com/arm-community-blogs/b/infrastructure-solutions-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon
|
|
|
|
uint8x16_t indices;
|
|
memcpy(&indices, self->index, sizeof(self->index));
|
|
// 0xff for each match
|
|
uint16x8_t results =
|
|
vreinterpretq_u16_u8(vceqq_u8(vdupq_n_u8(index), indices));
|
|
uint64_t mask = self->numChildren == 16
|
|
? uint64_t(-1)
|
|
: (uint64_t(1) << (self->numChildren * 4)) - 1;
|
|
// 0xf for each match in valid range
|
|
uint64_t bitfield =
|
|
vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(results, 4)), 0) & mask;
|
|
if (bitfield == 0)
|
|
return -1;
|
|
return std::countr_zero(bitfield) / 4;
|
|
#else
|
|
for (int i = 0; i < self->numChildren; ++i) {
|
|
if (self->index[i] == index) {
|
|
return i;
|
|
}
|
|
}
|
|
return -1;
|
|
#endif
|
|
}
|
|
|
|
#ifdef HAS_AVX
|
|
int firstNonNeg1(const int8_t x[16]) {
|
|
__m128i key_vec = _mm_set1_epi8(-1);
|
|
__m128i indices;
|
|
memcpy(&indices, x, 16);
|
|
__m128i results = _mm_cmpeq_epi8(key_vec, indices);
|
|
uint32_t bitfield = _mm_movemask_epi8(results) ^ 0xffff;
|
|
if (bitfield == 0)
|
|
return -1;
|
|
return std::countr_zero(bitfield);
|
|
}
|
|
|
|
int lastNonNeg1(const int8_t x[16]) {
|
|
__m128i key_vec = _mm_set1_epi8(-1);
|
|
__m128i indices;
|
|
memcpy(&indices, x, 16);
|
|
__m128i results = _mm_cmpeq_epi8(key_vec, indices);
|
|
uint32_t bitfield = _mm_movemask_epi8(results) ^ 0xffff;
|
|
if (bitfield == 0)
|
|
return -1;
|
|
return 31 - std::countl_zero(bitfield);
|
|
}
|
|
#endif
|
|
|
|
#ifdef HAS_ARM_NEON
|
|
int firstNonNeg1(const int8_t x[16]) {
|
|
uint8x16_t indices;
|
|
memcpy(&indices, x, 16);
|
|
uint16x8_t results = vreinterpretq_u16_u8(vceqq_u8(vdupq_n_u8(-1), indices));
|
|
uint64_t bitfield =
|
|
~vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(results, 4)), 0);
|
|
if (bitfield == 0)
|
|
return -1;
|
|
return std::countr_zero(bitfield) / 4;
|
|
}
|
|
|
|
int lastNonNeg1(const int8_t x[16]) {
|
|
uint8x16_t indices;
|
|
memcpy(&indices, x, 16);
|
|
uint16x8_t results = vreinterpretq_u16_u8(vceqq_u8(vdupq_n_u8(-1), indices));
|
|
uint64_t bitfield =
|
|
~vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(results, 4)), 0);
|
|
if (bitfield == 0)
|
|
return -1;
|
|
return 15 - std::countl_zero(bitfield) / 4;
|
|
}
|
|
#endif
|
|
|
|
[[maybe_unused]] Node *getChild(Node *self, uint8_t index) {
|
|
if (self->type == Type::Node4) {
|
|
auto *self4 = static_cast<Node4 *>(self);
|
|
int i = getNodeIndex(self4, index);
|
|
if (i >= 0) {
|
|
return self4->children[i];
|
|
}
|
|
return nullptr;
|
|
} else if (self->type == Type::Node16) {
|
|
auto *self16 = static_cast<Node16 *>(self);
|
|
int i = getNodeIndex(self16, index);
|
|
if (i >= 0) {
|
|
return self16->children[i];
|
|
}
|
|
return nullptr;
|
|
} else if (self->type == Type::Node48) {
|
|
auto *self48 = static_cast<Node48 *>(self);
|
|
int secondIndex = self48->index[index];
|
|
if (secondIndex >= 0) {
|
|
return self48->children[secondIndex];
|
|
}
|
|
return nullptr;
|
|
} else {
|
|
auto *self256 = static_cast<Node256 *>(self);
|
|
return self256->children[index];
|
|
}
|
|
}
|
|
|
|
// Precondition - an entry for index must exist in the node
|
|
Node *&getChildExists(Node *self, uint8_t index) {
|
|
if (self->type == Type::Node4) {
|
|
auto *self4 = static_cast<Node4 *>(self);
|
|
return self4->children[getNodeIndex(self4, index)];
|
|
} else if (self->type == Type::Node16) {
|
|
auto *self16 = static_cast<Node16 *>(self);
|
|
return self16->children[getNodeIndex(self16, index)];
|
|
} else if (self->type == Type::Node48) {
|
|
auto *self48 = static_cast<Node48 *>(self);
|
|
int secondIndex = self48->index[index];
|
|
if (secondIndex >= 0) {
|
|
return self48->children[secondIndex];
|
|
}
|
|
} else {
|
|
auto *self256 = static_cast<Node256 *>(self);
|
|
return self256->children[index];
|
|
}
|
|
__builtin_unreachable(); // GCOVR_EXCL_LINE
|
|
}
|
|
|
|
int getChildGeq(Node *self, int child) {
|
|
if (child > 255) {
|
|
return -1;
|
|
}
|
|
if (self->type == Type::Node4) {
|
|
auto *self4 = static_cast<Node4 *>(self);
|
|
for (int i = 0; i < self->numChildren; ++i) {
|
|
if (i > 0) {
|
|
assert(self4->index[i - 1] < self4->index[i]);
|
|
}
|
|
if (self4->index[i] >= child) {
|
|
return self4->index[i];
|
|
}
|
|
}
|
|
} else if (self->type == Type::Node16) {
|
|
auto *self16 = static_cast<Node16 *>(self);
|
|
#ifdef HAS_AVX
|
|
__m128i key_vec = _mm_set1_epi8(child);
|
|
__m128i indices;
|
|
memcpy(&indices, self16->index, sizeof(self16->index));
|
|
__m128i results = _mm_cmpeq_epi8(key_vec, _mm_min_epu8(key_vec, indices));
|
|
int mask = (1 << self16->numChildren) - 1;
|
|
int bitfield = _mm_movemask_epi8(results) & mask;
|
|
int result = bitfield == 0 ? -1 : self16->index[std::countr_zero(bitfield)];
|
|
assert(result == [&]() -> int {
|
|
for (int i = 0; i < self16->numChildren; ++i) {
|
|
if (self16->index[i] >= child) {
|
|
return self16->index[i];
|
|
}
|
|
}
|
|
return -1;
|
|
}());
|
|
return result;
|
|
#elif defined(HAS_ARM_NEON)
|
|
uint8x16_t indices;
|
|
memcpy(&indices, self16->index, sizeof(self16->index));
|
|
// 0xff for each leq
|
|
auto results = vcleq_u8(vdupq_n_u8(child), indices);
|
|
uint64_t mask = self->numChildren == 16
|
|
? uint64_t(-1)
|
|
: (uint64_t(1) << (self->numChildren * 4)) - 1;
|
|
// 0xf for each 0xff (within mask)
|
|
uint64_t bitfield =
|
|
vget_lane_u64(
|
|
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(results), 4)),
|
|
0) &
|
|
mask;
|
|
int simd =
|
|
bitfield == 0 ? -1 : self16->index[std::countr_zero(bitfield) / 4];
|
|
assert(simd == [&]() -> int {
|
|
for (int i = 0; i < self->numChildren; ++i) {
|
|
if (self16->index[i] >= child) {
|
|
return self16->index[i];
|
|
}
|
|
}
|
|
return -1;
|
|
}());
|
|
return simd;
|
|
#else
|
|
for (int i = 0; i < self->numChildren; ++i) {
|
|
if (i > 0) {
|
|
assert(self16->index[i - 1] < self16->index[i]);
|
|
}
|
|
if (self16->index[i] >= child) {
|
|
return self16->index[i];
|
|
}
|
|
}
|
|
#endif
|
|
} else if (self->type == Type::Node48) {
|
|
auto *self48 = static_cast<Node48 *>(self);
|
|
#if defined(HAS_AVX) || defined(HAS_ARM_NEON)
|
|
int i = child;
|
|
for (; (i & 0xf) != 0; ++i) {
|
|
if (self48->index[i] >= 0) {
|
|
assert(self48->children[self48->index[i]] != nullptr);
|
|
return i;
|
|
}
|
|
}
|
|
for (; i < 256; i += 16) {
|
|
auto result = firstNonNeg1(self48->index + i);
|
|
if (result != -1) {
|
|
return i + result;
|
|
}
|
|
}
|
|
#else
|
|
for (int i = child; i < 256; ++i) {
|
|
if (self48->index[i] >= 0) {
|
|
assert(self48->children[self48->index[i]] != nullptr);
|
|
return i;
|
|
}
|
|
}
|
|
#endif
|
|
} else {
|
|
auto *self256 = static_cast<Node256 *>(self);
|
|
#ifndef NDEBUG
|
|
for (int i = 0; i < 256; ++i) {
|
|
assert(self256->pointerSet.test(i) == (self256->children[i] != nullptr));
|
|
}
|
|
#endif
|
|
return self256->pointerSet.firstSetGeq(child);
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
int getChildLeq(Node *self, int child) {
|
|
if (child < 0) {
|
|
return -1;
|
|
}
|
|
if (self->type == Type::Node4) {
|
|
auto *self4 = static_cast<Node4 *>(self);
|
|
for (int i = self->numChildren - 1; i >= 0; --i) {
|
|
if (i > 0) {
|
|
assert(self4->index[i - 1] < self4->index[i]);
|
|
}
|
|
if (self4->index[i] <= child) {
|
|
return self4->index[i];
|
|
}
|
|
}
|
|
} else if (self->type == Type::Node16) {
|
|
auto *self16 = static_cast<Node16 *>(self);
|
|
#ifdef HAS_AVX
|
|
__m128i key_vec = _mm_set1_epi8(child);
|
|
__m128i indices;
|
|
memcpy(&indices, self16->index, sizeof(self16->index));
|
|
__m128i results = _mm_cmpeq_epi8(key_vec, _mm_max_epu8(key_vec, indices));
|
|
int mask = (1 << self16->numChildren) - 1;
|
|
int bitfield = _mm_movemask_epi8(results) & mask;
|
|
int result =
|
|
bitfield == 0 ? -1 : self16->index[31 - std::countl_zero(bitfield)];
|
|
assert(result == [&]() -> int {
|
|
for (int i = self16->numChildren - 1; i >= 0; --i) {
|
|
if (self16->index[i] <= child) {
|
|
return self16->index[i];
|
|
}
|
|
}
|
|
return -1;
|
|
}());
|
|
return result;
|
|
#elif defined(HAS_ARM_NEON)
|
|
uint8x16_t indices;
|
|
memcpy(&indices, self16->index, sizeof(self16->index));
|
|
// 0xff for each leq
|
|
auto results = vcleq_u8(indices, vdupq_n_u8(child));
|
|
uint64_t mask = self->numChildren == 16
|
|
? uint64_t(-1)
|
|
: (uint64_t(1) << (self->numChildren * 4)) - 1;
|
|
// 0xf for each 0xff (within mask)
|
|
uint64_t bitfield =
|
|
vget_lane_u64(
|
|
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(results), 4)),
|
|
0) &
|
|
mask;
|
|
int simd =
|
|
bitfield == 0 ? -1 : self16->index[15 - std::countl_zero(bitfield) / 4];
|
|
assert(simd == [&]() -> int {
|
|
for (int i = self->numChildren - 1; i >= 0; --i) {
|
|
if (self16->index[i] <= child) {
|
|
return self16->index[i];
|
|
}
|
|
}
|
|
return -1;
|
|
}());
|
|
return simd;
|
|
#else
|
|
for (int i = self->numChildren - 1; i >= 0; --i) {
|
|
if (self16->index[i] <= child) {
|
|
return self16->index[i];
|
|
}
|
|
}
|
|
return -1;
|
|
#endif
|
|
} else if (self->type == Type::Node48) {
|
|
auto *self48 = static_cast<Node48 *>(self);
|
|
#if defined(HAS_AVX) || defined(HAS_ARM_NEON)
|
|
int i = child;
|
|
if (i < 0) {
|
|
return -1;
|
|
}
|
|
for (; (i & 0xf) != 0; --i) {
|
|
if (self48->index[i] >= 0) {
|
|
assert(self48->children[self48->index[i]] != nullptr);
|
|
return i;
|
|
}
|
|
}
|
|
if (self48->index[i] >= 0) {
|
|
assert(self48->children[self48->index[i]] != nullptr);
|
|
return i;
|
|
}
|
|
i -= 16;
|
|
for (; i >= 0; i -= 16) {
|
|
auto result = lastNonNeg1(self48->index + i);
|
|
if (result != -1) {
|
|
return i + result;
|
|
}
|
|
}
|
|
#else
|
|
for (int i = child; i >= 0; --i) {
|
|
if (self48->index[i] >= 0) {
|
|
assert(self48->children[self48->index[i]] != nullptr);
|
|
return i;
|
|
}
|
|
}
|
|
#endif
|
|
} else {
|
|
auto *self256 = static_cast<Node256 *>(self);
|
|
#ifndef NDEBUG
|
|
for (int i = 0; i < 256; ++i) {
|
|
assert(self256->pointerSet.test(i) == (self256->children[i] != nullptr));
|
|
}
|
|
#endif
|
|
return self256->pointerSet.lastSetLeq(child);
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
void setChildrenParents(Node *node) {
|
|
for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) {
|
|
getChildExists(node, i)->parent = node;
|
|
}
|
|
}
|
|
|
|
// Caller is responsible for assigning a non-null pointer to the returned
|
|
// reference if null
|
|
Node *&getOrCreateChild(Node *&self, uint8_t index) {
|
|
if (self->type == Type::Node4) {
|
|
auto *self4 = static_cast<Node4 *>(self);
|
|
{
|
|
int i = getNodeIndex(self4, index);
|
|
if (i >= 0) {
|
|
return self4->children[i];
|
|
}
|
|
}
|
|
if (self->numChildren == 4) {
|
|
auto *newSelf = new (safe_malloc(sizeof(Node16))) Node16;
|
|
memcpy((void *)newSelf, self, offsetof(Node, type));
|
|
memcpy(newSelf->index, self4->index, 4);
|
|
memcpy(newSelf->children, self4->children, 4 * sizeof(void *));
|
|
free(std::exchange(self, newSelf));
|
|
setChildrenParents(self);
|
|
goto insert16;
|
|
} else {
|
|
++self->numChildren;
|
|
for (int i = 0; i < int(self->numChildren) - 1; ++i) {
|
|
if (int(self4->index[i]) > int(index)) {
|
|
memmove(self4->index + i + 1, self4->index + i,
|
|
self->numChildren - (i + 1));
|
|
memmove(self4->children + i + 1, self4->children + i,
|
|
(self->numChildren - (i + 1)) * sizeof(void *));
|
|
self4->index[i] = index;
|
|
self4->children[i] = nullptr;
|
|
return self4->children[i];
|
|
}
|
|
}
|
|
self4->index[self->numChildren - 1] = index;
|
|
self4->children[self->numChildren - 1] = nullptr;
|
|
return self4->children[self->numChildren - 1];
|
|
}
|
|
} else if (self->type == Type::Node16) {
|
|
insert16:
|
|
auto *self16 = static_cast<Node16 *>(self);
|
|
{
|
|
int i = getNodeIndex(self16, index);
|
|
if (i >= 0) {
|
|
return self16->children[i];
|
|
}
|
|
}
|
|
if (self->numChildren == 16) {
|
|
auto *newSelf = new (safe_malloc(sizeof(Node48))) Node48;
|
|
memcpy((void *)newSelf, self, offsetof(Node, type));
|
|
newSelf->nextFree = 16;
|
|
int i = 0;
|
|
for (auto x : self16->index) {
|
|
newSelf->children[i] = self16->children[i];
|
|
newSelf->index[x] = i;
|
|
++i;
|
|
}
|
|
assert(i == 16);
|
|
free(std::exchange(self, newSelf));
|
|
setChildrenParents(self);
|
|
goto insert48;
|
|
} else {
|
|
++self->numChildren;
|
|
for (int i = 0; i < int(self->numChildren) - 1; ++i) {
|
|
if (int(self16->index[i]) > int(index)) {
|
|
memmove(self16->index + i + 1, self16->index + i,
|
|
self->numChildren - (i + 1));
|
|
memmove(self16->children + i + 1, self16->children + i,
|
|
(self->numChildren - (i + 1)) * sizeof(void *));
|
|
self16->index[i] = index;
|
|
self16->children[i] = nullptr;
|
|
return self16->children[i];
|
|
}
|
|
}
|
|
self16->index[self->numChildren - 1] = index;
|
|
self16->children[self->numChildren - 1] = nullptr;
|
|
return self16->children[self->numChildren - 1];
|
|
}
|
|
} else if (self->type == Type::Node48) {
|
|
insert48:
|
|
auto *self48 = static_cast<Node48 *>(self);
|
|
int secondIndex = self48->index[index];
|
|
if (secondIndex >= 0) {
|
|
return self48->children[secondIndex];
|
|
}
|
|
if (self->numChildren == 48) {
|
|
auto *newSelf = new (safe_malloc(sizeof(Node256))) Node256;
|
|
memcpy((void *)newSelf, self, offsetof(Node, type));
|
|
for (int i = 0; i < 256; ++i) {
|
|
if (self48->index[i] >= 0) {
|
|
newSelf->pointerSet.set(i);
|
|
newSelf->children[i] = self48->children[self48->index[i]];
|
|
}
|
|
}
|
|
free(std::exchange(self, newSelf));
|
|
self = newSelf;
|
|
setChildrenParents(self);
|
|
goto insert256;
|
|
} else {
|
|
++self->numChildren;
|
|
assert(self48->nextFree < 48);
|
|
self48->index[index] = self48->nextFree;
|
|
self48->children[self48->nextFree] = nullptr;
|
|
return self48->children[self48->nextFree++];
|
|
}
|
|
} else {
|
|
insert256:
|
|
auto *self256 = static_cast<Node256 *>(self);
|
|
if (!self256->children[index]) {
|
|
++self->numChildren;
|
|
}
|
|
self256->pointerSet.set(index);
|
|
return self256->children[index];
|
|
}
|
|
}
|
|
|
|
// Precondition - an entry for index must exist in the node
|
|
[[maybe_unused]] void eraseChild(Node *self, uint8_t index) {
|
|
if (self->type == Type::Node4) {
|
|
auto *self4 = static_cast<Node4 *>(self);
|
|
int nodeIndex = getNodeIndex(self4, index);
|
|
memmove(self4->index + nodeIndex, self4->index + nodeIndex + 1,
|
|
sizeof(self4->index[0]) * (self->numChildren - (nodeIndex + 1)));
|
|
memmove(self4->children + nodeIndex, self4->children + nodeIndex + 1,
|
|
sizeof(self4->children[0]) * // NOLINT
|
|
(self->numChildren - (nodeIndex + 1)));
|
|
} else if (self->type == Type::Node16) {
|
|
auto *self16 = static_cast<Node16 *>(self);
|
|
int nodeIndex = getNodeIndex(self16, index);
|
|
memmove(self16->index + nodeIndex, self16->index + nodeIndex + 1,
|
|
sizeof(self16->index[0]) * (self->numChildren - (nodeIndex + 1)));
|
|
memmove(self16->children + nodeIndex, self16->children + nodeIndex + 1,
|
|
sizeof(self16->children[0]) * // NOLINT
|
|
(self->numChildren - (nodeIndex + 1)));
|
|
} else if (self->type == Type::Node48) {
|
|
auto *self48 = static_cast<Node48 *>(self);
|
|
int8_t toRemoveChildrenIndex = std::exchange(self48->index[index], -1);
|
|
int8_t lastChildrenIndex = --self48->nextFree;
|
|
assert(toRemoveChildrenIndex >= 0);
|
|
assert(lastChildrenIndex >= 0);
|
|
if (toRemoveChildrenIndex != lastChildrenIndex) {
|
|
self48->children[toRemoveChildrenIndex] =
|
|
std::exchange(self48->children[lastChildrenIndex], nullptr);
|
|
self48->index[self48->children[toRemoveChildrenIndex]->parentsIndex] =
|
|
toRemoveChildrenIndex;
|
|
}
|
|
} else {
|
|
auto *self256 = static_cast<Node256 *>(self);
|
|
self256->children[index] = nullptr;
|
|
}
|
|
--self->numChildren;
|
|
if (self->numChildren == 0 && !self->entryPresent &&
|
|
self->parent != nullptr) {
|
|
eraseChild(self->parent, self->parentsIndex);
|
|
}
|
|
}
|
|
|
|
Node *nextPhysical(Node *node) {
|
|
int index = -1;
|
|
for (;;) {
|
|
auto nextChild = getChildGeq(node, index + 1);
|
|
if (nextChild >= 0) {
|
|
return getChildExists(node, nextChild);
|
|
}
|
|
index = node->parentsIndex;
|
|
node = node->parent;
|
|
if (node == nullptr) {
|
|
return nullptr;
|
|
}
|
|
}
|
|
}
|
|
|
|
Node *nextLogical(Node *node) {
|
|
for (node = nextPhysical(node); node != nullptr && !node->entryPresent;
|
|
node = nextPhysical(node))
|
|
;
|
|
return node;
|
|
}
|
|
|
|
Node *prevPhysical(Node *node) {
|
|
assert(node->parent != nullptr);
|
|
auto prevChild = getChildLeq(node->parent, node->parentsIndex - 1);
|
|
assert(prevChild < node->parentsIndex);
|
|
if (prevChild >= 0) {
|
|
node = getChildExists(node->parent, prevChild);
|
|
// Move down the right spine
|
|
for (;;) {
|
|
auto rightMostChild = getChildLeq(node, 255);
|
|
if (rightMostChild >= 0) {
|
|
node = getChildExists(node, rightMostChild);
|
|
} else {
|
|
return node;
|
|
}
|
|
}
|
|
} else {
|
|
return node->parent;
|
|
}
|
|
}
|
|
|
|
struct Iterator {
|
|
Node *n;
|
|
int cmp;
|
|
};
|
|
|
|
std::string_view getSearchPath(Arena &arena, Node *n) {
|
|
auto result = vector<char>(arena);
|
|
for (;;) {
|
|
for (int i = n->compressedKeyLen - 1; i >= 0; --i) {
|
|
result.push_back(n->compressedKey[i]);
|
|
}
|
|
if (n->parent == nullptr) {
|
|
break;
|
|
}
|
|
result.push_back(n->parentsIndex);
|
|
n = n->parent;
|
|
}
|
|
std::reverse(result.begin(), result.end());
|
|
return std::string_view((const char *)&result[0], result.size()); // NOLINT
|
|
}
|
|
|
|
Iterator lastLeq(Node *n, const std::span<const uint8_t> key) {
|
|
auto remaining = key;
|
|
for (;;) {
|
|
Arena arena;
|
|
int commonLen = std::min<int>(n->compressedKeyLen, remaining.size());
|
|
if (commonLen > Node::kCompressedKeyMaxLen) {
|
|
__builtin_unreachable();
|
|
}
|
|
int c = memcmp(n->compressedKey, remaining.data(), commonLen);
|
|
if (c == 0 && commonLen == n->compressedKeyLen) {
|
|
// Compressed key matches
|
|
remaining = remaining.subspan(commonLen, remaining.size() - commonLen);
|
|
} else if (c < 0 ||
|
|
(c == 0 && n->compressedKeyLen < int(remaining.size()))) {
|
|
// n is the last physical node less than remaining, and there's no eq node
|
|
break;
|
|
} else if (c > 0) {
|
|
// n is the first physical node greater than remaining, and there's no eq
|
|
// node
|
|
n = prevPhysical(n);
|
|
break;
|
|
}
|
|
assert((std::string(getSearchPath(arena, n)) +
|
|
std::string((const char *)remaining.data(), remaining.size()))
|
|
.ends_with(std::string((const char *)key.data(), key.size())));
|
|
if (remaining.size() == 0) {
|
|
// We've found the physical node corresponding to search path `key`
|
|
if (n->entryPresent) {
|
|
return {n, 0};
|
|
} else {
|
|
break;
|
|
}
|
|
} else {
|
|
int c = getChildLeq(n, remaining[0]);
|
|
if (c == remaining[0]) {
|
|
n = getChildExists(n, c);
|
|
remaining = remaining.subspan(1, remaining.size() - 1);
|
|
} else {
|
|
// The physical node corresponding to search path `key` does not exist.
|
|
// Let's find the physical node corresponding to the highest search key
|
|
// (not necessarily present) less than key
|
|
// Move down the right spine
|
|
for (;;) {
|
|
if (c >= 0) {
|
|
n = getChildExists(n, c);
|
|
} else {
|
|
break;
|
|
}
|
|
c = getChildLeq(n, 255);
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
// Iterate backwards along existing physical nodes until we find a present
|
|
// entry
|
|
for (; !n->entryPresent; n = prevPhysical(n)) {
|
|
}
|
|
return {n, -1};
|
|
}
|
|
|
|
void insert(Node **self_, std::span<const uint8_t> key, int64_t writeVersion) {
|
|
for (;;) {
|
|
auto &self = *self_;
|
|
self->maxVersion = std::max(self->maxVersion, writeVersion);
|
|
int commonLen = std::min<int>(self->compressedKeyLen, key.size());
|
|
// Handle an existing compressed key
|
|
int compressedKeyIndex = 0;
|
|
for (; compressedKeyIndex < commonLen; ++compressedKeyIndex) {
|
|
if (self->compressedKey[compressedKeyIndex] != key[compressedKeyIndex]) {
|
|
auto *old = self;
|
|
self = newNode();
|
|
memcpy((void *)self, old, offsetof(Node, type));
|
|
self->entryPresent = false;
|
|
|
|
getOrCreateChild(self, old->compressedKey[compressedKeyIndex]) = old;
|
|
old->parent = self;
|
|
old->parentsIndex = old->compressedKey[compressedKeyIndex];
|
|
self->compressedKeyLen = compressedKeyIndex;
|
|
|
|
memmove(old->compressedKey, old->compressedKey + compressedKeyIndex + 1,
|
|
old->compressedKeyLen - (compressedKeyIndex + 1));
|
|
old->compressedKeyLen -= compressedKeyIndex + 1;
|
|
break;
|
|
}
|
|
}
|
|
key = key.subspan(compressedKeyIndex, key.size() - compressedKeyIndex);
|
|
|
|
// Consider adding a compressed key
|
|
if (self->numChildren == 0 && !self->entryPresent) {
|
|
self->compressedKeyLen =
|
|
std::min<int>(key.size(), self->kCompressedKeyMaxLen);
|
|
memcpy(self->compressedKey, key.data(), self->compressedKeyLen);
|
|
key = key.subspan(self->compressedKeyLen,
|
|
key.size() - self->compressedKeyLen);
|
|
}
|
|
|
|
if (key.size() == 0) {
|
|
auto l = lastLeq(self, key);
|
|
self->entryPresent = true;
|
|
self->entry.pointVersion = writeVersion;
|
|
assert(l.n != nullptr);
|
|
assert(l.n->entryPresent);
|
|
self->entry.rangeVersion = l.n->entry.rangeVersion;
|
|
return;
|
|
}
|
|
auto &child = getOrCreateChild(self, key.front());
|
|
if (!child) {
|
|
child = newNode();
|
|
child->parent = self;
|
|
child->parentsIndex = key.front();
|
|
}
|
|
self_ = &child;
|
|
key = key.subspan(1, key.size() - 1);
|
|
}
|
|
}
|
|
|
|
void destroyTree(Node *root) {
|
|
Arena arena;
|
|
auto toFree = vector<Node *>(arena);
|
|
toFree.push_back(root);
|
|
while (toFree.size() > 0) {
|
|
auto *n = toFree.back();
|
|
toFree.pop_back();
|
|
// Add all children to toFree
|
|
for (int child = getChildGeq(n, 0); child >= 0;
|
|
child = getChildGeq(n, child + 1)) {
|
|
auto *c = getChildExists(n, child);
|
|
assert(c != nullptr);
|
|
toFree.push_back(c);
|
|
}
|
|
free(n);
|
|
}
|
|
}
|
|
|
|
struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
|
|
void check(const ReadRange *reads, Result *result, int count) const {
|
|
for (int i = 0; i < count; ++i) {
|
|
const auto &r = reads[i];
|
|
if (r.readVersion < oldestVersion) {
|
|
result[i] = TooOld;
|
|
continue;
|
|
}
|
|
// TODO support non-point reads
|
|
assert(r.end.len == 0);
|
|
auto [l, c] =
|
|
lastLeq(root, std::span<const uint8_t>(r.begin.p, r.begin.len));
|
|
#if DEBUG_VERBOSE && !defined(NDEBUG)
|
|
Arena arena;
|
|
printf("LastLeq for `%s' got `%s'\n", printable(r.begin).c_str(),
|
|
printable(getSearchPath(arena, l)).c_str());
|
|
#endif
|
|
assert(l != nullptr);
|
|
assert(l->entryPresent);
|
|
result[i] = (c == 0 ? l->entry.pointVersion : l->entry.rangeVersion) >
|
|
r.readVersion
|
|
? Conflict
|
|
: Commit;
|
|
}
|
|
}
|
|
void addWrites(const WriteRange *writes, int count) {
|
|
for (int i = 0; i < count; ++i) {
|
|
const auto &w = writes[i];
|
|
// TODO support non-point writes
|
|
assert(w.end.len == 0);
|
|
insert(&root, std::span<const uint8_t>(w.begin.p, w.begin.len),
|
|
w.writeVersion);
|
|
}
|
|
}
|
|
void setOldestVersion(int64_t oldestVersion) {
|
|
this->oldestVersion = oldestVersion;
|
|
}
|
|
explicit Impl(int64_t oldestVersion) : oldestVersion(oldestVersion) {
|
|
// Insert ""
|
|
root = newNode();
|
|
root->maxVersion = oldestVersion;
|
|
root->entry.pointVersion = oldestVersion;
|
|
root->entry.rangeVersion = oldestVersion;
|
|
root->entryPresent = true;
|
|
}
|
|
~Impl() { destroyTree(root); }
|
|
|
|
Node *root;
|
|
int64_t oldestVersion;
|
|
};
|
|
|
|
// ==================== END IMPLEMENTATION ====================
|
|
|
|
// GCOVR_EXCL_START
|
|
|
|
void ConflictSet::check(const ReadRange *reads, Result *results,
|
|
int count) const {
|
|
return impl->check(reads, results, count);
|
|
}
|
|
|
|
void ConflictSet::addWrites(const WriteRange *writes, int count) {
|
|
return impl->addWrites(writes, count);
|
|
}
|
|
|
|
void ConflictSet::setOldestVersion(int64_t oldestVersion) {
|
|
return impl->setOldestVersion(oldestVersion);
|
|
}
|
|
|
|
ConflictSet::ConflictSet(int64_t oldestVersion)
|
|
: impl(new (safe_malloc(sizeof(Impl))) Impl{oldestVersion}) {}
|
|
|
|
ConflictSet::~ConflictSet() {
|
|
if (impl) {
|
|
impl->~Impl();
|
|
free(impl);
|
|
}
|
|
}
|
|
|
|
ConflictSet::ConflictSet(ConflictSet &&other) noexcept
|
|
: impl(std::exchange(other.impl, nullptr)) {}
|
|
|
|
ConflictSet &ConflictSet::operator=(ConflictSet &&other) noexcept {
|
|
impl = std::exchange(other.impl, nullptr);
|
|
return *this;
|
|
}
|
|
|
|
using ConflictSet_Result = ConflictSet::Result;
|
|
using ConflictSet_Key = ConflictSet::Key;
|
|
using ConflictSet_ReadRange = ConflictSet::ReadRange;
|
|
using ConflictSet_WriteRange = ConflictSet::WriteRange;
|
|
|
|
extern "C" {
|
|
__attribute__((__visibility__("default"))) void
|
|
ConflictSet_check(void *cs, const ConflictSet_ReadRange *reads,
|
|
ConflictSet_Result *results, int count) {
|
|
((ConflictSet::Impl *)cs)->check(reads, results, count);
|
|
}
|
|
__attribute__((__visibility__("default"))) void
|
|
ConflictSet_addWrites(void *cs, const ConflictSet_WriteRange *writes,
|
|
int count) {
|
|
((ConflictSet::Impl *)cs)->addWrites(writes, count);
|
|
}
|
|
__attribute__((__visibility__("default"))) void
|
|
ConflictSet_setOldestVersion(void *cs, int64_t oldestVersion) {
|
|
((ConflictSet::Impl *)cs)->setOldestVersion(oldestVersion);
|
|
}
|
|
__attribute__((__visibility__("default"))) void *
|
|
ConflictSet_create(int64_t oldestVersion) {
|
|
return new (safe_malloc(sizeof(ConflictSet::Impl)))
|
|
ConflictSet::Impl{oldestVersion};
|
|
}
|
|
__attribute__((__visibility__("default"))) void ConflictSet_destroy(void *cs) {
|
|
using Impl = ConflictSet::Impl;
|
|
((Impl *)cs)->~Impl();
|
|
free(cs);
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
|
|
void printLogical(std::string &result, Node *node) {
|
|
Arena arena;
|
|
for (Node *iter = node; iter != nullptr;) {
|
|
auto *next = nextLogical(iter);
|
|
std::string key;
|
|
for (uint8_t c : getSearchPath(arena, iter)) {
|
|
key += "x";
|
|
key += "0123456789abcdef"[c / 16];
|
|
key += "0123456789abcdef"[c % 16];
|
|
}
|
|
if (iter->entry.pointVersion == iter->entry.rangeVersion) {
|
|
result += key + " -> " + std::to_string(iter->entry.pointVersion) + "\n";
|
|
} else {
|
|
result += key + " -> " + std::to_string(iter->entry.pointVersion) + "\n";
|
|
if (next == nullptr || (getSearchPath(arena, next) !=
|
|
(std::string(getSearchPath(arena, iter)) +
|
|
std::string("\x00", 1)))) {
|
|
result +=
|
|
key + "x00 -> " + std::to_string(iter->entry.rangeVersion) + "\n";
|
|
}
|
|
}
|
|
iter = next;
|
|
}
|
|
}
|
|
|
|
[[maybe_unused]] void debugPrintDot(FILE *file, Node *node) {
|
|
|
|
struct DebugDotPrinter {
|
|
|
|
explicit DebugDotPrinter(FILE *file) : file(file) {}
|
|
|
|
void print(Node *n) {
|
|
assert(n != nullptr);
|
|
auto compressedKey =
|
|
printable(Key{n->compressedKey, n->compressedKeyLen});
|
|
if (n->entryPresent) {
|
|
fprintf(file, " k_%p [label=\"m=%d p=%d r=%d %s\"];\n", (void *)n,
|
|
int(n->maxVersion), int(n->entry.pointVersion),
|
|
int(n->entry.rangeVersion), compressedKey.c_str());
|
|
} else {
|
|
fprintf(file, " k_%p [label=\"m=%d %s\"];\n", (void *)n,
|
|
int(n->maxVersion), compressedKey.c_str());
|
|
}
|
|
for (int child = getChildGeq(n, 0); child >= 0;
|
|
child = getChildGeq(n, child + 1)) {
|
|
auto *c = getChildExists(n, child);
|
|
fprintf(file, " k_%p -> k_%p [label=\"x%02x\"];\n", (void *)n,
|
|
(void *)c, child);
|
|
print(c);
|
|
}
|
|
}
|
|
FILE *file;
|
|
};
|
|
|
|
fprintf(file, "digraph ConflictSet {\n");
|
|
assert(node != nullptr);
|
|
DebugDotPrinter printer{file};
|
|
printer.print(node);
|
|
fprintf(file, "}\n");
|
|
}
|
|
|
|
void checkCompressedKey(Node *node, bool &success) {
|
|
if (node->numChildren == 1 &&
|
|
node->compressedKeyLen < node->kCompressedKeyMaxLen) {
|
|
Arena arena;
|
|
fprintf(stderr, "%s has 1 child and %d < %d compressed key bytes\n",
|
|
printable(getSearchPath(arena, node)).c_str(),
|
|
int(node->compressedKeyLen), int(node->kCompressedKeyMaxLen));
|
|
|
|
success = false;
|
|
}
|
|
for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) {
|
|
auto *child = getChildExists(node, i);
|
|
if (child->parent != node) {
|
|
Arena arena;
|
|
fprintf(stderr, "%s child %d has parent pointer %p. Expected %p\n",
|
|
printable(getSearchPath(arena, node)).c_str(), i,
|
|
(void *)child->parent, (void *)node);
|
|
success = false;
|
|
}
|
|
checkCompressedKey(child, success);
|
|
}
|
|
}
|
|
|
|
void checkParentPointers(Node *node, bool &success) {
|
|
for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) {
|
|
auto *child = getChildExists(node, i);
|
|
if (child->parent != node) {
|
|
Arena arena;
|
|
fprintf(stderr, "%s child %d has parent pointer %p. Expected %p\n",
|
|
printable(getSearchPath(arena, node)).c_str(), i,
|
|
(void *)child->parent, (void *)node);
|
|
success = false;
|
|
}
|
|
checkParentPointers(child, success);
|
|
}
|
|
}
|
|
|
|
[[maybe_unused]] int64_t checkMaxVersion(Node *node, bool &success) {
|
|
int64_t expected =
|
|
node->entryPresent
|
|
? std::max(node->entry.pointVersion, node->entry.rangeVersion)
|
|
: std::numeric_limits<int64_t>::lowest();
|
|
for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) {
|
|
auto *child = getChildExists(node, i);
|
|
expected = std::max(expected, checkMaxVersion(child, success));
|
|
}
|
|
if (node->maxVersion != expected) {
|
|
Arena arena;
|
|
fprintf(stderr, "%s has max version %d. Expected %d\n",
|
|
printable(getSearchPath(arena, node)).c_str(),
|
|
int(node->maxVersion), int(expected));
|
|
success = false;
|
|
}
|
|
return expected;
|
|
}
|
|
|
|
bool checkCorrectness(Node *node, ReferenceImpl &refImpl) {
|
|
bool success = true;
|
|
|
|
checkParentPointers(node, success);
|
|
checkCompressedKey(node, success);
|
|
|
|
std::string logicalMap;
|
|
std::string referenceLogicalMap;
|
|
printLogical(logicalMap, node);
|
|
refImpl.printLogical(referenceLogicalMap);
|
|
if (logicalMap != referenceLogicalMap) {
|
|
fprintf(stderr,
|
|
"Logical map not equal to reference logical map.\n\nActual:\n"
|
|
"%s\nExpected:\n%s\n",
|
|
logicalMap.c_str(), referenceLogicalMap.c_str());
|
|
success = false;
|
|
}
|
|
|
|
return success;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
namespace std {
|
|
void __throw_length_error(const char *) { __builtin_unreachable(); }
|
|
} // namespace std
|
|
|
|
#ifdef ENABLE_MAIN
|
|
#define ANKERL_NANOBENCH_IMPLEMENT
|
|
#include "third_party/nanobench.h"
|
|
|
|
void bench() {
|
|
ankerl::nanobench::Bench bench;
|
|
{
|
|
auto *n = newNode();
|
|
for (int i = 0; i < 64; ++i) {
|
|
getOrCreateChild(n, i) = newNode();
|
|
bench.run("getChildLeq" + std::to_string(i),
|
|
[&]() { bench.doNotOptimizeAway(getChildLeq(n, 255)); });
|
|
}
|
|
destroyTree(n);
|
|
}
|
|
{
|
|
auto *n = newNode();
|
|
for (int i = 255; i >= 3 * 64; --i) {
|
|
getOrCreateChild(n, i) = newNode();
|
|
bench.run("getChildGeq" + std::to_string(i),
|
|
[&]() { bench.doNotOptimizeAway(getChildGeq(n, 0)); });
|
|
}
|
|
destroyTree(n);
|
|
}
|
|
}
|
|
|
|
void printTree() {
|
|
int64_t writeVersion = 0;
|
|
ConflictSet::Impl cs{writeVersion};
|
|
ReferenceImpl refImpl{writeVersion};
|
|
Arena arena;
|
|
constexpr int kNumKeys = 5;
|
|
auto *write = new (arena) ConflictSet::WriteRange[kNumKeys];
|
|
for (int i = 0; i < kNumKeys; ++i) {
|
|
write[i].begin = toKey(arena, i);
|
|
write[i].end.len = 0;
|
|
write[i].writeVersion = ++writeVersion;
|
|
}
|
|
cs.addWrites(write, kNumKeys);
|
|
for (int i = 0; i < kNumKeys; ++i) {
|
|
write[i].writeVersion = ++writeVersion;
|
|
}
|
|
cs.addWrites(write, kNumKeys);
|
|
debugPrintDot(stdout, cs.root);
|
|
}
|
|
|
|
int main(void) {
|
|
// bench();
|
|
printTree();
|
|
return 0;
|
|
}
|
|
#endif
|
|
|
|
#ifdef ENABLE_FUZZ
|
|
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
|
TestDriver<ConflictSet::Impl> driver{data, size};
|
|
|
|
do {
|
|
bool success = checkCorrectness(driver.cs.root, driver.refImpl);
|
|
if (!success) {
|
|
abort();
|
|
}
|
|
} while (!driver.next());
|
|
|
|
return 0;
|
|
}
|
|
#endif
|
|
|
|
// GCOVR_EXCL_STOP
|