weaselab/conflict-set
2
0
Fork 0
Code Issues 4 Packages Releases 13 Activity
Files
47faa51a358dfaa48346c6544a7f3ab69b69032f
conflict-set/ConflictSet.cpp
Andrew Noyes 47faa51a35 Add Internal.h
2024-01-30 10:49:49 -08:00

1112 lines
33 KiB
C++
Raw Blame History

#include "ConflictSet.h"
#include "Internal.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <cstring>
#include <limits>
#include <span>
#include <string>
#include <string_view>
#include <utility>
#ifdef HAS_AVX
#include <immintrin.h>
#elif defined(HAS_ARM_NEON)
#include <arm_neon.h>
#endif
#define DEBUG_VERBOSE 0
// ==================== BEGIN IMPLEMENTATION ====================
struct Entry {
int64_t pointVersion;
int64_t rangeVersion;
};
enum class Type : int8_t {
Node4,
Node16,
Node48,
Node256,
Invalid,
};
struct Node {
/* begin section that's copied to the next node */
Node *parent = nullptr;
int64_t maxVersion;
Entry entry;
int16_t numChildren = 0;
bool entryPresent = false;
uint8_t parentsIndex = 0;
/* end section that's copied to the next node */
Type type = Type::Invalid;
};
Node *getChild(Node *self, uint8_t index);
int getChildGeq(Node *self, int child);
Node *&getOrCreateChild(Node *&self, uint8_t index);
Node *newNode();
void eraseChild(Node *self, uint8_t index);
struct Node4 : Node {
// Sorted
uint8_t index[4] = {};
Node *children[4] = {};
Node4() { this->type = Type::Node4; }
};
Node *newNode() { return new (safe_malloc(sizeof(Node4))) Node4; }
struct Node16 : Node {
// Sorted
uint8_t index[16] = {};
Node *children[16] = {};
Node16() { this->type = Type::Node16; }
};
struct Node48 : Node {
int8_t nextFree = 0;
int8_t index[256];
Node *children[48] = {};
Node48() {
this->type = Type::Node48;
memset(index, -1, 256);
}
};
struct Node256 : Node {
Node *children[256] = {};
Node256() { this->type = Type::Node256; }
};
int getNodeIndex(Node4 *self, uint8_t index) {
for (int i = 0; i < self->numChildren; ++i) {
if (self->index[i] == index) {
return i;
}
}
return -1;
}
int getNodeIndex(Node16 *self, uint8_t index) {
#ifdef HAS_AVX
// Based on https://www.the-paper-trail.org/post/art-paper-notes/
// key_vec is 16 repeated copies of the searched-for byte, one for every
// possible position in child_keys that needs to be searched.
__m128i key_vec = _mm_set1_epi8(index);
// Compare all child_keys to 'index' in parallel. Don't worry if some of the
// keys aren't valid, we'll mask the results to only consider the valid ones
// below.
__m128i indices;
memcpy(&indices, self->index, sizeof(self->index));
__m128i results = _mm_cmpeq_epi8(key_vec, indices);
// Build a mask to select only the first node->num_children values from the
// comparison (because the other values are meaningless)
int mask = (1 << self->numChildren) - 1;
// Change the results of the comparison into a bitfield, masking off any
// invalid comparisons.
int bitfield = _mm_movemask_epi8(results) & mask;
// No match if there are no '1's in the bitfield.
if (bitfield == 0)
return -1;
// Find the index of the first '1' in the bitfield by counting the leading
// zeros.
return __builtin_ctz(bitfield);
#elif defined(HAS_ARM_NEON)
// Based on
// https://community.arm.com/arm-community-blogs/b/infrastructure-solutions-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon
uint8x16_t indices;
memcpy(&indices, self->index, sizeof(self->index));
// 0xff for each match
uint16x8_t results =
vreinterpretq_u16_u8(vceqq_u8(vdupq_n_u8(index), indices));
uint64_t mask = self->numChildren == 16
? uint64_t(-1)
: (uint64_t(1) << (self->numChildren * 4)) - 1;
// 0xf for each match in valid range
uint64_t bitfield =
vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(results, 4)), 0) & mask;
if (bitfield == 0)
return -1;
return __builtin_ctzll(bitfield) / 4;
#else
for (int i = 0; i < self->numChildren; ++i) {
if (self->index[i] == index) {
return i;
}
}
return -1;
#endif
}
#ifdef HAS_AVX
int firstNonNeg1(const int8_t x[16]) {
__m128i key_vec = _mm_set1_epi8(-1);
__m128i indices;
memcpy(&indices, x, 16);
__m128i results = _mm_cmpeq_epi8(key_vec, indices);
uint32_t bitfield = _mm_movemask_epi8(results) ^ 0xffff;
if (bitfield == 0)
return -1;
return __builtin_ctz(bitfield);
}
int lastNonNeg1(const int8_t x[16]) {
__m128i key_vec = _mm_set1_epi8(-1);
__m128i indices;
memcpy(&indices, x, 16);
__m128i results = _mm_cmpeq_epi8(key_vec, indices);
uint32_t bitfield = _mm_movemask_epi8(results) ^ 0xffff;
if (bitfield == 0)
return -1;
return 31 - __builtin_clz(bitfield);
}
#endif
#ifdef HAS_ARM_NEON
int firstNonNeg1(const int8_t x[16]) {
uint8x16_t indices;
memcpy(&indices, x, 16);
uint16x8_t results = vreinterpretq_u16_u8(vceqq_u8(vdupq_n_u8(-1), indices));
uint64_t bitfield =
~vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(results, 4)), 0);
if (bitfield == 0)
return -1;
return __builtin_ctzll(bitfield) / 4;
}
int lastNonNeg1(const int8_t x[16]) {
uint8x16_t indices;
memcpy(&indices, x, 16);
uint16x8_t results = vreinterpretq_u16_u8(vceqq_u8(vdupq_n_u8(-1), indices));
uint64_t bitfield =
~vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(results, 4)), 0);
if (bitfield == 0)
return -1;
return 15 - __builtin_clzll(bitfield) / 4;
}
#endif
[[maybe_unused]] Node *getChild(Node *self, uint8_t index) {
if (self->type == Type::Node4) {
auto *self4 = static_cast<Node4 *>(self);
int i = getNodeIndex(self4, index);
if (i >= 0) {
return self4->children[i];
}
return nullptr;
} else if (self->type == Type::Node16) {
auto *self16 = static_cast<Node16 *>(self);
int i = getNodeIndex(self16, index);
if (i >= 0) {
return self16->children[i];
}
return nullptr;
} else if (self->type == Type::Node48) {
auto *self48 = static_cast<Node48 *>(self);
int secondIndex = self48->index[index];
if (secondIndex >= 0) {
return self48->children[secondIndex];
}
return nullptr;
} else {
auto *self256 = static_cast<Node256 *>(self);
return self256->children[index];
}
}
// Precondition - an entry for index must exist in the node
Node *&getChildExists(Node *self, uint8_t index) {
if (self->type == Type::Node4) {
auto *self4 = static_cast<Node4 *>(self);
return self4->children[getNodeIndex(self4, index)];
} else if (self->type == Type::Node16) {
auto *self16 = static_cast<Node16 *>(self);
return self16->children[getNodeIndex(self16, index)];
} else if (self->type == Type::Node48) {
auto *self48 = static_cast<Node48 *>(self);
int secondIndex = self48->index[index];
if (secondIndex >= 0) {
return self48->children[secondIndex];
}
} else {
auto *self256 = static_cast<Node256 *>(self);
return self256->children[index];
}
__builtin_unreachable(); // GCOVR_EXCL_LINE
}
int getChildGeq(Node *self, int child) {
if (child > 255) {
return -1;
}
if (self->type == Type::Node4) {
auto *self4 = static_cast<Node4 *>(self);
for (int i = 0; i < self->numChildren; ++i) {
if (i > 0) {
assert(self4->index[i - 1] < self4->index[i]);
}
if (self4->index[i] >= child) {
return self4->index[i];
}
}
} else if (self->type == Type::Node16) {
auto *self16 = static_cast<Node16 *>(self);
#ifdef HAS_AVX
__m128i key_vec = _mm_set1_epi8(child);
__m128i indices;
memcpy(&indices, self16->index, sizeof(self16->index));
__m128i results = _mm_cmpeq_epi8(key_vec, _mm_min_epu8(key_vec, indices));
int mask = (1 << self16->numChildren) - 1;
int bitfield = _mm_movemask_epi8(results) & mask;
int result = bitfield == 0 ? -1 : self16->index[__builtin_ctz(bitfield)];
assert(result == [&]() -> int {
for (int i = 0; i < self16->numChildren; ++i) {
if (self16->index[i] >= child) {
return self16->index[i];
}
}
return -1;
}());
return result;
#elif defined(HAS_ARM_NEON)
uint8x16_t indices;
memcpy(&indices, self16->index, sizeof(self16->index));
// 0xff for each leq
auto results = vcleq_u8(vdupq_n_u8(child), indices);
uint64_t mask = self->numChildren == 16
? uint64_t(-1)
: (uint64_t(1) << (self->numChildren * 4)) - 1;
// 0xf for each 0xff (within mask)
uint64_t bitfield =
vget_lane_u64(
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(results), 4)),
0) &
mask;
int simd =
bitfield == 0 ? -1 : self16->index[__builtin_ctzll(bitfield) / 4];
assert(simd == [&]() -> int {
for (int i = 0; i < self->numChildren; ++i) {
if (self16->index[i] >= child) {
return self16->index[i];
}
}
return -1;
}());
return simd;
#else
for (int i = 0; i < self->numChildren; ++i) {
if (i > 0) {
assert(self16->index[i - 1] < self16->index[i]);
}
if (self16->index[i] >= child) {
return self16->index[i];
}
}
#endif
} else if (self->type == Type::Node48) {
auto *self48 = static_cast<Node48 *>(self);
#if defined(HAS_AVX) || defined(HAS_ARM_NEON)
int i = child;
for (; (i & 0xf) != 0; ++i) {
if (self48->index[i] >= 0) {
assert(self48->children[self48->index[i]] != nullptr);
return i;
}
}
for (; i < 256; i += 16) {
auto result = firstNonNeg1(self48->index + i);
if (result != -1) {
return i + result;
}
}
#else
for (int i = child; i < 256; ++i) {
if (self48->index[i] >= 0) {
assert(self48->children[self48->index[i]] != nullptr);
return i;
}
}
#endif
} else {
auto *self256 = static_cast<Node256 *>(self);
for (int i = child; i < 256; ++i) {
if (self256->children[i]) {
return i;
}
}
}
return -1;
}
int getChildLeq(Node *self, int child) {
if (child < 0) {
return -1;
}
if (self->type == Type::Node4) {
auto *self4 = static_cast<Node4 *>(self);
for (int i = self->numChildren - 1; i >= 0; --i) {
if (i > 0) {
assert(self4->index[i - 1] < self4->index[i]);
}
if (self4->index[i] <= child) {
return self4->index[i];
}
}
} else if (self->type == Type::Node16) {
auto *self16 = static_cast<Node16 *>(self);
#ifdef HAS_AVX
__m128i key_vec = _mm_set1_epi8(child);
__m128i indices;
memcpy(&indices, self16->index, sizeof(self16->index));
__m128i results = _mm_cmpeq_epi8(key_vec, _mm_max_epu8(key_vec, indices));
int mask = (1 << self16->numChildren) - 1;
int bitfield = _mm_movemask_epi8(results) & mask;
int result =
bitfield == 0 ? -1 : self16->index[31 - __builtin_clz(bitfield)];
assert(result == [&]() -> int {
for (int i = self16->numChildren - 1; i >= 0; --i) {
if (self16->index[i] <= child) {
return self16->index[i];
}
}
return -1;
}());
return result;
#elif defined(HAS_ARM_NEON)
uint8x16_t indices;
memcpy(&indices, self16->index, sizeof(self16->index));
// 0xff for each leq
auto results = vcleq_u8(indices, vdupq_n_u8(child));
uint64_t mask = self->numChildren == 16
? uint64_t(-1)
: (uint64_t(1) << (self->numChildren * 4)) - 1;
// 0xf for each 0xff (within mask)
uint64_t bitfield =
vget_lane_u64(
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(results), 4)),
0) &
mask;
int simd =
bitfield == 0 ? -1 : self16->index[15 - __builtin_clzll(bitfield) / 4];
assert(simd == [&]() -> int {
for (int i = self->numChildren - 1; i >= 0; --i) {
if (self16->index[i] <= child) {
return self16->index[i];
}
}
return -1;
}());
return simd;
#else
for (int i = self->numChildren - 1; i >= 0; --i) {
if (self16->index[i] <= child) {
return self16->index[i];
}
}
return -1;
#endif
} else if (self->type == Type::Node48) {
auto *self48 = static_cast<Node48 *>(self);
// TODO the plain loop is faster?
#if 0 && (defined(HAS_AVX) || defined(HAS_ARM_NEON))
int i = child;
if (i < 0) {
return -1;
}
for (; (i & 0xf) != 0; --i) {
if (self48->index[i] >= 0) {
assert(self48->children[self48->index[i]] != nullptr);
return i;
}
}
if (self48->index[i] >= 0) {
assert(self48->children[self48->index[i]] != nullptr);
return i;
}
i -= 16;
for (; i >= 0; i -= 16) {
auto result = lastNonNeg1(self48->index + i);
if (result != -1) {
return i + result;
}
}
#else
for (int i = child; i >= 0; --i) {
if (self48->index[i] >= 0) {
assert(self48->children[self48->index[i]] != nullptr);
return i;
}
}
#endif
} else {
auto *self256 = static_cast<Node256 *>(self);
for (int i = child; i >= 0; --i) {
if (self256->children[i]) {
return i;
}
}
}
return -1;
}
void setChildrenParents(Node *node) {
for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) {
getChildExists(node, i)->parent = node;
}
}
Node *&getOrCreateChild(Node *&self, uint8_t index) {
if (self->type == Type::Node4) {
auto *self4 = static_cast<Node4 *>(self);
{
int i = getNodeIndex(self4, index);
if (i >= 0) {
return self4->children[i];
}
}
if (self->numChildren == 4) {
auto *newSelf = new (safe_malloc(sizeof(Node16))) Node16;
memcpy((void *)newSelf, self, offsetof(Node, type));
memcpy(newSelf->index, self4->index, 4);
memcpy(newSelf->children, self4->children, 4 * sizeof(void *));
free(std::exchange(self, newSelf));
setChildrenParents(self);
goto insert16;
} else {
++self->numChildren;
for (int i = 0; i < int(self->numChildren) - 1; ++i) {
if (int(self4->index[i]) > int(index)) {
memmove(self4->index + i + 1, self4->index + i,
self->numChildren - (i + 1));
memmove(self4->children + i + 1, self4->children + i,
(self->numChildren - (i + 1)) * sizeof(void *));
self4->index[i] = index;
self4->children[i] = nullptr;
return self4->children[i];
}
}
self4->index[self->numChildren - 1] = index;
self4->children[self->numChildren - 1] = nullptr;
return self4->children[self->numChildren - 1];
}
} else if (self->type == Type::Node16) {
insert16:
auto *self16 = static_cast<Node16 *>(self);
{
int i = getNodeIndex(self16, index);
if (i >= 0) {
return self16->children[i];
}
}
if (self->numChildren == 16) {
auto *newSelf = new (safe_malloc(sizeof(Node48))) Node48;
memcpy((void *)newSelf, self, offsetof(Node, type));
newSelf->nextFree = 16;
int i = 0;
for (auto x : self16->index) {
newSelf->children[i] = self16->children[i];
newSelf->index[x] = i;
++i;
}
assert(i == 16);
free(std::exchange(self, newSelf));
setChildrenParents(self);
goto insert48;
} else {
++self->numChildren;
for (int i = 0; i < int(self->numChildren) - 1; ++i) {
if (int(self16->index[i]) > int(index)) {
memmove(self16->index + i + 1, self16->index + i,
self->numChildren - (i + 1));
memmove(self16->children + i + 1, self16->children + i,
(self->numChildren - (i + 1)) * sizeof(void *));
self16->index[i] = index;
self16->children[i] = nullptr;
return self16->children[i];
}
}
self16->index[self->numChildren - 1] = index;
self16->children[self->numChildren - 1] = nullptr;
return self16->children[self->numChildren - 1];
}
} else if (self->type == Type::Node48) {
insert48:
auto *self48 = static_cast<Node48 *>(self);
int secondIndex = self48->index[index];
if (secondIndex >= 0) {
return self48->children[secondIndex];
}
if (self->numChildren == 48) {
auto *newSelf = new (safe_malloc(sizeof(Node256))) Node256;
memcpy((void *)newSelf, self, offsetof(Node, type));
for (int i = 0; i < 256; ++i) {
if (self48->index[i] >= 0) {
newSelf->children[i] = self48->children[self48->index[i]];
}
}
free(std::exchange(self, newSelf));
self = newSelf;
setChildrenParents(self);
goto insert256;
} else {
++self->numChildren;
assert(self48->nextFree < 48);
self48->index[index] = self48->nextFree;
self48->children[self48->nextFree] = nullptr;
return self48->children[self48->nextFree++];
}
} else {
insert256:
auto *self256 = static_cast<Node256 *>(self);
if (!self256->children[index]) {
++self->numChildren;
}
return self256->children[index];
}
}
// Precondition - an entry for index must exist in the node
[[maybe_unused]] void eraseChild(Node *self, uint8_t index) {
if (self->type == Type::Node4) {
auto *self4 = static_cast<Node4 *>(self);
int nodeIndex = getNodeIndex(self4, index);
memmove(self4->index + nodeIndex, self4->index + nodeIndex + 1,
sizeof(self4->index[0]) * (self->numChildren - (nodeIndex + 1)));
memmove(self4->children + nodeIndex, self4->children + nodeIndex + 1,
sizeof(self4->children[0]) * // NOLINT
(self->numChildren - (nodeIndex + 1)));
} else if (self->type == Type::Node16) {
auto *self16 = static_cast<Node16 *>(self);
int nodeIndex = getNodeIndex(self16, index);
memmove(self16->index + nodeIndex, self16->index + nodeIndex + 1,
sizeof(self16->index[0]) * (self->numChildren - (nodeIndex + 1)));
memmove(self16->children + nodeIndex, self16->children + nodeIndex + 1,
sizeof(self16->children[0]) * // NOLINT
(self->numChildren - (nodeIndex + 1)));
} else if (self->type == Type::Node48) {
auto *self48 = static_cast<Node48 *>(self);
int8_t toRemoveChildrenIndex = std::exchange(self48->index[index], -1);
int8_t lastChildrenIndex = --self48->nextFree;
assert(toRemoveChildrenIndex >= 0);
assert(lastChildrenIndex >= 0);
if (toRemoveChildrenIndex != lastChildrenIndex) {
self48->children[toRemoveChildrenIndex] =
std::exchange(self48->children[lastChildrenIndex], nullptr);
self48->index[self48->children[toRemoveChildrenIndex]->parentsIndex] =
toRemoveChildrenIndex;
}
} else {
auto *self256 = static_cast<Node256 *>(self);
self256->children[index] = nullptr;
}
--self->numChildren;
if (self->numChildren == 0 && !self->entryPresent &&
self->parent != nullptr) {
eraseChild(self->parent, self->parentsIndex);
}
}
Node *nextPhysical(Node *node) {
int index = -1;
for (;;) {
auto nextChild = getChildGeq(node, index + 1);
if (nextChild >= 0) {
return getChildExists(node, nextChild);
}
index = node->parentsIndex;
node = node->parent;
if (node == nullptr) {
return nullptr;
}
}
}
Node *nextLogical(Node *node) {
for (node = nextPhysical(node); node != nullptr && !node->entryPresent;
node = nextPhysical(node))
;
return node;
}
Node *prevPhysical(Node *node) {
assert(node->parent != nullptr);
auto prevChild = getChildLeq(node->parent, node->parentsIndex - 1);
assert(prevChild < node->parentsIndex);
if (prevChild >= 0) {
node = getChildExists(node->parent, prevChild);
// Move down the right spine
for (;;) {
auto rightMostChild = getChildLeq(node, 255);
if (rightMostChild >= 0) {
node = getChildExists(node, rightMostChild);
} else {
return node;
}
}
} else {
return node->parent;
}
}
struct Iterator {
Node *n;
int cmp;
};
std::string_view getSearchPath(Arena &arena, Node *n) {
if (n->parent == nullptr) {
return {};
}
auto result = vector<char>(arena);
for (; n->parent != nullptr; n = n->parent) {
result.push_back(n->parentsIndex);
}
std::reverse(result.begin(), result.end());
return std::string_view((const char *)&result[0], result.size()); // NOLINT
}
Iterator lastLeq(Node *n, const std::span<const uint8_t> key) {
auto remaining = key;
for (;;) {
Arena arena;
assert((std::string(getSearchPath(arena, n)) +
std::string((const char *)remaining.data(), remaining.size()))
.ends_with(std::string((const char *)key.data(), key.size())));
if (remaining.size() == 0) {
// We've found the physical node corresponding to search path `key`
if (n->entryPresent) {
return {n, 0};
} else {
break;
}
} else {
int c = getChildLeq(n, remaining[0]);
if (c == remaining[0]) {
n = getChildExists(n, c);
remaining = remaining.subspan(1, remaining.size() - 1);
} else {
// The physical node corresponding to search path `key` does not exist.
// Let's find the physical node corresponding to the highest search key
// (not necessarily present) less than key
// Move down the right spine
for (;;) {
if (c >= 0) {
n = getChildExists(n, c);
} else {
break;
}
c = getChildLeq(n, 255);
}
break;
}
}
}
// Iterate backwards along existing physical nodes until we find a present
// entry
for (; !n->entryPresent; n = prevPhysical(n)) {
}
return {n, -1};
}
void insert(Node **self_, std::span<const uint8_t> key, int64_t writeVersion) {
for (;;) {
auto &self = *self_;
self->maxVersion = std::max(self->maxVersion, writeVersion);
if (key.size() == 0) {
auto l = lastLeq(self, key);
self->entryPresent = true;
self->entry.pointVersion = writeVersion;
assert(l.n != nullptr);
assert(l.n->entryPresent);
self->entry.rangeVersion = l.n->entry.rangeVersion;
return;
}
auto &child = getOrCreateChild(self, key.front());
if (!child) {
child = newNode();
child->parent = self;
child->parentsIndex = key.front();
}
self_ = &child;
key = key.subspan(1, key.size() - 1);
}
}
struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
void check(const ReadRange *reads, Result *result, int count) const {
for (int i = 0; i < count; ++i) {
const auto &r = reads[i];
if (r.readVersion < oldestVersion) {
result[i] = TooOld;
continue;
}
// TODO support non-point reads
assert(r.end.len == 0);
auto [l, c] =
lastLeq(root, std::span<const uint8_t>(r.begin.p, r.begin.len));
#if DEBUG_VERBOSE && !defined(NDEBUG)
Arena arena;
printf("LastLeq for `%s' got `%s'\n", printable(r.begin).c_str(),
printable(getSearchPath(arena, l)).c_str());
#endif
assert(l != nullptr);
assert(l->entryPresent);
result[i] = (c == 0 ? l->entry.pointVersion : l->entry.rangeVersion) >
r.readVersion
? Conflict
: Commit;
}
}
void addWrites(const WriteRange *writes, int count) {
for (int i = 0; i < count; ++i) {
const auto &w = writes[i];
// TODO support non-point writes
assert(w.end.len == 0);
insert(&root, std::span<const uint8_t>(w.begin.p, w.begin.len),
w.writeVersion);
}
}
void setOldestVersion(int64_t oldestVersion) {
this->oldestVersion = oldestVersion;
}
explicit Impl(int64_t oldestVersion) : oldestVersion(oldestVersion) {
// Insert ""
root = newNode();
root->maxVersion = oldestVersion;
root->entry.pointVersion = oldestVersion;
root->entry.rangeVersion = oldestVersion;
root->entryPresent = true;
}
~Impl() {
Arena arena;
auto toFree = vector<Node *>(arena);
toFree.push_back(root);
while (toFree.size() > 0) {
auto *n = toFree.back();
toFree.pop_back();
// Add all children to toFree
for (int child = getChildGeq(n, 0); child >= 0;
child = getChildGeq(n, child + 1)) {
auto *c = getChildExists(n, child);
assert(c != nullptr);
toFree.push_back(c);
}
free(n);
}
}
Node *root;
int64_t oldestVersion;
};
// ==================== END IMPLEMENTATION ====================
// GCOVR_EXCL_START
void ConflictSet::check(const ReadRange *reads, Result *results,
int count) const {
return impl->check(reads, results, count);
}
void ConflictSet::addWrites(const WriteRange *writes, int count) {
return impl->addWrites(writes, count);
}
void ConflictSet::setOldestVersion(int64_t oldestVersion) {
return impl->setOldestVersion(oldestVersion);
}
ConflictSet::ConflictSet(int64_t oldestVersion, [[maybe_unused]] uint64_t seed)
: impl(new (safe_malloc(sizeof(Impl))) Impl{oldestVersion}) {}
ConflictSet::~ConflictSet() {
if (impl) {
impl->~Impl();
free(impl);
}
}
ConflictSet::ConflictSet(ConflictSet &&other) noexcept
: impl(std::exchange(other.impl, nullptr)) {}
ConflictSet &ConflictSet::operator=(ConflictSet &&other) noexcept {
impl = std::exchange(other.impl, nullptr);
return *this;
}
using ConflictSet_Result = ConflictSet::Result;
using ConflictSet_Key = ConflictSet::Key;
using ConflictSet_ReadRange = ConflictSet::ReadRange;
using ConflictSet_WriteRange = ConflictSet::WriteRange;
extern "C" {
__attribute__((__visibility__("default"))) void
ConflictSet_check(void *cs, const ConflictSet_ReadRange *reads,
ConflictSet_Result *results, int count) {
((ConflictSet::Impl *)cs)->check(reads, results, count);
}
__attribute__((__visibility__("default"))) void
ConflictSet_addWrites(void *cs, const ConflictSet_WriteRange *writes,
int count) {
((ConflictSet::Impl *)cs)->addWrites(writes, count);
}
__attribute__((__visibility__("default"))) void
ConflictSet_setOldestVersion(void *cs, int64_t oldestVersion) {
((ConflictSet::Impl *)cs)->setOldestVersion(oldestVersion);
}
__attribute__((__visibility__("default"))) void *
ConflictSet_create(int64_t oldestVersion, uint64_t) {
return new (safe_malloc(sizeof(ConflictSet::Impl)))
ConflictSet::Impl{oldestVersion};
}
__attribute__((__visibility__("default"))) void ConflictSet_destroy(void *cs) {
using Impl = ConflictSet::Impl;
((Impl *)cs)->~Impl();
free(cs);
}
}
void printLogical(std::string &result, Node *node) {
Arena arena;
for (Node *iter = node; iter != nullptr;) {
auto *next = nextLogical(iter);
std::string key;
for (uint8_t c : getSearchPath(arena, iter)) {
key += "x";
key += "0123456789abcdef"[c / 16];
key += "0123456789abcdef"[c % 16];
}
if (iter->entry.pointVersion == iter->entry.rangeVersion) {
result += key + " -> " + std::to_string(iter->entry.pointVersion) + "\n";
} else {
result += key + " -> " + std::to_string(iter->entry.pointVersion) + "\n";
if (next == nullptr || (getSearchPath(arena, next) !=
(std::string(getSearchPath(arena, iter)) +
std::string("\x00", 1)))) {
result +=
key + "x00 -> " + std::to_string(iter->entry.rangeVersion) + "\n";
}
}
iter = next;
}
}
[[maybe_unused]] void debugPrintDot(FILE *file, Node *node) {
struct DebugDotPrinter {
explicit DebugDotPrinter(FILE *file) : file(file) {}
void print(Node *n) {
assert(n != nullptr);
if (n->entryPresent) {
fprintf(file, " k_%p [label=\"m=%d p=%d r=%d\"];\n", (void *)n,
int(n->maxVersion), int(n->entry.pointVersion),
int(n->entry.rangeVersion));
} else {
fprintf(file, " k_%p [label=\"m=%d\"];\n", (void *)n,
int(n->maxVersion));
}
for (int child = getChildGeq(n, 0); child >= 0;
child = getChildGeq(n, child + 1)) {
auto *c = getChildExists(n, child);
fprintf(file, " k_%p -> k_%p [label=\"'%02x'\"];\n", (void *)n,
(void *)c, child);
print(c);
}
}
FILE *file;
};
fprintf(file, "digraph ConflictSet {\n");
assert(node != nullptr);
DebugDotPrinter printer{file};
printer.print(node);
fprintf(file, "}\n");
}
void checkParentPointers(Node *node, bool &success) {
for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) {
auto *child = getChildExists(node, i);
if (child->parent != node) {
Arena arena;
fprintf(stderr, "%s child %d has parent pointer %p. Expected %p\n",
printable(getSearchPath(arena, node)).c_str(), i,
(void *)child->parent, (void *)node);
success = false;
}
checkParentPointers(child, success);
}
}
[[maybe_unused]] int64_t checkMaxVersion(Node *node, bool &success) {
int64_t expected =
node->entryPresent
? std::max(node->entry.pointVersion, node->entry.rangeVersion)
: std::numeric_limits<int64_t>::lowest();
for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) {
auto *child = getChildExists(node, i);
expected = std::max(expected, checkMaxVersion(child, success));
}
if (node->maxVersion != expected) {
Arena arena;
fprintf(stderr, "%s has max version %d. Expected %d\n",
printable(getSearchPath(arena, node)).c_str(),
int(node->maxVersion), int(expected));
success = false;
}
return expected;
}
bool checkCorrectness(Node *node, ReferenceImpl &refImpl) {
bool success = true;
checkParentPointers(node, success);
std::string logicalMap;
std::string referenceLogicalMap;
printLogical(logicalMap, node);
refImpl.printLogical(referenceLogicalMap);
if (logicalMap != referenceLogicalMap) {
fprintf(stderr,
"Logical map not equal to reference logical map.\n\nActual:\n"
"%s\nExpected:\n%s\n",
logicalMap.c_str(), referenceLogicalMap.c_str());
success = false;
}
return success;
}
namespace std {
void __throw_length_error(const char *) { __builtin_unreachable(); }
} // namespace std
#ifdef ENABLE_MAIN
int main(void) {
int64_t writeVersion = 0;
ConflictSet::Impl cs{writeVersion};
ReferenceImpl refImpl{writeVersion};
Arena arena;
constexpr int kNumKeys = 10;
auto *write = new (arena) ConflictSet::WriteRange[kNumKeys];
for (int i = 0; i < kNumKeys; ++i) {
write[i].begin = toKey(arena, i);
write[i].end.len = 0;
write[i].writeVersion = ++writeVersion;
}
cs.addWrites(write, kNumKeys);
refImpl.addWrites(write, kNumKeys);
debugPrintDot(stdout, cs.root);
return 0;
}
#endif
#ifdef ENABLE_FUZZ
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
// TODO call setOldestVersion, and check range writes/reads
Arbitrary arbitrary{{data, size}};
int64_t writeVersion = 0;
ConflictSet::Impl cs{writeVersion};
ReferenceImpl refImpl{writeVersion};
while (arbitrary.hasEntropy()) {
Arena arena;
{
int numWrites = arbitrary.bounded(10);
int64_t v = ++writeVersion;
auto *writes = new (arena) ConflictSet::WriteRange[numWrites];
auto keys = set<std::string_view>(arena);
while (int(keys.size()) < numWrites) {
if (!arbitrary.hasEntropy()) {
// Tell the fuzzer it's not interesting
return -1;
}
int keyLen = arbitrary.bounded(8);
auto *begin = new (arena) uint8_t[keyLen];
arbitrary.randomBytes(begin, keyLen);
keys.insert(std::string_view((const char *)begin, keyLen));
}
auto iter = keys.begin();
for (int i = 0; i < numWrites; ++i) {
writes[i].begin.p = (const uint8_t *)iter->data();
writes[i].begin.len = iter->size();
++iter;
writes[i].end.len = 0;
writes[i].writeVersion = v;
#if DEBUG_VERBOSE && !defined(NDEBUG)
printf("Write: {%s} -> %d\n", printable(writes[i].begin).c_str(),
int(writes[i].writeVersion));
#endif
}
assert(iter == keys.end());
cs.addWrites(writes, numWrites);
refImpl.addWrites(writes, numWrites);
}
bool success = checkCorrectness(cs.root, refImpl);
if (!success) {
abort();
}
{
int numReads = arbitrary.bounded(10);
int64_t v = writeVersion - arbitrary.bounded(10);
auto *reads = new (arena) ConflictSet::ReadRange[numReads];
auto keys = set<std::string_view>(arena);
while (int(keys.size()) < numReads) {
if (!arbitrary.hasEntropy()) {
// Tell the fuzzer it's not interesting
return -1;
}
int keyLen = arbitrary.bounded(8);
auto *begin = new (arena) uint8_t[keyLen];
arbitrary.randomBytes(begin, keyLen);
keys.insert(std::string_view((const char *)begin, keyLen));
}
auto iter = keys.begin();
for (int i = 0; i < numReads; ++i) {
reads[i].begin.p = (const uint8_t *)iter->data();
reads[i].begin.len = iter->size();
++iter;
reads[i].end.len = 0;
reads[i].readVersion = v;
#if DEBUG_VERBOSE && !defined(NDEBUG)
printf("Read: {%s} at %d\n", printable(reads[i].begin).c_str(),
int(reads[i].readVersion));
#endif
}
assert(iter == keys.end());
auto *results1 = new (arena) ConflictSet::Result[numReads];
auto *results2 = new (arena) ConflictSet::Result[numReads];
cs.check(reads, results1, numReads);
refImpl.check(reads, results2, numReads);
for (int i = 0; i < numReads; ++i) {
if (results1[i] != results2[i]) {
fprintf(stderr, "Expected %d, got %d for read of %s at version %d\n",
results2[i], results1[i], printable(reads[i].begin).c_str(),
int(reads[i].readVersion));
std::string referenceLogicalMap;
refImpl.printLogical(referenceLogicalMap);
fprintf(stderr, "Logical map:\n\n%s\n", referenceLogicalMap.c_str());
abort();
}
}
}
}
return 0;
}
#endif
// GCOVR_EXCL_STOP
Reference in New Issue View Git Blame Copy Permalink