weaselab/conflict-set
2
0
Fork 0
Code Issues 4 Packages Releases 13 Activity
Files
35cf3f3132ab82e85feb14717412fe015b8e8389
conflict-set/ConflictSet.cpp
Andrew Noyes 35cf3f3132 MacOS fixes
2024-01-24 15:55:44 -08:00

1530 lines
45 KiB
C++
Raw Blame History

#include "ConflictSet.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <cstring>
#include <limits>
#include <map>
#include <set>
#include <span>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
#ifdef HAS_AVX
#include <immintrin.h>
#elif defined(HAS_ARM_NEON)
#include <arm_neon.h>
#endif
#define DEBUG_VERBOSE 0
// GCOVR_EXCL_START
__attribute__((always_inline)) inline void *safe_malloc(size_t s) {
if (void *p = malloc(s)) {
return p;
}
abort();
}
// ==================== BEGIN ARENA IMPL ====================
namespace {
/// Group allocations with similar lifetimes to amortize the cost of malloc/free
struct Arena {
explicit Arena(int initialSize = 0);
/// O(log n) in the number of allocations
~Arena();
struct ArenaImpl;
Arena(const Arena &) = delete;
Arena &operator=(const Arena &) = delete;
Arena(Arena &&other) noexcept;
Arena &operator=(Arena &&other) noexcept;
ArenaImpl *impl = nullptr;
};
} // namespace
[[maybe_unused]] inline void operator delete(void *, std::align_val_t,
Arena &) {}
inline void *operator new(size_t size, std::align_val_t align, Arena &arena);
void *operator new(size_t size, std::align_val_t align, Arena *arena) = delete;
[[maybe_unused]] inline void operator delete(void *, Arena &) {}
inline void *operator new(size_t size, Arena &arena) {
return operator new(size, std::align_val_t(alignof(std::max_align_t)), arena);
}
inline void *operator new(size_t size, Arena *arena) = delete;
[[maybe_unused]] inline void operator delete[](void *, Arena &) {}
inline void *operator new[](size_t size, Arena &arena) {
return operator new(size, arena);
}
inline void *operator new[](size_t size, Arena *arena) = delete;
[[maybe_unused]] inline void operator delete[](void *, std::align_val_t,
Arena &) {}
inline void *operator new[](size_t size, std::align_val_t align, Arena &arena) {
return operator new(size, align, arena);
}
inline void *operator new[](size_t size, std::align_val_t align,
Arena *arena) = delete;
namespace {
/// align must be a power of two
template <class T> T *align_up(T *t, size_t align) {
auto unaligned = uintptr_t(t);
auto aligned = (unaligned + align - 1) & ~(align - 1);
return reinterpret_cast<T *>(reinterpret_cast<char *>(t) + aligned -
unaligned);
}
/// align must be a power of two
constexpr inline int align_up(uint32_t unaligned, uint32_t align) {
return (unaligned + align - 1) & ~(align - 1);
}
/// Returns the smallest power of two >= x
[[maybe_unused]] constexpr inline uint32_t nextPowerOfTwo(uint32_t x) {
return x <= 1 ? 1 : 1 << (32 - __builtin_clz(x - 1));
}
struct Arena::ArenaImpl {
Arena::ArenaImpl *prev;
int capacity;
int used;
uint8_t *begin() { return reinterpret_cast<uint8_t *>(this + 1); }
};
static_assert(sizeof(Arena::ArenaImpl) == 16);
static_assert(alignof(Arena::ArenaImpl) == 8);
Arena::Arena(int initialSize) : impl(nullptr) {
if (initialSize > 0) {
auto allocationSize = align_up(initialSize + sizeof(ArenaImpl), 16);
impl = (Arena::ArenaImpl *)safe_malloc(allocationSize);
impl->prev = nullptr;
impl->capacity = allocationSize - sizeof(ArenaImpl);
impl->used = 0;
}
}
void onDestroy(Arena::ArenaImpl *impl) {
while (impl) {
auto *prev = impl->prev;
free(impl);
impl = prev;
}
}
Arena::Arena(Arena &&other) noexcept
: impl(std::exchange(other.impl, nullptr)) {}
Arena &Arena::operator=(Arena &&other) noexcept {
onDestroy(impl);
impl = std::exchange(other.impl, nullptr);
return *this;
}
Arena::~Arena() { onDestroy(impl); }
} // namespace
inline void *operator new(size_t size, std::align_val_t align, Arena &arena) {
int64_t aligned_size = size + size_t(align) - 1;
if (arena.impl == nullptr ||
(arena.impl->capacity - arena.impl->used) < aligned_size) {
auto allocationSize = align_up(
sizeof(Arena::ArenaImpl) +
std::max<int>(aligned_size,
(arena.impl ? std::max<int>(sizeof(Arena::ArenaImpl),
arena.impl->capacity * 2)
: 0)),
16);
auto *impl = (Arena::ArenaImpl *)safe_malloc(allocationSize);
impl->prev = arena.impl;
impl->capacity = allocationSize - sizeof(Arena::ArenaImpl);
impl->used = 0;
arena.impl = impl;
}
auto *result =
align_up(arena.impl->begin() + arena.impl->used, size_t(align));
auto usedDelta = (result - arena.impl->begin()) + size - arena.impl->used;
arena.impl->used += usedDelta;
return result;
}
namespace {
/// STL-friendly allocator using an arena
template <class T> struct ArenaAlloc {
typedef T value_type;
ArenaAlloc() = delete;
explicit ArenaAlloc(Arena *arena) : arena(arena) {}
Arena *arena;
template <class U> constexpr ArenaAlloc(const ArenaAlloc<U> &other) noexcept {
arena = other.arena;
}
[[nodiscard]] T *allocate(size_t n) {
if (n > 0xfffffffffffffffful / sizeof(T)) { // NOLINT
__builtin_unreachable();
}
return static_cast<T *>((void *)new (std::align_val_t(alignof(T)), *arena)
uint8_t[n * sizeof(T)]); // NOLINT
}
void deallocate(T *, size_t) noexcept {}
};
template <class T> using Vector = std::vector<T, ArenaAlloc<T>>;
template <class T> auto vector(Arena &arena) {
return Vector<T>(ArenaAlloc<T>(&arena));
}
template <class T> using Set = std::set<T, std::less<T>, ArenaAlloc<T>>;
template <class T> auto set(Arena &arena) {
return Set<T>(ArenaAlloc<T>(&arena));
}
template <class T, class U>
bool operator==(const ArenaAlloc<T> &lhs, const ArenaAlloc<U> &rhs) {
return lhs.arena == rhs.arena;
}
template <class T, class U>
bool operator!=(const ArenaAlloc<T> &lhs, const ArenaAlloc<U> &rhs) {
return !(lhs == rhs);
}
} // namespace
// ==================== END ARENA IMPL ====================
// ==================== BEGIN ARBITRARY IMPL ====================
namespace {
/// Think of `Arbitrary` as an attacker-controlled random number generator.
/// Usually you want your random number generator to be fair, so that you can
/// sensibly analyze probabilities. E.g. The analysis that shows that quicksort
/// is expected O(n log n) with a random pivot relies on the random pivot being
/// selected uniformly from a fair distribution.
///
/// Other times you want your randomness to be diabolically unfair, like when
/// looking for bugs and fuzzing. The random-number-like interface is still
/// convenient here, but you can potentially get much better coverage by
/// allowing the possibility of e.g. flipping heads 100 times in a row.
///
/// When it runs out of entropy, it always returns 0.
struct Arbitrary {
Arbitrary() = default;
explicit Arbitrary(std::span<const uint8_t> bytecode) : bytecode(bytecode) {}
/// Draws an arbitrary uint32_t
uint32_t next() { return consume<4>(); }
/// Draws an arbitrary element from [0, s)
uint32_t bounded(uint32_t s);
/// Fill `bytes` with `size` arbitrary bytes
void randomBytes(uint8_t *bytes, int size) {
int toFill = std::min<int>(size, bytecode.size());
if (toFill > 0) {
memcpy(bytes, bytecode.data(), toFill);
}
bytecode = bytecode.subspan(toFill, bytecode.size() - toFill);
memset(bytes + toFill, 0, size - toFill);
}
/// Fill `bytes` with `size` random hex bytes
void randomHex(uint8_t *bytes, int size) {
for (int i = 0; i < size;) {
uint8_t arbitrary = consume<1>();
bytes[i++] = "0123456789abcdef"[arbitrary & 0xf];
arbitrary >>= 4;
if (i < size) {
bytes[i++] = "0123456789abcdef"[arbitrary & 0xf];
}
}
}
template <class T, class = std::enable_if_t<std::is_trivially_copyable_v<T>>>
T randT() {
T t;
randomBytes((uint8_t *)&t, sizeof(T));
return t;
}
bool hasEntropy() const { return bytecode.size() != 0; }
private:
uint8_t consumeByte() {
if (bytecode.size() == 0) {
return 0;
}
auto result = bytecode[0];
bytecode = bytecode.subspan(1, bytecode.size() - 1);
return result;
}
template <int kBytes> uint32_t consume() {
uint32_t result = 0;
static_assert(kBytes <= 4);
for (int i = 0; i < kBytes; ++i) {
result <<= 8;
result |= consumeByte();
}
return result;
}
std::span<const uint8_t> bytecode;
};
uint32_t Arbitrary::bounded(uint32_t s) {
if (s == 1) {
return 0;
}
switch (32 - __builtin_clz(s - 1)) {
case 1:
case 2:
case 3:
case 4:
case 5:
case 6:
case 7:
case 8:
return consume<1>() % s;
case 9:
case 10:
case 11:
case 12:
case 13:
case 14:
case 15:
case 16:
return consume<2>() % s;
case 17:
case 18:
case 19:
case 20:
case 21:
case 22:
case 23:
case 24:
return consume<3>() % s;
default:
return consume<4>() % s;
}
}
} // namespace
// ==================== END ARBITRARY IMPL ====================
// ==================== BEGIN UTILITIES IMPL ====================
// Call Stepwise::step for each element of remaining until it returns true.
// Applies a permutation to `remaining` as a side effect.
template <class Stepwise> void runInterleaved(std::span<Stepwise> remaining) {
while (remaining.size() > 0) {
for (int i = 0; i < int(remaining.size());) {
bool done = remaining[i].step();
if (done) {
if (i != int(remaining.size()) - 1) {
using std::swap;
swap(remaining[i], remaining.back());
}
remaining = remaining.subspan(0, remaining.size() - 1);
} else {
++i;
}
}
}
};
template <class Stepwise> void runSequential(std::span<Stepwise> remaining) {
for (auto &r : remaining) {
while (!r.step()) {
}
}
}
struct ReferenceImpl {
explicit ReferenceImpl(int64_t oldestVersion) : oldestVersion(oldestVersion) {
writeVersionMap[""] = oldestVersion;
}
void check(const ConflictSet::ReadRange *reads, ConflictSet::Result *results,
int count) const {
for (int i = 0; i < count; ++i) {
if (reads[i].readVersion < oldestVersion) {
results[i] = ConflictSet::TooOld;
continue;
}
auto begin =
std::string((const char *)reads[i].begin.p, reads[i].begin.len);
auto end =
reads[i].end.len == 0
? begin + std::string("\x00", 1)
: std::string((const char *)reads[i].end.p, reads[i].end.len);
int64_t maxVersion = oldestVersion;
for (auto iter = --writeVersionMap.upper_bound(begin),
endIter = writeVersionMap.lower_bound(end);
iter != endIter; ++iter) {
maxVersion = std::max(maxVersion, iter->second);
}
results[i] = maxVersion > reads[i].readVersion ? ConflictSet::Conflict
: ConflictSet::Commit;
}
}
void addWrites(const ConflictSet::WriteRange *writes, int count) {
for (int i = 0; i < count; ++i) {
auto begin =
std::string((const char *)writes[i].begin.p, writes[i].begin.len);
auto end =
writes[i].end.len == 0
? begin + std::string("\x00", 1)
: std::string((const char *)writes[i].end.p, writes[i].end.len);
auto writeVersion = writes[i].writeVersion;
auto prevVersion = (--writeVersionMap.upper_bound(end))->second;
for (auto iter = writeVersionMap.lower_bound(begin),
endIter = writeVersionMap.lower_bound(end);
iter != endIter;) {
iter = writeVersionMap.erase(iter);
}
writeVersionMap[begin] = writeVersion;
writeVersionMap[end] = prevVersion;
}
}
void setOldestVersion(int64_t oldestVersion) {
this->oldestVersion = oldestVersion;
}
void printLogical(std::string &result) {
for (const auto &[k, v] : writeVersionMap) {
std::string key;
for (uint8_t c : k) {
key += "x";
key += "0123456789abcdef"[c / 16];
key += "0123456789abcdef"[c % 16];
}
result += key + " -> " + std::to_string(v) + "\n";
}
}
int64_t oldestVersion;
std::map<std::string, int64_t> writeVersionMap;
};
using Key = ConflictSet::Key;
[[maybe_unused]] Key toKey(Arena &arena, int n) {
constexpr int kMaxLength = 8;
int i = kMaxLength;
uint8_t *itoaBuf = new (arena) uint8_t[kMaxLength];
memset(itoaBuf, '0', kMaxLength);
do {
itoaBuf[--i] = "0123456789abcdef"[n % 16];
n /= 16;
} while (n);
return Key{itoaBuf, kMaxLength};
}
[[maybe_unused]] Key toKeyAfter(Arena &arena, int n) {
constexpr int kMaxLength = 8;
int i = kMaxLength;
uint8_t *itoaBuf = new (arena) uint8_t[kMaxLength + 1];
memset(itoaBuf, '0', kMaxLength);
itoaBuf[kMaxLength] = 0;
do {
itoaBuf[--i] = "0123456789abcdef"[n % 16];
n /= 16;
} while (n);
return Key{itoaBuf, kMaxLength + 1};
}
// ==================== END UTILITIES IMPL ====================
// ==================== BEGIN IMPLEMENTATION ====================
// GCOVR_EXCL_STOP
namespace {
struct Entry {
int64_t pointVersion;
int64_t rangeVersion;
};
enum class Type : int8_t {
Node4,
Node16,
Node48,
Node256,
Invalid,
};
struct Node {
/* begin section that's copied to the next node */
Node *parent = nullptr;
int64_t maxVersion;
Entry entry;
int16_t numChildren = 0;
bool entryPresent = false;
uint8_t parentsIndex = 0;
/* end section that's copied to the next node */
Type type = Type::Invalid;
};
Node *getChild(Node *self, uint8_t index);
int getChildGeq(Node *self, int child);
Node *&getOrCreateChild(Node *&self, uint8_t index);
Node *newNode();
void eraseChild(Node *self, uint8_t index);
struct Node4 : Node {
// Sorted
uint8_t index[4] = {};
Node *children[4] = {};
Node4() { this->type = Type::Node4; }
};
Node *newNode() { return new (safe_malloc(sizeof(Node4))) Node4; }
struct Node16 : Node {
// Sorted
uint8_t index[16] = {};
Node *children[16] = {};
Node16() { this->type = Type::Node16; }
};
struct Node48 : Node {
int8_t nextFree = 0;
int8_t index[256];
Node *children[48] = {};
Node48() {
this->type = Type::Node48;
memset(index, -1, 256);
}
};
struct Node256 : Node {
Node *children[256] = {};
Node256() { this->type = Type::Node256; }
};
int getNodeIndex(Node4 *self, uint8_t index) {
for (int i = 0; i < self->numChildren; ++i) {
if (self->index[i] == index) {
return i;
}
}
return -1;
}
int getNodeIndex(Node16 *self, uint8_t index) {
#ifdef HAS_AVX
// Based on https://www.the-paper-trail.org/post/art-paper-notes/
// key_vec is 16 repeated copies of the searched-for byte, one for every
// possible position in child_keys that needs to be searched.
__m128i key_vec = _mm_set1_epi8(index);
// Compare all child_keys to 'index' in parallel. Don't worry if some of the
// keys aren't valid, we'll mask the results to only consider the valid ones
// below.
__m128i indices;
memcpy(&indices, self->index, sizeof(self->index));
__m128i results = _mm_cmpeq_epi8(key_vec, indices);
// Build a mask to select only the first node->num_children values from the
// comparison (because the other values are meaningless)
int mask = (1 << self->numChildren) - 1;
// Change the results of the comparison into a bitfield, masking off any
// invalid comparisons.
int bitfield = _mm_movemask_epi8(results) & mask;
// No match if there are no '1's in the bitfield.
if (bitfield == 0)
return -1;
// Find the index of the first '1' in the bitfield by counting the leading
// zeros.
return __builtin_ctz(bitfield);
#elif defined(HAS_ARM_NEON)
// Based on
// https://community.arm.com/arm-community-blogs/b/infrastructure-solutions-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon
uint8x16_t indices;
memcpy(&indices, self->index, sizeof(self->index));
// 0xff for each match
uint16x8_t results =
vreinterpretq_u16_u8(vceqq_u8(vdupq_n_u8(index), indices));
uint64_t mask = self->numChildren == 16
? uint64_t(-1)
: (uint64_t(1) << (self->numChildren * 4)) - 1;
// 0xf for each match in valid range
uint64_t bitfield =
vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(results, 4)), 0) & mask;
if (bitfield == 0)
return -1;
return __builtin_ctzll(bitfield) / 4;
#else
for (int i = 0; i < self->numChildren; ++i) {
if (self->index[i] == index) {
return i;
}
}
return -1;
#endif
}
#ifdef HAS_AVX
int firstNonNeg1(const int8_t x[16]) {
__m128i key_vec = _mm_set1_epi8(-1);
__m128i indices;
memcpy(&indices, x, 16);
__m128i results = _mm_cmpeq_epi8(key_vec, indices);
uint32_t bitfield = _mm_movemask_epi8(results) ^ 0xffff;
if (bitfield == 0)
return -1;
return __builtin_ctz(bitfield);
}
int lastNonNeg1(const int8_t x[16]) {
__m128i key_vec = _mm_set1_epi8(-1);
__m128i indices;
memcpy(&indices, x, 16);
__m128i results = _mm_cmpeq_epi8(key_vec, indices);
uint32_t bitfield = _mm_movemask_epi8(results) ^ 0xffff;
if (bitfield == 0)
return -1;
return 31 - __builtin_clz(bitfield);
}
#endif
#ifdef HAS_ARM_NEON
int firstNonNeg1(const int8_t x[16]) {
uint8x16_t indices;
memcpy(&indices, x, 16);
uint16x8_t results = vreinterpretq_u16_u8(vceqq_u8(vdupq_n_u8(-1), indices));
uint64_t bitfield =
~vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(results, 4)), 0);
if (bitfield == 0)
return -1;
return __builtin_ctzll(bitfield) / 4;
}
int lastNonNeg1(const int8_t x[16]) {
uint8x16_t indices;
memcpy(&indices, x, 16);
uint16x8_t results = vreinterpretq_u16_u8(vceqq_u8(vdupq_n_u8(-1), indices));
uint64_t bitfield =
~vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(results, 4)), 0);
if (bitfield == 0)
return -1;
return 15 - __builtin_clzll(bitfield) / 4;
}
#endif
[[maybe_unused]] Node *getChild(Node *self, uint8_t index) {
if (self->type == Type::Node4) {
auto *self4 = static_cast<Node4 *>(self);
int i = getNodeIndex(self4, index);
if (i >= 0) {
return self4->children[i];
}
return nullptr;
} else if (self->type == Type::Node16) {
auto *self16 = static_cast<Node16 *>(self);
int i = getNodeIndex(self16, index);
if (i >= 0) {
return self16->children[i];
}
return nullptr;
} else if (self->type == Type::Node48) {
auto *self48 = static_cast<Node48 *>(self);
int secondIndex = self48->index[index];
if (secondIndex >= 0) {
return self48->children[secondIndex];
}
return nullptr;
} else {
auto *self256 = static_cast<Node256 *>(self);
return self256->children[index];
}
}
// Precondition - an entry for index must exist in the node
Node *&getChildExists(Node *self, uint8_t index) {
if (self->type == Type::Node4) {
auto *self4 = static_cast<Node4 *>(self);
return self4->children[getNodeIndex(self4, index)];
} else if (self->type == Type::Node16) {
auto *self16 = static_cast<Node16 *>(self);
return self16->children[getNodeIndex(self16, index)];
} else if (self->type == Type::Node48) {
auto *self48 = static_cast<Node48 *>(self);
int secondIndex = self48->index[index];
if (secondIndex >= 0) {
return self48->children[secondIndex];
}
} else {
auto *self256 = static_cast<Node256 *>(self);
return self256->children[index];
}
__builtin_unreachable(); // GCOVR_EXCL_LINE
}
int getChildGeq(Node *self, int child) {
if (self->type == Type::Node4) {
auto *self4 = static_cast<Node4 *>(self);
for (int i = 0; i < self->numChildren; ++i) {
if (i > 0) {
assert(self4->index[i - 1] < self4->index[i]);
}
if (self4->index[i] >= child) {
return self4->index[i];
}
}
} else if (self->type == Type::Node16) {
auto *self16 = static_cast<Node16 *>(self);
for (int i = 0; i < self->numChildren; ++i) {
if (i > 0) {
assert(self16->index[i - 1] < self16->index[i]);
}
if (self16->index[i] >= child) {
return self16->index[i];
}
}
} else if (self->type == Type::Node48) {
auto *self48 = static_cast<Node48 *>(self);
#if defined(HAS_AVX) || defined(HAS_ARM_NEON)
int i = child;
for (; (i & 0xf) != 0; ++i) {
if (self48->index[i] >= 0) {
assert(self48->children[self48->index[i]] != nullptr);
return i;
}
}
for (; i < 256; i += 16) {
auto result = firstNonNeg1(self48->index + i);
if (result != -1) {
return i + result;
}
}
#else
for (int i = child; i < 256; ++i) {
if (self48->index[i] >= 0) {
assert(self48->children[self48->index[i]] != nullptr);
return i;
}
}
#endif
} else {
auto *self256 = static_cast<Node256 *>(self);
// For some reason gcc can't auto vectorize this, and the plain loop is
// faster.
#if defined(__clang__)
int i = child;
constexpr int kUnrollCount = 8; // Must be a power of two and <= 8
for (; (i & (kUnrollCount - 1)) != 0; ++i) {
if (self256->children[i]) {
return i;
}
}
for (; i < 256; i += kUnrollCount) {
uint8_t nonNull[kUnrollCount];
for (int j = 0; j < kUnrollCount; ++j) {
nonNull[j] = self256->children[i + j] != nullptr ? 0xff : 0;
}
uint64_t word;
memcpy(&word, nonNull, kUnrollCount);
if (word) {
return i + __builtin_ctzll(word) / 8;
}
}
#else
for (int i = child; i < 256; ++i) {
if (self256->children[i]) {
return i;
}
}
#endif
}
return -1;
}
int getChildLeq(Node *self, int child) {
if (self->type == Type::Node4) {
auto *self4 = static_cast<Node4 *>(self);
for (int i = self->numChildren - 1; i >= 0; --i) {
if (i > 0) {
assert(self4->index[i - 1] < self4->index[i]);
}
if (self4->index[i] <= child) {
return self4->index[i];
}
}
} else if (self->type == Type::Node16) {
auto *self16 = static_cast<Node16 *>(self);
for (int i = self->numChildren - 1; i >= 0; --i) {
if (i > 0) {
assert(self16->index[i - 1] < self16->index[i]);
}
if (self16->index[i] <= child) {
return self16->index[i];
}
}
} else if (self->type == Type::Node48) {
auto *self48 = static_cast<Node48 *>(self);
// TODO the plain loop is faster?
#if defined(HAS_AVX) || defined(HAS_ARM_NEON)
int i = child;
if (i < 0) {
return -1;
}
for (; (i & 0xf) != 0; --i) {
if (self48->index[i] >= 0) {
assert(self48->children[self48->index[i]] != nullptr);
return i;
}
}
if (self48->index[i] >= 0) {
assert(self48->children[self48->index[i]] != nullptr);
return i;
}
i -= 16;
for (; i >= 0; i -= 16) {
auto result = lastNonNeg1(self48->index + i);
if (result != -1) {
return i + result;
}
}
#else
for (int i = child; i >= 0; --i) {
if (self48->index[i] >= 0) {
assert(self48->children[self48->index[i]] != nullptr);
return i;
}
}
#endif
} else {
auto *self256 = static_cast<Node256 *>(self);
// TODO: The plain loop is faster?
#if defined(__clang__)
int i = child;
constexpr int kUnrollCount = 8; // Must be a power of two and <= 8
for (; (i & (kUnrollCount - 1)) != 0; --i) {
if (self256->children[i]) {
return i;
}
}
if (self256->children[i]) {
return i;
}
i -= kUnrollCount;
for (; i >= 0; i -= kUnrollCount) {
uint8_t nonNull[kUnrollCount];
for (int j = 0; j < kUnrollCount; ++j) {
nonNull[j] = self256->children[i + j] != nullptr ? 0xff : 0;
}
uint64_t word;
memcpy(&word, nonNull, kUnrollCount);
if (word) {
return i + 7 - __builtin_clzll(word) / 8;
}
}
#else
for (int i = child; i >= 0; --i) {
if (self256->children[i]) {
return i;
}
}
#endif
}
return -1;
}
void setChildrenParents(Node *node) {
for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) {
getChildExists(node, i)->parent = node;
}
}
Node *&getOrCreateChild(Node *&self, uint8_t index) {
if (self->type == Type::Node4) {
auto *self4 = static_cast<Node4 *>(self);
{
int i = getNodeIndex(self4, index);
if (i >= 0) {
return self4->children[i];
}
}
if (self->numChildren == 4) {
auto *newSelf = new (safe_malloc(sizeof(Node16))) Node16;
memcpy((void *)newSelf, self, offsetof(Node, type));
memcpy(newSelf->index, self4->index, 4);
memcpy(newSelf->children, self4->children, 4 * sizeof(void *));
free(std::exchange(self, newSelf));
setChildrenParents(self);
goto insert16;
} else {
++self->numChildren;
for (int i = 0; i < int(self->numChildren) - 1; ++i) {
if (int(self4->index[i]) > int(index)) {
memmove(self4->index + i + 1, self4->index + i,
self->numChildren - (i + 1));
memmove(self4->children + i + 1, self4->children + i,
(self->numChildren - (i + 1)) * sizeof(void *));
self4->index[i] = index;
self4->children[i] = nullptr;
return self4->children[i];
}
}
self4->index[self->numChildren - 1] = index;
self4->children[self->numChildren - 1] = nullptr;
return self4->children[self->numChildren - 1];
}
} else if (self->type == Type::Node16) {
insert16:
auto *self16 = static_cast<Node16 *>(self);
{
int i = getNodeIndex(self16, index);
if (i >= 0) {
return self16->children[i];
}
}
if (self->numChildren == 16) {
auto *newSelf = new (safe_malloc(sizeof(Node48))) Node48;
memcpy((void *)newSelf, self, offsetof(Node, type));
newSelf->nextFree = 16;
int i = 0;
for (auto x : self16->index) {
newSelf->children[i] = self16->children[i];
newSelf->index[x] = i;
++i;
}
assert(i == 16);
free(std::exchange(self, newSelf));
setChildrenParents(self);
goto insert48;
} else {
++self->numChildren;
for (int i = 0; i < int(self->numChildren) - 1; ++i) {
if (int(self16->index[i]) > int(index)) {
memmove(self16->index + i + 1, self16->index + i,
self->numChildren - (i + 1));
memmove(self16->children + i + 1, self16->children + i,
(self->numChildren - (i + 1)) * sizeof(void *));
self16->index[i] = index;
self16->children[i] = nullptr;
return self16->children[i];
}
}
self16->index[self->numChildren - 1] = index;
self16->children[self->numChildren - 1] = nullptr;
return self16->children[self->numChildren - 1];
}
} else if (self->type == Type::Node48) {
insert48:
auto *self48 = static_cast<Node48 *>(self);
int secondIndex = self48->index[index];
if (secondIndex >= 0) {
return self48->children[secondIndex];
}
if (self->numChildren == 48) {
auto *newSelf = new (safe_malloc(sizeof(Node256))) Node256;
memcpy((void *)newSelf, self, offsetof(Node, type));
for (int i = 0; i < 256; ++i) {
if (self48->index[i] >= 0) {
newSelf->children[i] = self48->children[self48->index[i]];
}
}
free(std::exchange(self, newSelf));
self = newSelf;
setChildrenParents(self);
goto insert256;
} else {
++self->numChildren;
assert(self48->nextFree < 48);
self48->index[index] = self48->nextFree;
self48->children[self48->nextFree] = nullptr;
return self48->children[self48->nextFree++];
}
} else {
insert256:
auto *self256 = static_cast<Node256 *>(self);
if (!self256->children[index]) {
++self->numChildren;
}
return self256->children[index];
}
}
// Precondition - an entry for index must exist in the node
[[maybe_unused]] void eraseChild(Node *self, uint8_t index) {
if (self->type == Type::Node4) {
auto *self4 = static_cast<Node4 *>(self);
int nodeIndex = getNodeIndex(self4, index);
memmove(self4->index + nodeIndex, self4->index + nodeIndex + 1,
sizeof(self4->index[0]) * (self->numChildren - (nodeIndex + 1)));
memmove(self4->children + nodeIndex, self4->children + nodeIndex + 1,
sizeof(self4->children[0]) * // NOLINT
(self->numChildren - (nodeIndex + 1)));
} else if (self->type == Type::Node16) {
auto *self16 = static_cast<Node16 *>(self);
int nodeIndex = getNodeIndex(self16, index);
memmove(self16->index + nodeIndex, self16->index + nodeIndex + 1,
sizeof(self16->index[0]) * (self->numChildren - (nodeIndex + 1)));
memmove(self16->children + nodeIndex, self16->children + nodeIndex + 1,
sizeof(self16->children[0]) * // NOLINT
(self->numChildren - (nodeIndex + 1)));
} else if (self->type == Type::Node48) {
auto *self48 = static_cast<Node48 *>(self);
int8_t toRemoveChildrenIndex = std::exchange(self48->index[index], -1);
int8_t lastChildrenIndex = --self48->nextFree;
assert(toRemoveChildrenIndex >= 0);
assert(lastChildrenIndex >= 0);
if (toRemoveChildrenIndex != lastChildrenIndex) {
self48->children[toRemoveChildrenIndex] =
std::exchange(self48->children[lastChildrenIndex], nullptr);
self48->index[self48->children[toRemoveChildrenIndex]->parentsIndex] =
toRemoveChildrenIndex;
}
} else {
auto *self256 = static_cast<Node256 *>(self);
self256->children[index] = nullptr;
}
--self->numChildren;
if (self->numChildren == 0 && !self->entryPresent &&
self->parent != nullptr) {
eraseChild(self->parent, self->parentsIndex);
}
}
Node *nextPhysical(Node *node) {
int index = -1;
for (;;) {
auto nextChild = getChildGeq(node, index + 1);
if (nextChild >= 0) {
return getChildExists(node, nextChild);
}
index = node->parentsIndex;
node = node->parent;
if (node == nullptr) {
return nullptr;
}
}
}
Node *nextLogical(Node *node) {
for (node = nextPhysical(node); node != nullptr && !node->entryPresent;
node = nextPhysical(node))
;
return node;
}
Node *prevPhysical(Node *node) {
assert(node->parent != nullptr);
auto prevChild = getChildLeq(node->parent, node->parentsIndex - 1);
assert(prevChild < node->parentsIndex);
if (prevChild >= 0) {
node = getChildExists(node->parent, prevChild);
// Move down the right spine
for (;;) {
auto rightMostChild = getChildLeq(node, 255);
if (rightMostChild >= 0) {
node = getChildExists(node, rightMostChild);
} else {
return node;
}
}
} else {
return node->parent;
}
}
struct Iterator {
Node *n;
int cmp;
};
std::string_view getSearchPath(Arena &arena, Node *n);
Iterator lastLeq(Node *n, const std::span<const uint8_t> key) {
auto remaining = key;
for (;;) {
Arena arena;
assert((std::string(getSearchPath(arena, n)) +
std::string((const char *)remaining.data(), remaining.size()))
.ends_with(std::string((const char *)key.data(), key.size())));
if (remaining.size() == 0) {
// We've found the physical node corresponding to search path `key`
if (n->entryPresent) {
return {n, 0};
} else {
break;
}
} else {
int c = getChildLeq(n, remaining[0]);
if (c == remaining[0]) {
n = getChildExists(n, c);
remaining = remaining.subspan(1, remaining.size() - 1);
} else {
// The physical node corresponding to search path `key` does not exist.
// Let's find the physical node corresponding to the highest search key
// (not necessarily present) less than key
// Move down the right spine
for (;;) {
if (c >= 0) {
n = getChildExists(n, c);
} else {
break;
}
c = getChildLeq(n, 255);
}
break;
}
}
}
// Iterate backwards along existing physical nodes until we find a present
// entry
for (; !n->entryPresent; n = prevPhysical(n)) {
}
return {n, -1};
}
void insert(Node **self_, std::span<const uint8_t> key, int64_t writeVersion) {
for (;;) {
auto &self = *self_;
self->maxVersion = writeVersion;
if (key.size() == 0) {
auto l = lastLeq(self, key);
self->entryPresent = true;
self->entry.pointVersion = writeVersion;
assert(l.n != nullptr);
assert(l.n->entryPresent);
self->entry.rangeVersion = l.n->entry.rangeVersion;
return;
}
auto &child = getOrCreateChild(self, key.front());
if (!child) {
child = newNode();
child->parent = self;
child->parentsIndex = key.front();
}
self_ = &child;
key = key.subspan(1, key.size() - 1);
}
}
std::string printable(std::string_view key);
std::string printable(const Key &key);
} // namespace
struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
void check(const ReadRange *reads, Result *result, int count) const {
for (int i = 0; i < count; ++i) {
const auto &r = reads[i];
if (r.readVersion < oldestVersion) {
result[i] = TooOld;
continue;
}
// TODO support non-point reads
assert(r.end.len == 0);
auto [l, c] =
lastLeq(root, std::span<const uint8_t>(r.begin.p, r.begin.len));
#if DEBUG_VERBOSE && !defined(NDEBUG)
Arena arena;
printf("LastLeq for `%s' got `%s'\n", printable(r.begin).c_str(),
printable(getSearchPath(arena, l)).c_str());
#endif
assert(l != nullptr);
assert(l->entryPresent);
result[i] = (c == 0 ? l->entry.pointVersion : l->entry.rangeVersion) >
r.readVersion
? Conflict
: Commit;
}
}
void addWrites(const WriteRange *writes, int count) {
for (int i = 0; i < count; ++i) {
const auto &w = writes[i];
// TODO support non-point writes
assert(w.end.len == 0);
insert(&root, std::span<const uint8_t>(w.begin.p, w.begin.len),
w.writeVersion);
}
}
void setOldestVersion(int64_t oldestVersion) {
this->oldestVersion = oldestVersion;
}
explicit Impl(int64_t oldestVersion) : oldestVersion(oldestVersion) {
// Insert ""
root = newNode();
root->maxVersion = oldestVersion;
root->entry.pointVersion = oldestVersion;
root->entry.rangeVersion = oldestVersion;
root->entryPresent = true;
}
~Impl() {
Arena arena;
auto toFree = vector<Node *>(arena);
toFree.push_back(root);
while (toFree.size() > 0) {
auto *n = toFree.back();
toFree.pop_back();
// Add all children to toFree
for (int child = getChildGeq(n, 0); child >= 0;
child = getChildGeq(n, child + 1)) {
auto *c = getChildExists(n, child);
assert(c != nullptr);
toFree.push_back(c);
}
free(n);
}
}
Node *root;
int64_t oldestVersion;
};
// ==================== END IMPLEMENTATION ====================
// GCOVR_EXCL_START
void ConflictSet::check(const ReadRange *reads, Result *results,
int count) const {
return impl->check(reads, results, count);
}
void ConflictSet::addWrites(const WriteRange *writes, int count) {
return impl->addWrites(writes, count);
}
void ConflictSet::setOldestVersion(int64_t oldestVersion) {
return impl->setOldestVersion(oldestVersion);
}
ConflictSet::ConflictSet(int64_t oldestVersion, [[maybe_unused]] uint64_t seed)
: impl(new (safe_malloc(sizeof(Impl))) Impl{oldestVersion}) {}
ConflictSet::~ConflictSet() {
if (impl) {
impl->~Impl();
free(impl);
}
}
ConflictSet::ConflictSet(ConflictSet &&other) noexcept
: impl(std::exchange(other.impl, nullptr)) {}
ConflictSet &ConflictSet::operator=(ConflictSet &&other) noexcept {
impl = std::exchange(other.impl, nullptr);
return *this;
}
using ConflictSet_Result = ConflictSet::Result;
using ConflictSet_Key = ConflictSet::Key;
using ConflictSet_ReadRange = ConflictSet::ReadRange;
using ConflictSet_WriteRange = ConflictSet::WriteRange;
extern "C" {
__attribute__((__visibility__("default"))) void
ConflictSet_check(void *cs, const ConflictSet_ReadRange *reads,
ConflictSet_Result *results, int count) {
((ConflictSet::Impl *)cs)->check(reads, results, count);
}
__attribute__((__visibility__("default"))) void
ConflictSet_addWrites(void *cs, const ConflictSet_WriteRange *writes,
int count) {
((ConflictSet::Impl *)cs)->addWrites(writes, count);
}
__attribute__((__visibility__("default"))) void
ConflictSet_setOldestVersion(void *cs, int64_t oldestVersion) {
((ConflictSet::Impl *)cs)->setOldestVersion(oldestVersion);
}
__attribute__((__visibility__("default"))) void *
ConflictSet_create(int64_t oldestVersion, uint64_t) {
return new (safe_malloc(sizeof(ConflictSet::Impl)))
ConflictSet::Impl{oldestVersion};
}
__attribute__((__visibility__("default"))) void ConflictSet_destroy(void *cs) {
using Impl = ConflictSet::Impl;
((Impl *)cs)->~Impl();
free(cs);
}
}
namespace {
std::string printable(std::string_view key) {
std::string result;
for (uint8_t c : key) {
result += "x";
result += "0123456789abcdef"[c / 16];
result += "0123456789abcdef"[c % 16];
}
return result;
}
std::string printable(const Key &key) {
return printable(std::string_view((const char *)key.p, key.len));
}
std::string_view getSearchPath(Arena &arena, Node *n) {
if (n->parent == nullptr) {
return {};
}
auto result = vector<char>(arena);
for (; n->parent != nullptr; n = n->parent) {
result.push_back(n->parentsIndex);
}
std::reverse(result.begin(), result.end());
return std::string_view((const char *)&result[0], result.size()); // NOLINT
}
void printLogical(std::string &result, Node *node) {
Arena arena;
for (Node *iter = node; iter != nullptr;) {
auto *next = nextLogical(iter);
std::string key;
for (uint8_t c : getSearchPath(arena, iter)) {
key += "x";
key += "0123456789abcdef"[c / 16];
key += "0123456789abcdef"[c % 16];
}
if (iter->entry.pointVersion == iter->entry.rangeVersion) {
result += key + " -> " + std::to_string(iter->entry.pointVersion) + "\n";
} else {
result += key + " -> " + std::to_string(iter->entry.pointVersion) + "\n";
if (next == nullptr || (getSearchPath(arena, next) !=
(std::string(getSearchPath(arena, iter)) +
std::string("\x00", 1)))) {
result +=
key + "x00 -> " + std::to_string(iter->entry.rangeVersion) + "\n";
}
}
iter = next;
}
}
[[maybe_unused]] void debugPrintDot(FILE *file, Node *node) {
struct DebugDotPrinter {
explicit DebugDotPrinter(FILE *file) : file(file) {}
void print(Node *n) {
assert(n != nullptr);
if (n->entryPresent) {
fprintf(file, " k_%p [label=\"m=%d p=%d r=%d\"];\n", (void *)n,
int(n->maxVersion), int(n->entry.pointVersion),
int(n->entry.rangeVersion));
} else {
fprintf(file, " k_%p [label=\"m=%d\"];\n", (void *)n,
int(n->maxVersion));
}
for (int child = getChildGeq(n, 0); child >= 0;
child = getChildGeq(n, child + 1)) {
auto *c = getChildExists(n, child);
fprintf(file, " k_%p -> k_%p [label=\"'%02x'\"];\n", (void *)n,
(void *)c, child);
print(c);
}
}
FILE *file;
};
fprintf(file, "digraph ConflictSet {\n");
assert(node != nullptr);
DebugDotPrinter printer{file};
printer.print(node);
fprintf(file, "}\n");
}
void checkParentPointers(Node *node, bool &success) {
for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) {
auto *child = getChildExists(node, i);
if (child->parent != node) {
Arena arena;
fprintf(stderr, "%s child %d has parent pointer %p. Expected %p\n",
printable(getSearchPath(arena, node)).c_str(), i,
(void *)child->parent, (void *)node);
success = false;
}
checkParentPointers(child, success);
}
}
[[maybe_unused]] int64_t checkMaxVersion(Node *node, bool &success) {
int64_t expected =
node->entryPresent
? std::max(node->entry.pointVersion, node->entry.rangeVersion)
: std::numeric_limits<int64_t>::lowest();
for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) {
auto *child = getChildExists(node, i);
expected = std::max(expected, checkMaxVersion(child, success));
}
if (node->maxVersion != expected) {
Arena arena;
fprintf(stderr, "%s has max version %d. Expected %d\n",
printable(getSearchPath(arena, node)).c_str(),
int(node->maxVersion), int(expected));
success = false;
}
return expected;
}
bool checkCorrectness(Node *node, ReferenceImpl &refImpl) {
bool success = true;
checkParentPointers(node, success);
std::string logicalMap;
std::string referenceLogicalMap;
printLogical(logicalMap, node);
refImpl.printLogical(referenceLogicalMap);
if (logicalMap != referenceLogicalMap) {
fprintf(stderr,
"Logical map not equal to reference logical map.\n\nActual:\n"
"%s\nExpected:\n%s\n",
logicalMap.c_str(), referenceLogicalMap.c_str());
success = false;
}
return success;
}
} // namespace
namespace std {
void __throw_length_error(const char *) { __builtin_unreachable(); }
} // namespace std
#ifdef ENABLE_MAIN
int main(void) {
int64_t writeVersion = 0;
ConflictSet::Impl cs{writeVersion};
ReferenceImpl refImpl{writeVersion};
Arena arena;
constexpr int kNumKeys = 10;
auto *write = new (arena) ConflictSet::WriteRange[kNumKeys];
for (int i = 0; i < kNumKeys; ++i) {
write[i].begin = toKey(arena, i);
write[i].end.len = 0;
write[i].writeVersion = ++writeVersion;
}
cs.addWrites(write, kNumKeys);
refImpl.addWrites(write, kNumKeys);
debugPrintDot(stdout, cs.root);
return 0;
}
#endif
#ifdef ENABLE_FUZZ
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
// TODO call setOldestVersion, and check range writes/reads
Arbitrary arbitrary{{data, size}};
int64_t writeVersion = 0;
ConflictSet::Impl cs{writeVersion};
ReferenceImpl refImpl{writeVersion};
while (arbitrary.hasEntropy()) {
Arena arena;
{
int numWrites = arbitrary.bounded(10);
int64_t v = ++writeVersion;
auto *writes = new (arena) ConflictSet::WriteRange[numWrites];
auto keys = set<std::string_view>(arena);
while (int(keys.size()) < numWrites) {
if (!arbitrary.hasEntropy()) {
// Tell the fuzzer it's not interesting
return -1;
}
int keyLen = arbitrary.bounded(8);
auto *begin = new (arena) uint8_t[keyLen];
arbitrary.randomBytes(begin, keyLen);
keys.insert(std::string_view((const char *)begin, keyLen));
}
auto iter = keys.begin();
for (int i = 0; i < numWrites; ++i) {
writes[i].begin.p = (const uint8_t *)iter->data();
writes[i].begin.len = iter->size();
++iter;
writes[i].end.len = 0;
writes[i].writeVersion = v;
#if DEBUG_VERBOSE && !defined(NDEBUG)
printf("Write: {%s} -> %d\n", printable(writes[i].begin).c_str(),
int(writes[i].writeVersion));
#endif
}
assert(iter == keys.end());
cs.addWrites(writes, numWrites);
refImpl.addWrites(writes, numWrites);
}
bool success = checkCorrectness(cs.root, refImpl);
if (!success) {
abort();
}
{
int numReads = arbitrary.bounded(10);
int64_t v = writeVersion - arbitrary.bounded(10);
auto *reads = new (arena) ConflictSet::ReadRange[numReads];
auto keys = set<std::string_view>(arena);
while (int(keys.size()) < numReads) {
if (!arbitrary.hasEntropy()) {
// Tell the fuzzer it's not interesting
return -1;
}
int keyLen = arbitrary.bounded(8);
auto *begin = new (arena) uint8_t[keyLen];
arbitrary.randomBytes(begin, keyLen);
keys.insert(std::string_view((const char *)begin, keyLen));
}
auto iter = keys.begin();
for (int i = 0; i < numReads; ++i) {
reads[i].begin.p = (const uint8_t *)iter->data();
reads[i].begin.len = iter->size();
++iter;
reads[i].end.len = 0;
reads[i].readVersion = v;
#if DEBUG_VERBOSE && !defined(NDEBUG)
printf("Read: {%s} at %d\n", printable(reads[i].begin).c_str(),
int(reads[i].readVersion));
#endif
}
assert(iter == keys.end());
auto *results1 = new (arena) ConflictSet::Result[numReads];
auto *results2 = new (arena) ConflictSet::Result[numReads];
cs.check(reads, results1, numReads);
refImpl.check(reads, results2, numReads);
for (int i = 0; i < numReads; ++i) {
if (results1[i] != results2[i]) {
fprintf(stderr, "Expected %d, got %d for read of %s at version %d\n",
results2[i], results1[i], printable(reads[i].begin).c_str(),
int(reads[i].readVersion));
std::string referenceLogicalMap;
refImpl.printLogical(referenceLogicalMap);
fprintf(stderr, "Logical map:\n\n%s\n", referenceLogicalMap.c_str());
abort();
}
}
}
}
return 0;
}
#endif
// GCOVR_EXCL_STOP
Reference in New Issue View Git Blame Copy Permalink