#include "ConflictSet.h" #include #include #include #include #include #include #include #include #include #include #include #include #ifdef HAS_AVX #include #elif defined(HAS_ARM_NEON) #include #endif #define DEBUG_VERBOSE 0 // GCOVR_EXCL_START __attribute__((always_inline)) inline void *safe_malloc(size_t s) { if (void *p = malloc(s)) { return p; } abort(); } // ==================== BEGIN ARENA IMPL ==================== /// Group allocations with similar lifetimes to amortize the cost of malloc/free struct Arena { explicit Arena(int initialSize = 0); /// O(log n) in the number of allocations ~Arena(); struct ArenaImpl; Arena(const Arena &) = delete; Arena &operator=(const Arena &) = delete; Arena(Arena &&other) noexcept; Arena &operator=(Arena &&other) noexcept; private: ArenaImpl *impl = nullptr; friend void *operator new(size_t size, std::align_val_t align, Arena &arena); }; inline void operator delete(void *, std::align_val_t, Arena &) {} void *operator new(size_t size, std::align_val_t align, Arena &arena); void *operator new(size_t size, std::align_val_t align, Arena *arena) = delete; inline void operator delete(void *, Arena &) {} inline void *operator new(size_t size, Arena &arena) { return operator new(size, std::align_val_t(alignof(std::max_align_t)), arena); } inline void *operator new(size_t size, Arena *arena) = delete; inline void operator delete[](void *, Arena &) {} inline void *operator new[](size_t size, Arena &arena) { return operator new(size, arena); } inline void *operator new[](size_t size, Arena *arena) = delete; inline void operator delete[](void *, std::align_val_t, Arena &) {} inline void *operator new[](size_t size, std::align_val_t align, Arena &arena) { return operator new(size, align, arena); } inline void *operator new[](size_t size, std::align_val_t align, Arena *arena) = delete; /// align must be a power of two template T *align_up(T *t, size_t align) { auto unaligned = uintptr_t(t); auto aligned = (unaligned + align - 1) & ~(align - 1); return reinterpret_cast(reinterpret_cast(t) + aligned - unaligned); } /// align must be a power of two constexpr inline int align_up(uint32_t unaligned, uint32_t align) { return (unaligned + align - 1) & ~(align - 1); } /// Returns the smallest power of two >= x constexpr inline uint32_t nextPowerOfTwo(uint32_t x) { return x <= 1 ? 1 : 1 << (32 - __builtin_clz(x - 1)); } struct Arena::ArenaImpl { Arena::ArenaImpl *prev; int capacity; int used; uint8_t *begin() { return reinterpret_cast(this + 1); } }; static_assert(sizeof(Arena::ArenaImpl) == 16); static_assert(alignof(Arena::ArenaImpl) == 8); Arena::Arena(int initialSize) : impl(nullptr) { if (initialSize > 0) { auto allocationSize = align_up(initialSize + sizeof(ArenaImpl), 16); impl = (Arena::ArenaImpl *)safe_malloc(allocationSize); impl->prev = nullptr; impl->capacity = allocationSize - sizeof(ArenaImpl); impl->used = 0; } } void onDestroy(Arena::ArenaImpl *impl) { while (impl) { auto *prev = impl->prev; free(impl); impl = prev; } } Arena::Arena(Arena &&other) noexcept : impl(std::exchange(other.impl, nullptr)) {} Arena &Arena::operator=(Arena &&other) noexcept { onDestroy(impl); impl = std::exchange(other.impl, nullptr); return *this; } Arena::~Arena() { onDestroy(impl); } void *operator new(size_t size, std::align_val_t align, Arena &arena) { int64_t aligned_size = size + size_t(align) - 1; if (arena.impl == nullptr || (arena.impl->capacity - arena.impl->used) < aligned_size) { auto allocationSize = align_up( sizeof(Arena::ArenaImpl) + std::max(aligned_size, (arena.impl ? std::max(sizeof(Arena::ArenaImpl), arena.impl->capacity * 2) : 0)), 16); auto *impl = (Arena::ArenaImpl *)safe_malloc(allocationSize); impl->prev = arena.impl; impl->capacity = allocationSize - sizeof(Arena::ArenaImpl); impl->used = 0; arena.impl = impl; } auto *result = align_up(arena.impl->begin() + arena.impl->used, size_t(align)); auto usedDelta = (result - arena.impl->begin()) + size - arena.impl->used; arena.impl->used += usedDelta; return result; } /// STL-friendly allocator using an arena template struct ArenaAlloc { typedef T value_type; ArenaAlloc() = delete; explicit ArenaAlloc(Arena *arena) : arena(arena) {} Arena *arena; template constexpr ArenaAlloc(const ArenaAlloc &other) noexcept { arena = other.arena; } [[nodiscard]] T *allocate(size_t n) { if (n > 0xfffffffffffffffful / sizeof(T)) { // NOLINT __builtin_unreachable(); } return static_cast((void *)new (std::align_val_t(alignof(T)), *arena) uint8_t[n * sizeof(T)]); // NOLINT } void deallocate(T *, size_t) noexcept {} }; template using Vector = std::vector>; template auto vector(Arena &arena) { return Vector(ArenaAlloc(&arena)); } template using Set = std::set, ArenaAlloc>; template auto set(Arena &arena) { return Set(ArenaAlloc(&arena)); } template bool operator==(const ArenaAlloc &lhs, const ArenaAlloc &rhs) { return lhs.arena == rhs.arena; } template bool operator!=(const ArenaAlloc &lhs, const ArenaAlloc &rhs) { return !(lhs == rhs); } // ==================== END ARENA IMPL ==================== // ==================== BEGIN ARBITRARY IMPL ==================== /// Think of `Arbitrary` as an attacker-controlled random number generator. /// Usually you want your random number generator to be fair, so that you can /// sensibly analyze probabilities. E.g. The analysis that shows that quicksort /// is expected O(n log n) with a random pivot relies on the random pivot being /// selected uniformly from a fair distribution. /// /// Other times you want your randomness to be diabolically unfair, like when /// looking for bugs and fuzzing. The random-number-like interface is still /// convenient here, but you can potentially get much better coverage by /// allowing the possibility of e.g. flipping heads 100 times in a row. /// /// When it runs out of entropy, it always returns 0. struct Arbitrary { Arbitrary() = default; explicit Arbitrary(std::span bytecode) : bytecode(bytecode) {} /// Draws an arbitrary uint32_t uint32_t next() { return consume<4>(); } /// Draws an arbitrary element from [0, s) uint32_t bounded(uint32_t s); /// Fill `bytes` with `size` arbitrary bytes void randomBytes(uint8_t *bytes, int size) { int toFill = std::min(size, bytecode.size()); if (toFill > 0) { memcpy(bytes, bytecode.data(), toFill); } bytecode = bytecode.subspan(toFill, bytecode.size() - toFill); memset(bytes + toFill, 0, size - toFill); } /// Fill `bytes` with `size` random hex bytes void randomHex(uint8_t *bytes, int size) { for (int i = 0; i < size;) { uint8_t arbitrary = consume<1>(); bytes[i++] = "0123456789abcdef"[arbitrary & 0xf]; arbitrary >>= 4; if (i < size) { bytes[i++] = "0123456789abcdef"[arbitrary & 0xf]; } } } template >> T randT() { T t; randomBytes((uint8_t *)&t, sizeof(T)); return t; } bool hasEntropy() const { return bytecode.size() != 0; } private: uint8_t consumeByte() { if (bytecode.size() == 0) { return 0; } auto result = bytecode[0]; bytecode = bytecode.subspan(1, bytecode.size() - 1); return result; } template uint32_t consume() { uint32_t result = 0; static_assert(kBytes <= 4); for (int i = 0; i < kBytes; ++i) { result <<= 8; result |= consumeByte(); } return result; } std::span bytecode; }; Arbitrary gArbitrary; uint32_t Arbitrary::bounded(uint32_t s) { if (s == 1) { return 0; } switch (32 - __builtin_clz(s - 1)) { case 1: case 2: case 3: case 4: case 5: case 6: case 7: case 8: return consume<1>() % s; case 9: case 10: case 11: case 12: case 13: case 14: case 15: case 16: return consume<2>() % s; case 17: case 18: case 19: case 20: case 21: case 22: case 23: case 24: return consume<3>() % s; default: return consume<4>() % s; } } // ==================== END ARBITRARY IMPL ==================== // ==================== BEGIN UTILITIES IMPL ==================== // Call Stepwise::step for each element of remaining until it returns true. // Applies a permutation to `remaining` as a side effect. template void runInterleaved(std::span remaining) { while (remaining.size() > 0) { for (int i = 0; i < int(remaining.size());) { bool done = remaining[i].step(); if (done) { if (i != int(remaining.size()) - 1) { using std::swap; swap(remaining[i], remaining.back()); } remaining = remaining.subspan(0, remaining.size() - 1); } else { ++i; } } } }; template void runSequential(std::span remaining) { for (auto &r : remaining) { while (!r.step()) { } } } struct ReferenceImpl { explicit ReferenceImpl(int64_t oldestVersion) : oldestVersion(oldestVersion) { writeVersionMap[""] = oldestVersion; } void check(const ConflictSet::ReadRange *reads, ConflictSet::Result *results, int count) const { for (int i = 0; i < count; ++i) { if (reads[i].readVersion < oldestVersion) { results[i] = ConflictSet::TooOld; continue; } auto begin = std::string((const char *)reads[i].begin.p, reads[i].begin.len); auto end = reads[i].end.len == 0 ? begin + std::string("\x00", 1) : std::string((const char *)reads[i].end.p, reads[i].end.len); int64_t maxVersion = oldestVersion; for (auto iter = --writeVersionMap.upper_bound(begin), endIter = writeVersionMap.lower_bound(end); iter != endIter; ++iter) { maxVersion = std::max(maxVersion, iter->second); } results[i] = maxVersion > reads[i].readVersion ? ConflictSet::Conflict : ConflictSet::Commit; } } void addWrites(const ConflictSet::WriteRange *writes, int count) { for (int i = 0; i < count; ++i) { auto begin = std::string((const char *)writes[i].begin.p, writes[i].begin.len); auto end = writes[i].end.len == 0 ? begin + std::string("\x00", 1) : std::string((const char *)writes[i].end.p, writes[i].end.len); auto writeVersion = writes[i].writeVersion; auto prevVersion = (--writeVersionMap.upper_bound(end))->second; for (auto iter = writeVersionMap.lower_bound(begin), endIter = writeVersionMap.lower_bound(end); iter != endIter;) { iter = writeVersionMap.erase(iter); } writeVersionMap[begin] = writeVersion; writeVersionMap[end] = prevVersion; } } void setOldestVersion(int64_t oldestVersion) { this->oldestVersion = oldestVersion; } void printLogical(std::string &result) { for (const auto &[k, v] : writeVersionMap) { std::string key; for (uint8_t c : k) { key += "x"; key += "0123456789abcdef"[c / 16]; key += "0123456789abcdef"[c % 16]; } result += key + " -> " + std::to_string(v) + "\n"; } } int64_t oldestVersion; std::map writeVersionMap; }; using Key = ConflictSet::Key; Key toKey(Arena &arena, int n) { constexpr int kMaxLength = 8; int i = kMaxLength; uint8_t *itoaBuf = new (arena) uint8_t[kMaxLength]; memset(itoaBuf, '0', kMaxLength); do { itoaBuf[--i] = "0123456789abcdef"[n % 16]; n /= 16; } while (n); return Key{itoaBuf, kMaxLength}; } Key toKeyAfter(Arena &arena, int n) { constexpr int kMaxLength = 8; int i = kMaxLength; uint8_t *itoaBuf = new (arena) uint8_t[kMaxLength + 1]; memset(itoaBuf, '0', kMaxLength); itoaBuf[kMaxLength] = 0; do { itoaBuf[--i] = "0123456789abcdef"[n % 16]; n /= 16; } while (n); return Key{itoaBuf, kMaxLength + 1}; } // ==================== END UTILITIES IMPL ==================== // ==================== BEGIN IMPLEMENTATION ==================== // GCOVR_EXCL_STOP struct Entry { int64_t pointVersion; int64_t rangeVersion; }; enum class Type : int8_t { Node4, Node16, Node48, Node256, Invalid, }; struct Node { /* begin section that's copied to the next node */ Node *parent = nullptr; int64_t maxVersion; Entry entry; int16_t numChildren = 0; bool entryPresent = false; uint8_t parentsIndex = 0; /* end section that's copied to the next node */ Type type = Type::Invalid; }; Node *getChild(Node *self, uint8_t index); int getChildGeq(Node *self, int child); Node *&getOrCreateChild(Node *&self, uint8_t index); Node *newNode(); void eraseChild(Node *self, uint8_t index); struct Node4 : Node { // Sorted uint8_t index[4] = {}; Node *children[4] = {}; Node4() { this->type = Type::Node4; } }; Node *newNode() { return new (safe_malloc(sizeof(Node4))) Node4; } struct Node16 : Node { // Sorted uint8_t index[16] = {}; Node *children[16] = {}; Node16() { this->type = Type::Node16; } }; struct Node48 : Node { int8_t nextFree = 0; int8_t index[256]; Node *children[48] = {}; Node48() { this->type = Type::Node48; memset(index, -1, 256); } }; struct Node256 : Node { Node *children[256] = {}; Node256() { this->type = Type::Node256; } }; int getNodeIndex(Node4 *self, uint8_t index) { for (int i = 0; i < self->numChildren; ++i) { if (self->index[i] == index) { return i; } } return -1; } int getNodeIndex(Node16 *self, uint8_t index) { #ifdef HAS_AVX // Based on https://www.the-paper-trail.org/post/art-paper-notes/ // key_vec is 16 repeated copies of the searched-for byte, one for every // possible position in child_keys that needs to be searched. __m128i key_vec = _mm_set1_epi8(index); // Compare all child_keys to 'index' in parallel. Don't worry if some of the // keys aren't valid, we'll mask the results to only consider the valid ones // below. __m128i indices; memcpy(&indices, self->index, sizeof(self->index)); __m128i results = _mm_cmpeq_epi8(key_vec, indices); // Build a mask to select only the first node->num_children values from the // comparison (because the other values are meaningless) int mask = (1 << self->numChildren) - 1; // Change the results of the comparison into a bitfield, masking off any // invalid comparisons. int bitfield = _mm_movemask_epi8(results) & mask; // No match if there are no '1's in the bitfield. if (bitfield == 0) return -1; // Find the index of the first '1' in the bitfield by counting the leading // zeros. return __builtin_ctz(bitfield); #elif defined(HAS_ARM_NEON) // Based on // https://community.arm.com/arm-community-blogs/b/infrastructure-solutions-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon uint8x16_t indices; memcpy(&indices, self->index, sizeof(self->index)); // 0xff for each match uint16x8_t results = vreinterpretq_u16_u8(vceqq_u8(vdupq_n_u8(index), indices)); uint64_t mask = self->numChildren == 16 ? uint64_t(-1) : (uint64_t(1) << (self->numChildren * 4)) - 1; // 0xf for each match in valid range uint64_t bitfield = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(results, 4)), 0) & mask; if (bitfield == 0) return -1; return __builtin_ctzll(bitfield) / 4; #else for (int i = 0; i < self->numChildren; ++i) { if (self->index[i] == index) { return i; } } return -1; #endif } #ifdef HAS_AVX int firstNonNeg1(const int8_t x[16]) { __m128i key_vec = _mm_set1_epi8(-1); __m128i indices; memcpy(&indices, x, 16); __m128i results = _mm_cmpeq_epi8(key_vec, indices); uint32_t bitfield = _mm_movemask_epi8(results) ^ 0xffff; if (bitfield == 0) return -1; return __builtin_ctz(bitfield); } int lastNonNeg1(const int8_t x[16]) { __m128i key_vec = _mm_set1_epi8(-1); __m128i indices; memcpy(&indices, x, 16); __m128i results = _mm_cmpeq_epi8(key_vec, indices); uint32_t bitfield = _mm_movemask_epi8(results) ^ 0xffff; if (bitfield == 0) return -1; return 31 - __builtin_clz(bitfield); } #endif #ifdef HAS_ARM_NEON int firstNonNeg1(const int8_t x[16]) { uint8x16_t indices; memcpy(&indices, x, 16); uint16x8_t results = vreinterpretq_u16_u8(vceqq_u8(vdupq_n_u8(-1), indices)); uint64_t bitfield = ~vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(results, 4)), 0); if (bitfield == 0) return -1; return __builtin_ctzll(bitfield) / 4; } int lastNonNeg1(const int8_t x[16]) { uint8x16_t indices; memcpy(&indices, x, 16); uint16x8_t results = vreinterpretq_u16_u8(vceqq_u8(vdupq_n_u8(-1), indices)); uint64_t bitfield = ~vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(results, 4)), 0); if (bitfield == 0) return -1; return 15 - __builtin_clzll(bitfield) / 4; } #endif Node *getChild(Node *self, uint8_t index) { if (self->type == Type::Node4) { auto *self4 = static_cast(self); int i = getNodeIndex(self4, index); if (i >= 0) { return self4->children[i]; } return nullptr; } else if (self->type == Type::Node16) { auto *self16 = static_cast(self); int i = getNodeIndex(self16, index); if (i >= 0) { return self16->children[i]; } return nullptr; } else if (self->type == Type::Node48) { auto *self48 = static_cast(self); int secondIndex = self48->index[index]; if (secondIndex >= 0) { return self48->children[secondIndex]; } return nullptr; } else { auto *self256 = static_cast(self); return self256->children[index]; } } // Precondition - an entry for index must exist in the node Node *&getChildExists(Node *self, uint8_t index) { if (self->type == Type::Node4) { auto *self4 = static_cast(self); return self4->children[getNodeIndex(self4, index)]; } else if (self->type == Type::Node16) { auto *self16 = static_cast(self); return self16->children[getNodeIndex(self16, index)]; } else if (self->type == Type::Node48) { auto *self48 = static_cast(self); int secondIndex = self48->index[index]; if (secondIndex >= 0) { return self48->children[secondIndex]; } } else { auto *self256 = static_cast(self); return self256->children[index]; } __builtin_unreachable(); // GCOVR_EXCL_LINE } int getChildGeq(Node *self, int child) { if (self->type == Type::Node4) { auto *self4 = static_cast(self); for (int i = 0; i < self->numChildren; ++i) { if (i > 0) { assert(self4->index[i - 1] < self4->index[i]); } if (self4->index[i] >= child) { return self4->index[i]; } } } else if (self->type == Type::Node16) { auto *self16 = static_cast(self); for (int i = 0; i < self->numChildren; ++i) { if (i > 0) { assert(self16->index[i - 1] < self16->index[i]); } if (self16->index[i] >= child) { return self16->index[i]; } } } else if (self->type == Type::Node48) { auto *self48 = static_cast(self); #if defined(HAS_AVX) || defined(HAS_ARM_NEON) int i = child; for (; (i & 0xf) != 0; ++i) { if (self48->index[i] >= 0) { assert(self48->children[self48->index[i]] != nullptr); return i; } } for (; i < 256; i += 16) { auto result = firstNonNeg1(self48->index + i); if (result != -1) { return i + result; } } #else for (int i = child; i < 256; ++i) { if (self48->index[i] >= 0) { assert(self48->children[self48->index[i]] != nullptr); return i; } } #endif } else { auto *self256 = static_cast(self); // For some reason gcc can't auto vectorize this, and the plain loop is // faster. #if defined(__clang__) int i = child; constexpr int kUnrollCount = 8; // Must be a power of two and <= 8 for (; (i & (kUnrollCount - 1)) != 0; ++i) { if (self256->children[i]) { return i; } } for (; i < 256; i += kUnrollCount) { uint8_t nonNull[kUnrollCount]; for (int j = 0; j < kUnrollCount; ++j) { nonNull[j] = self256->children[i + j] != nullptr ? 0xff : 0; } uint64_t word; memcpy(&word, nonNull, kUnrollCount); if (word) { return i + __builtin_ctzll(word) / 8; } } #else for (int i = child; i < 256; ++i) { if (self256->children[i]) { return i; } } #endif } return -1; } int getChildLeq(Node *self, int child) { if (self->type == Type::Node4) { auto *self4 = static_cast(self); for (int i = self->numChildren - 1; i >= 0; --i) { if (i > 0) { assert(self4->index[i - 1] < self4->index[i]); } if (self4->index[i] <= child) { return self4->index[i]; } } } else if (self->type == Type::Node16) { auto *self16 = static_cast(self); for (int i = self->numChildren - 1; i >= 0; --i) { if (i > 0) { assert(self16->index[i - 1] < self16->index[i]); } if (self16->index[i] <= child) { return self16->index[i]; } } } else if (self->type == Type::Node48) { auto *self48 = static_cast(self); // TODO the plain loop is faster? #if defined(HAS_AVX) || defined(HAS_ARM_NEON) int i = child; if (i < 0) { return -1; } for (; (i & 0xf) != 0; --i) { if (self48->index[i] >= 0) { assert(self48->children[self48->index[i]] != nullptr); return i; } } if (self48->index[i] >= 0) { assert(self48->children[self48->index[i]] != nullptr); return i; } i -= 16; for (; i >= 0; i -= 16) { auto result = lastNonNeg1(self48->index + i); if (result != -1) { return i + result; } } #else for (int i = child; i >= 0; --i) { if (self48->index[i] >= 0) { assert(self48->children[self48->index[i]] != nullptr); return i; } } #endif } else { auto *self256 = static_cast(self); // TODO: The plain loop is faster? #if defined(__clang__) int i = child; constexpr int kUnrollCount = 8; // Must be a power of two and <= 8 for (; (i & (kUnrollCount - 1)) != 0; --i) { if (self256->children[i]) { return i; } } if (self256->children[i]) { return i; } i -= kUnrollCount; for (; i >= 0; i -= kUnrollCount) { uint8_t nonNull[kUnrollCount]; for (int j = 0; j < kUnrollCount; ++j) { nonNull[j] = self256->children[i + j] != nullptr ? 0xff : 0; } uint64_t word; memcpy(&word, nonNull, kUnrollCount); if (word) { return i + 7 - __builtin_clzll(word) / 8; } } #else for (int i = child; i >= 0; --i) { if (self256->children[i]) { return i; } } #endif } return -1; } void setChildrenParents(Node *node) { for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) { getChildExists(node, i)->parent = node; } } Node *&getOrCreateChild(Node *&self, uint8_t index) { if (self->type == Type::Node4) { auto *self4 = static_cast(self); { int i = getNodeIndex(self4, index); if (i >= 0) { return self4->children[i]; } } if (self->numChildren == 4) { auto *newSelf = new (safe_malloc(sizeof(Node16))) Node16; memcpy((void *)newSelf, self, offsetof(Node, type)); memcpy(newSelf->index, self4->index, 4); memcpy(newSelf->children, self4->children, 4 * sizeof(void *)); free(std::exchange(self, newSelf)); setChildrenParents(self); goto insert16; } else { ++self->numChildren; for (int i = 0; i < int(self->numChildren) - 1; ++i) { if (int(self4->index[i]) > int(index)) { memmove(self4->index + i + 1, self4->index + i, self->numChildren - (i + 1)); memmove(self4->children + i + 1, self4->children + i, (self->numChildren - (i + 1)) * sizeof(void *)); self4->index[i] = index; self4->children[i] = nullptr; return self4->children[i]; } } self4->index[self->numChildren - 1] = index; self4->children[self->numChildren - 1] = nullptr; return self4->children[self->numChildren - 1]; } } else if (self->type == Type::Node16) { insert16: auto *self16 = static_cast(self); { int i = getNodeIndex(self16, index); if (i >= 0) { return self16->children[i]; } } if (self->numChildren == 16) { auto *newSelf = new (safe_malloc(sizeof(Node48))) Node48; memcpy((void *)newSelf, self, offsetof(Node, type)); newSelf->nextFree = 16; int i = 0; for (auto x : self16->index) { newSelf->children[i] = self16->children[i]; newSelf->index[x] = i; ++i; } assert(i == 16); free(std::exchange(self, newSelf)); setChildrenParents(self); goto insert48; } else { ++self->numChildren; for (int i = 0; i < int(self->numChildren) - 1; ++i) { if (int(self16->index[i]) > int(index)) { memmove(self16->index + i + 1, self16->index + i, self->numChildren - (i + 1)); memmove(self16->children + i + 1, self16->children + i, (self->numChildren - (i + 1)) * sizeof(void *)); self16->index[i] = index; self16->children[i] = nullptr; return self16->children[i]; } } self16->index[self->numChildren - 1] = index; self16->children[self->numChildren - 1] = nullptr; return self16->children[self->numChildren - 1]; } } else if (self->type == Type::Node48) { insert48: auto *self48 = static_cast(self); int secondIndex = self48->index[index]; if (secondIndex >= 0) { return self48->children[secondIndex]; } if (self->numChildren == 48) { auto *newSelf = new (safe_malloc(sizeof(Node256))) Node256; memcpy((void *)newSelf, self, offsetof(Node, type)); for (int i = 0; i < 256; ++i) { if (self48->index[i] >= 0) { newSelf->children[i] = self48->children[self48->index[i]]; } } free(std::exchange(self, newSelf)); self = newSelf; setChildrenParents(self); goto insert256; } else { ++self->numChildren; assert(self48->nextFree < 48); self48->index[index] = self48->nextFree; self48->children[self48->nextFree] = nullptr; return self48->children[self48->nextFree++]; } } else { insert256: auto *self256 = static_cast(self); if (!self256->children[index]) { ++self->numChildren; } return self256->children[index]; } } // Precondition - an entry for index must exist in the node void eraseChild(Node *self, uint8_t index) { if (self->type == Type::Node4) { auto *self4 = static_cast(self); int nodeIndex = getNodeIndex(self4, index); memmove(self4->index + nodeIndex, self4->index + nodeIndex + 1, sizeof(self4->index[0]) * (self->numChildren - (nodeIndex + 1))); memmove(self4->children + nodeIndex, self4->children + nodeIndex + 1, sizeof(self4->children[0]) * // NOLINT (self->numChildren - (nodeIndex + 1))); } else if (self->type == Type::Node16) { auto *self16 = static_cast(self); int nodeIndex = getNodeIndex(self16, index); memmove(self16->index + nodeIndex, self16->index + nodeIndex + 1, sizeof(self16->index[0]) * (self->numChildren - (nodeIndex + 1))); memmove(self16->children + nodeIndex, self16->children + nodeIndex + 1, sizeof(self16->children[0]) * // NOLINT (self->numChildren - (nodeIndex + 1))); } else if (self->type == Type::Node48) { auto *self48 = static_cast(self); int8_t toRemoveChildrenIndex = std::exchange(self48->index[index], -1); int8_t lastChildrenIndex = --self48->nextFree; assert(toRemoveChildrenIndex >= 0); assert(lastChildrenIndex >= 0); if (toRemoveChildrenIndex != lastChildrenIndex) { self48->children[toRemoveChildrenIndex] = std::exchange(self48->children[lastChildrenIndex], nullptr); self48->index[self48->children[toRemoveChildrenIndex]->parentsIndex] = toRemoveChildrenIndex; } } else { auto *self256 = static_cast(self); self256->children[index] = nullptr; } --self->numChildren; if (self->numChildren == 0 && !self->entryPresent && self->parent != nullptr) { eraseChild(self->parent, self->parentsIndex); } } Node *nextPhysical(Node *node) { int index = -1; for (;;) { auto nextChild = getChildGeq(node, index + 1); if (nextChild >= 0) { return getChildExists(node, nextChild); } index = node->parentsIndex; node = node->parent; if (node == nullptr) { return nullptr; } } } Node *nextLogical(Node *node) { for (node = nextPhysical(node); node != nullptr && !node->entryPresent; node = nextPhysical(node)) ; return node; } std::string printable(std::string_view key) { std::string result; for (uint8_t c : key) { result += "x"; result += "0123456789abcdef"[c / 16]; result += "0123456789abcdef"[c % 16]; } return result; } std::string printable(const Key &key) { return printable(std::string_view((const char *)key.p, key.len)); } std::string_view getSearchPath(Arena &arena, Node *n) { if (n->parent == nullptr) { return {}; } auto result = vector(arena); for (; n->parent != nullptr; n = n->parent) { result.push_back(n->parentsIndex); } std::reverse(result.begin(), result.end()); return std::string_view((const char *)&result[0], result.size()); // NOLINT } void printLogical(std::string &result, Node *node) { Arena arena; for (Node *iter = node; iter != nullptr;) { auto *next = nextLogical(iter); std::string key; for (uint8_t c : getSearchPath(arena, iter)) { key += "x"; key += "0123456789abcdef"[c / 16]; key += "0123456789abcdef"[c % 16]; } if (iter->entry.pointVersion == iter->entry.rangeVersion) { result += key + " -> " + std::to_string(iter->entry.pointVersion) + "\n"; } else { result += key + " -> " + std::to_string(iter->entry.pointVersion) + "\n"; if (next == nullptr || (getSearchPath(arena, next) != (std::string(getSearchPath(arena, iter)) + std::string("\x00", 1)))) { result += key + "x00 -> " + std::to_string(iter->entry.rangeVersion) + "\n"; } } iter = next; } } Node *prevPhysical(Node *node) { assert(node->parent != nullptr); auto prevChild = getChildLeq(node->parent, node->parentsIndex - 1); assert(prevChild < node->parentsIndex); if (prevChild >= 0) { node = getChildExists(node->parent, prevChild); // Move down the right spine for (;;) { auto rightMostChild = getChildLeq(node, 255); if (rightMostChild >= 0) { node = getChildExists(node, rightMostChild); } else { return node; } } } else { return node->parent; } } struct Iterator { Node *n; int cmp; }; std::string_view getSearchPath(Arena &arena, Node *n); Iterator lastLeq(Node *n, const std::span key) { auto remaining = key; for (;;) { Arena arena; assert((std::string(getSearchPath(arena, n)) + std::string((const char *)remaining.data(), remaining.size())) .ends_with(std::string((const char *)key.data(), key.size()))); if (remaining.size() == 0) { // We've found the physical node corresponding to search path `key` if (n->entryPresent) { return {n, 0}; } else { break; } } else { int c = getChildLeq(n, remaining[0]); if (c == remaining[0]) { n = getChildExists(n, c); remaining = remaining.subspan(1, remaining.size() - 1); } else { // The physical node corresponding to search path `key` does not exist. // Let's find the physical node corresponding to the highest search key // (not necessarily present) less than key // Move down the right spine for (;;) { if (c >= 0) { n = getChildExists(n, c); } else { break; } c = getChildLeq(n, 255); } break; } } } // Iterate backwards along existing physical nodes until we find a present // entry for (; !n->entryPresent; n = prevPhysical(n)) { } return {n, -1}; } void insert(Node **self_, std::span key, int64_t writeVersion) { for (;;) { auto &self = *self_; self->maxVersion = writeVersion; if (key.size() == 0) { auto l = lastLeq(self, key); self->entryPresent = true; self->entry.pointVersion = writeVersion; assert(l.n != nullptr); assert(l.n->entryPresent); self->entry.rangeVersion = l.n->entry.rangeVersion; return; } auto &child = getOrCreateChild(self, key.front()); if (!child) { child = newNode(); child->parent = self; child->parentsIndex = key.front(); } self_ = &child; key = key.subspan(1, key.size() - 1); } } std::string printable(std::string_view key); std::string printable(const Key &key); struct __attribute__((visibility("hidden"))) ConflictSet::Impl { void check(const ReadRange *reads, Result *result, int count) const { for (int i = 0; i < count; ++i) { const auto &r = reads[i]; if (r.readVersion < oldestVersion) { result[i] = TooOld; continue; } // TODO support non-point reads assert(r.end.len == 0); auto [l, c] = lastLeq(root, std::span(r.begin.p, r.begin.len)); #if DEBUG_VERBOSE && !defined(NDEBUG) Arena arena; printf("LastLeq for `%s' got `%s'\n", printable(r.begin).c_str(), printable(getSearchPath(arena, l)).c_str()); #endif assert(l != nullptr); assert(l->entryPresent); result[i] = (c == 0 ? l->entry.pointVersion : l->entry.rangeVersion) > r.readVersion ? Conflict : Commit; } } void addWrites(const WriteRange *writes, int count) { for (int i = 0; i < count; ++i) { const auto &w = writes[i]; // TODO support non-point writes assert(w.end.len == 0); insert(&root, std::span(w.begin.p, w.begin.len), w.writeVersion); } } void setOldestVersion(int64_t oldestVersion) { this->oldestVersion = oldestVersion; } explicit Impl(int64_t oldestVersion) : oldestVersion(oldestVersion) { // Insert "" root = newNode(); root->maxVersion = oldestVersion; root->entry.pointVersion = oldestVersion; root->entry.rangeVersion = oldestVersion; root->entryPresent = true; } ~Impl() { Arena arena; auto toFree = vector(arena); toFree.push_back(root); while (toFree.size() > 0) { auto *n = toFree.back(); toFree.pop_back(); // Add all children to toFree for (int child = getChildGeq(n, 0); child >= 0; child = getChildGeq(n, child + 1)) { auto *c = getChildExists(n, child); assert(c != nullptr); toFree.push_back(c); } free(n); } } Node *root; int64_t oldestVersion; }; void ConflictSet::check(const ReadRange *reads, Result *results, int count) const { return impl->check(reads, results, count); } void ConflictSet::addWrites(const WriteRange *writes, int count) { return impl->addWrites(writes, count); } void ConflictSet::setOldestVersion(int64_t oldestVersion) { return impl->setOldestVersion(oldestVersion); } ConflictSet::ConflictSet(int64_t oldestVersion, [[maybe_unused]] uint64_t seed) : impl(new(safe_malloc(sizeof(Impl))) Impl{oldestVersion}) {} ConflictSet::~ConflictSet() { if (impl) { impl->~Impl(); free(impl); } } ConflictSet::ConflictSet(ConflictSet &&other) noexcept : impl(std::exchange(other.impl, nullptr)) {} ConflictSet &ConflictSet::operator=(ConflictSet &&other) noexcept { impl = std::exchange(other.impl, nullptr); return *this; } using ConflictSet_Result = ConflictSet::Result; using ConflictSet_Key = ConflictSet::Key; using ConflictSet_ReadRange = ConflictSet::ReadRange; using ConflictSet_WriteRange = ConflictSet::WriteRange; extern "C" { __attribute__((__visibility__("default"))) void ConflictSet_check(void *cs, const ConflictSet_ReadRange *reads, ConflictSet_Result *results, int count) { ((ConflictSet::Impl *)cs)->check(reads, results, count); } __attribute__((__visibility__("default"))) void ConflictSet_addWrites(void *cs, const ConflictSet_WriteRange *writes, int count) { ((ConflictSet::Impl *)cs)->addWrites(writes, count); } __attribute__((__visibility__("default"))) void ConflictSet_setOldestVersion(void *cs, int64_t oldestVersion) { ((ConflictSet::Impl *)cs)->setOldestVersion(oldestVersion); } __attribute__((__visibility__("default"))) void * ConflictSet_create(int64_t oldestVersion, uint64_t) { return new (safe_malloc(sizeof(ConflictSet::Impl))) ConflictSet::Impl{oldestVersion}; } __attribute__((__visibility__("default"))) void ConflictSet_destroy(void *cs) { using Impl = ConflictSet::Impl; ((Impl *)cs)->~Impl(); free(cs); } } // ==================== END IMPLEMENTATION ==================== // GCOVR_EXCL_START std::string printable(std::string_view key) { std::string result; for (uint8_t c : key) { result += "x"; result += "0123456789abcdef"[c / 16]; result += "0123456789abcdef"[c % 16]; } return result; } std::string printable(const Key &key) { return printable(std::string_view((const char *)key.p, key.len)); } std::string_view getSearchPath(Arena &arena, Node *n) { if (n->parent == nullptr) { return {}; } auto result = vector(arena); for (; n->parent != nullptr; n = n->parent) { result.push_back(n->parentsIndex); } std::reverse(result.begin(), result.end()); return std::string_view((const char *)&result[0], result.size()); // NOLINT } void printLogical(std::string &result, Node *node) { Arena arena; for (Node *iter = node; iter != nullptr;) { auto *next = nextLogical(iter); std::string key; for (uint8_t c : getSearchPath(arena, iter)) { key += "x"; key += "0123456789abcdef"[c / 16]; key += "0123456789abcdef"[c % 16]; } if (iter->entry.pointVersion == iter->entry.rangeVersion) { result += key + " -> " + std::to_string(iter->entry.pointVersion) + "\n"; } else { result += key + " -> " + std::to_string(iter->entry.pointVersion) + "\n"; if (next == nullptr || (getSearchPath(arena, next) != (std::string(getSearchPath(arena, iter)) + std::string("\x00", 1)))) { result += key + "x00 -> " + std::to_string(iter->entry.rangeVersion) + "\n"; } } iter = next; } } void debugPrintDot(FILE *file, Node *node) { struct DebugDotPrinter { explicit DebugDotPrinter(FILE *file) : file(file) {} void print(Node *n) { assert(n != nullptr); if (n->entryPresent) { fprintf(file, " k_%p [label=\"m=%d p=%d r=%d\"];\n", (void *)n, int(n->maxVersion), int(n->entry.pointVersion), int(n->entry.rangeVersion)); } else { fprintf(file, " k_%p [label=\"m=%d\"];\n", (void *)n, int(n->maxVersion)); } for (int child = getChildGeq(n, 0); child >= 0; child = getChildGeq(n, child + 1)) { auto *c = getChildExists(n, child); fprintf(file, " k_%p -> k_%p [label=\"'%02x'\"];\n", (void *)n, (void *)c, child); print(c); } } FILE *file; }; fprintf(file, "digraph ConflictSet {\n"); assert(node != nullptr); DebugDotPrinter printer{file}; printer.print(node); fprintf(file, "}\n"); } void checkParentPointers(Node *node, bool &success) { for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) { auto *child = getChildExists(node, i); if (child->parent != node) { Arena arena; fprintf(stderr, "%s child %d has parent pointer %p. Expected %p\n", printable(getSearchPath(arena, node)).c_str(), i, (void *)child->parent, (void *)node); success = false; } checkParentPointers(child, success); } } int64_t checkMaxVersion(Node *node, bool &success) { int64_t expected = node->entryPresent ? std::max(node->entry.pointVersion, node->entry.rangeVersion) : std::numeric_limits::lowest(); for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) { auto *child = getChildExists(node, i); expected = std::max(expected, checkMaxVersion(child, success)); } if (node->maxVersion != expected) { Arena arena; fprintf(stderr, "%s has max version %d. Expected %d\n", printable(getSearchPath(arena, node)).c_str(), int(node->maxVersion), int(expected)); success = false; } return expected; } bool checkCorrectness(Node *node, ReferenceImpl &refImpl) { bool success = true; checkParentPointers(node, success); std::string logicalMap; std::string referenceLogicalMap; printLogical(logicalMap, node); refImpl.printLogical(referenceLogicalMap); if (logicalMap != referenceLogicalMap) { fprintf(stderr, "Logical map not equal to reference logical map.\n\nActual:\n" "%s\nExpected:\n%s\n", logicalMap.c_str(), referenceLogicalMap.c_str()); success = false; } return success; } namespace std { void __throw_length_error(const char *) { __builtin_unreachable(); } } // namespace std #ifdef ENABLE_MAIN int main(void) { int64_t writeVersion = 0; ConflictSet::Impl cs{writeVersion}; ReferenceImpl refImpl{writeVersion}; Arena arena; constexpr int kNumKeys = 10; auto *write = new (arena) ConflictSet::WriteRange[kNumKeys]; for (int i = 0; i < kNumKeys; ++i) { write[i].begin = toKey(arena, i); write[i].end.len = 0; write[i].writeVersion = ++writeVersion; } cs.addWrites(write, kNumKeys); refImpl.addWrites(write, kNumKeys); debugPrintDot(stdout, cs.root); return 0; } #endif #ifdef ENABLE_FUZZ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { // TODO call setOldestVersion, and check range writes/reads gArbitrary = Arbitrary{{data, size}}; int64_t writeVersion = 0; ConflictSet::Impl cs{writeVersion}; ReferenceImpl refImpl{writeVersion}; while (gArbitrary.hasEntropy()) { Arena arena; { int numWrites = gArbitrary.bounded(10); int64_t v = ++writeVersion; auto *writes = new (arena) ConflictSet::WriteRange[numWrites]; auto keys = set(arena); while (int(keys.size()) < numWrites) { if (!gArbitrary.hasEntropy()) { // Tell the fuzzer it's not interesting return -1; } int keyLen = gArbitrary.bounded(8); auto *begin = new (arena) uint8_t[keyLen]; gArbitrary.randomBytes(begin, keyLen); keys.insert(std::string_view((const char *)begin, keyLen)); } auto iter = keys.begin(); for (int i = 0; i < numWrites; ++i) { writes[i].begin.p = (const uint8_t *)iter->data(); writes[i].begin.len = iter->size(); ++iter; writes[i].end.len = 0; writes[i].writeVersion = v; #if DEBUG_VERBOSE && !defined(NDEBUG) printf("Write: {%s} -> %d\n", printable(writes[i].begin).c_str(), int(writes[i].writeVersion)); #endif } assert(iter == keys.end()); cs.addWrites(writes, numWrites); refImpl.addWrites(writes, numWrites); } bool success = checkCorrectness(cs.root, refImpl); if (!success) { abort(); } { int numReads = gArbitrary.bounded(10); int64_t v = writeVersion - gArbitrary.bounded(10); auto *reads = new (arena) ConflictSet::ReadRange[numReads]; auto keys = set(arena); while (int(keys.size()) < numReads) { if (!gArbitrary.hasEntropy()) { // Tell the fuzzer it's not interesting return -1; } int keyLen = gArbitrary.bounded(8); auto *begin = new (arena) uint8_t[keyLen]; gArbitrary.randomBytes(begin, keyLen); keys.insert(std::string_view((const char *)begin, keyLen)); } auto iter = keys.begin(); for (int i = 0; i < numReads; ++i) { reads[i].begin.p = (const uint8_t *)iter->data(); reads[i].begin.len = iter->size(); ++iter; reads[i].end.len = 0; reads[i].readVersion = v; #if DEBUG_VERBOSE && !defined(NDEBUG) printf("Read: {%s} at %d\n", printable(reads[i].begin).c_str(), int(reads[i].readVersion)); #endif } assert(iter == keys.end()); auto *results1 = new (arena) ConflictSet::Result[numReads]; auto *results2 = new (arena) ConflictSet::Result[numReads]; cs.check(reads, results1, numReads); refImpl.check(reads, results2, numReads); for (int i = 0; i < numReads; ++i) { if (results1[i] != results2[i]) { fprintf(stderr, "Expected %d, got %d for read of %s at version %d\n", results2[i], results1[i], printable(reads[i].begin).c_str(), int(reads[i].readVersion)); std::string referenceLogicalMap; refImpl.printLogical(referenceLogicalMap); fprintf(stderr, "Logical map:\n\n%s\n", referenceLogicalMap.c_str()); abort(); } } } } return 0; } #endif // GCOVR_EXCL_STOP