weaselab/conflict-set
2
0
Fork 0
Code Issues 4 Packages Releases 13 Activity

Partial keys seem to be working

Browse Source
This commit is contained in:
Andrew Noyes
2024-01-31 11:10:41 -08:00
parent 2e6265b784
commit 5a69ec82df
2 changed files with 57 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
ConflictSet.cpp
View File

@@ -37,12 +37,12 @@ struct Node {
Node *parent = nullptr; Node *parent = nullptr;
int64_t maxVersion = std::numeric_limits<int64_t>::lowest(); int64_t maxVersion = std::numeric_limits<int64_t>::lowest();
Entry entry; Entry entry;
constexpr static auto kCompressedKeyMaxLen = 18; constexpr static auto kPartialKeyMaxLen = 18;
int16_t numChildren = 0; int16_t numChildren = 0;
bool entryPresent = false; bool entryPresent = false;
uint8_t parentsIndex = 0; uint8_t parentsIndex = 0;
uint8_t compressedKey[kCompressedKeyMaxLen]; uint8_t partialKey[kPartialKeyMaxLen];
int8_t compressedKeyLen = 0; int8_t partialKeyLen = 0;
/* end section that's copied to the next node */ /* end section that's copied to the next node */
Type type = Type::Invalid; Type type = Type::Invalid;
@@ -728,8 +728,8 @@ struct Iterator {
std::string_view getSearchPath(Arena &arena, Node *n) { std::string_view getSearchPath(Arena &arena, Node *n) {
auto result = vector<char>(arena); auto result = vector<char>(arena);
for (;;) { for (;;) {
for (int i = n->compressedKeyLen - 1; i >= 0; --i) { for (int i = n->partialKeyLen - 1; i >= 0; --i) {
result.push_back(n->compressedKey[i]); result.push_back(n->partialKey[i]);
} }
if (n->parent == nullptr) { if (n->parent == nullptr) {
break; break;
@@ -738,26 +738,29 @@ std::string_view getSearchPath(Arena &arena, Node *n) {
n = n->parent; n = n->parent;
} }
std::reverse(result.begin(), result.end()); std::reverse(result.begin(), result.end());
return std::string_view((const char *)&result[0], result.size()); // NOLINT if (result.size() > 0) {
return std::string_view((const char *)&result[0], result.size()); // NOLINT
} else {
return std::string_view();
}
} }
Iterator lastLeq(Node *n, const std::span<const uint8_t> key) { Iterator lastLeq(Node *n, const std::span<const uint8_t> key) {
auto remaining = key; auto remaining = key;
for (;;) { for (;;) {
Arena arena; Arena arena;
int commonLen = std::min<int>(n->compressedKeyLen, remaining.size()); int commonLen = std::min<int>(n->partialKeyLen, remaining.size());
if (commonLen > Node::kCompressedKeyMaxLen) { if (commonLen > Node::kPartialKeyMaxLen) {
__builtin_unreachable(); __builtin_unreachable();
} }
int c = memcmp(n->compressedKey, remaining.data(), commonLen); int c = memcmp(n->partialKey, remaining.data(), commonLen);
if (c == 0 && commonLen == n->compressedKeyLen) { if (c == 0 && commonLen == n->partialKeyLen) {
// Compressed key matches // partial key matches
remaining = remaining.subspan(commonLen, remaining.size() - commonLen); remaining = remaining.subspan(commonLen, remaining.size() - commonLen);
} else if (c < 0 || } else if (c < 0 || (c == 0 && n->partialKeyLen < int(remaining.size()))) {
(c == 0 && n->compressedKeyLen < int(remaining.size()))) {
// n is the last physical node less than remaining, and there's no eq node // n is the last physical node less than remaining, and there's no eq node
break; break;
} else if (c > 0) { } else if (c > 0 || (c == 0 && n->partialKeyLen > int(remaining.size()))) {
// n is the first physical node greater than remaining, and there's no eq // n is the first physical node greater than remaining, and there's no eq
// node // node
n = prevPhysical(n); n = prevPhysical(n);
@@ -805,39 +808,40 @@ Iterator lastLeq(Node *n, const std::span<const uint8_t> key) {
void insert(Node **self_, std::span<const uint8_t> key, int64_t writeVersion) { void insert(Node **self_, std::span<const uint8_t> key, int64_t writeVersion) {
for (;;) { for (;;) {
auto &self = *self_; auto &self = *self_;
self->maxVersion = std::max(self->maxVersion, writeVersion); // Handle an existing partial key
int commonLen = std::min<int>(self->compressedKeyLen, key.size()); int partialKeyIndex = 0;
// Handle an existing compressed key for (; partialKeyIndex < self->partialKeyLen; ++partialKeyIndex) {
int compressedKeyIndex = 0; if (partialKeyIndex == int(key.size()) ||
for (; compressedKeyIndex < commonLen; ++compressedKeyIndex) { self->partialKey[partialKeyIndex] != key[partialKeyIndex]) {
if (self->compressedKey[compressedKeyIndex] != key[compressedKeyIndex]) {
auto *old = self; auto *old = self;
self = newNode(); self = newNode();
memcpy((void *)self, old, offsetof(Node, type)); self->maxVersion = old->maxVersion;
self->entryPresent = false; self->partialKeyLen = partialKeyIndex;
self->parent = old->parent;
self->parentsIndex = old->parentsIndex;
memcpy(self->partialKey, old->partialKey, partialKeyIndex);
getOrCreateChild(self, old->compressedKey[compressedKeyIndex]) = old; getOrCreateChild(self, old->partialKey[partialKeyIndex]) = old;
old->parent = self; old->parent = self;
old->parentsIndex = old->compressedKey[compressedKeyIndex]; old->parentsIndex = old->partialKey[partialKeyIndex];
self->compressedKeyLen = compressedKeyIndex;
memmove(old->compressedKey, old->compressedKey + compressedKeyIndex + 1, memmove(old->partialKey, old->partialKey + partialKeyIndex + 1,
old->compressedKeyLen - (compressedKeyIndex + 1)); old->partialKeyLen - (partialKeyIndex + 1));
old->compressedKeyLen -= compressedKeyIndex + 1; old->partialKeyLen -= partialKeyIndex + 1;
break; break;
} }
} }
key = key.subspan(compressedKeyIndex, key.size() - compressedKeyIndex); key = key.subspan(partialKeyIndex, key.size() - partialKeyIndex);
// Consider adding a compressed key // Consider adding a partial key
if (self->numChildren == 0 && !self->entryPresent) { if (self->numChildren == 0 && !self->entryPresent) {
self->compressedKeyLen = self->partialKeyLen = std::min<int>(key.size(), self->kPartialKeyMaxLen);
std::min<int>(key.size(), self->kCompressedKeyMaxLen); memcpy(self->partialKey, key.data(), self->partialKeyLen);
memcpy(self->compressedKey, key.data(), self->compressedKeyLen); key = key.subspan(self->partialKeyLen, key.size() - self->partialKeyLen);
key = key.subspan(self->compressedKeyLen,
key.size() - self->compressedKeyLen);
} }
self->maxVersion = std::max(self->maxVersion, writeVersion);
if (key.size() == 0) { if (key.size() == 0) {
auto l = lastLeq(self, key); auto l = lastLeq(self, key);
self->entryPresent = true; self->entryPresent = true;
@@ -1029,15 +1033,14 @@ void printLogical(std::string &result, Node *node) {
void print(Node *n) { void print(Node *n) {
assert(n != nullptr); assert(n != nullptr);
auto compressedKey = auto partialKey = printable(Key{n->partialKey, n->partialKeyLen});
printable(Key{n->compressedKey, n->compressedKeyLen});
if (n->entryPresent) { if (n->entryPresent) {
fprintf(file, " k_%p [label=\"m=%d p=%d r=%d %s\"];\n", (void *)n, fprintf(file, " k_%p [label=\"m=%d p=%d r=%d %s\"];\n", (void *)n,
int(n->maxVersion), int(n->entry.pointVersion), int(n->maxVersion), int(n->entry.pointVersion),
int(n->entry.rangeVersion), compressedKey.c_str()); int(n->entry.rangeVersion), partialKey.c_str());
} else { } else {
fprintf(file, " k_%p [label=\"m=%d %s\"];\n", (void *)n, fprintf(file, " k_%p [label=\"m=%d %s\"];\n", (void *)n,
int(n->maxVersion), compressedKey.c_str()); int(n->maxVersion), partialKey.c_str());
} }
for (int child = getChildGeq(n, 0); child >= 0; for (int child = getChildGeq(n, 0); child >= 0;
child = getChildGeq(n, child + 1)) { child = getChildGeq(n, child + 1)) {
@@ -1057,29 +1060,6 @@ void printLogical(std::string &result, Node *node) {
fprintf(file, "}\n"); fprintf(file, "}\n");
} }
void checkCompressedKey(Node *node, bool &success) {
if (node->numChildren == 1 &&
node->compressedKeyLen < node->kCompressedKeyMaxLen) {
Arena arena;
fprintf(stderr, "%s has 1 child and %d < %d compressed key bytes\n",
printable(getSearchPath(arena, node)).c_str(),
int(node->compressedKeyLen), int(node->kCompressedKeyMaxLen));
success = false;
}
for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) {
auto *child = getChildExists(node, i);
if (child->parent != node) {
Arena arena;
fprintf(stderr, "%s child %d has parent pointer %p. Expected %p\n",
printable(getSearchPath(arena, node)).c_str(), i,
(void *)child->parent, (void *)node);
success = false;
}
checkCompressedKey(child, success);
}
}
void checkParentPointers(Node *node, bool &success) { void checkParentPointers(Node *node, bool &success) {
for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) { for (int i = getChildGeq(node, 0); i >= 0; i = getChildGeq(node, i + 1)) {
auto *child = getChildExists(node, i); auto *child = getChildExists(node, i);
@@ -1117,7 +1097,7 @@ bool checkCorrectness(Node *node, ReferenceImpl &refImpl) {
bool success = true; bool success = true;
checkParentPointers(node, success); checkParentPointers(node, success);
checkCompressedKey(node, success); checkMaxVersion(node, success);
std::string logicalMap; std::string logicalMap;
std::string referenceLogicalMap; std::string referenceLogicalMap;
@@ -1196,10 +1176,13 @@ int main(void) {
#ifdef ENABLE_FUZZ #ifdef ENABLE_FUZZ
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
TestDriver<ConflictSet::Impl> driver{data, size}; TestDriver<ConflictSet::Impl> driver{data, size};
static_assert(driver.kMaxKeyLen > Node::kPartialKeyMaxLen);
do { do {
bool success = checkCorrectness(driver.cs.root, driver.refImpl); bool success = checkCorrectness(driver.cs.root, driver.refImpl);
if (!success) { if (!success) {
debugPrintDot(stdout, driver.cs.root);
fflush(stdout);
abort(); abort();
} }
} while (!driver.next()); } while (!driver.next());

19
Internal.h
View File

@@ -449,6 +449,8 @@ template <class ConflictSetImpl> struct TestDriver {
ConflictSetImpl cs{writeVersion}; ConflictSetImpl cs{writeVersion};
ReferenceImpl refImpl{writeVersion}; ReferenceImpl refImpl{writeVersion};
constexpr static auto kMaxKeyLen = 24;
// Call until it returns true, for "done". Check internal invariants etc // Call until it returns true, for "done". Check internal invariants etc
// between calls to next. // between calls to next.
bool next() { bool next() {
@@ -457,7 +459,7 @@ template <class ConflictSetImpl> struct TestDriver {
} }
Arena arena; Arena arena;
{ {
int numWrites = arbitrary.bounded(10); int numWrites = arbitrary.bounded(kMaxKeyLen);
int64_t v = ++writeVersion; int64_t v = ++writeVersion;
auto *writes = new (arena) ConflictSet::WriteRange[numWrites]; auto *writes = new (arena) ConflictSet::WriteRange[numWrites];
auto keys = set<std::string_view>(arena); auto keys = set<std::string_view>(arena);
@@ -465,7 +467,7 @@ template <class ConflictSetImpl> struct TestDriver {
if (!arbitrary.hasEntropy()) { if (!arbitrary.hasEntropy()) {
return true; return true;
} }
int keyLen = arbitrary.bounded(8); int keyLen = arbitrary.bounded(kMaxKeyLen);
auto *begin = new (arena) uint8_t[keyLen]; auto *begin = new (arena) uint8_t[keyLen];
arbitrary.randomBytes(begin, keyLen); arbitrary.randomBytes(begin, keyLen);
keys.insert(std::string_view((const char *)begin, keyLen)); keys.insert(std::string_view((const char *)begin, keyLen));
@@ -478,8 +480,9 @@ template <class ConflictSetImpl> struct TestDriver {
writes[i].end.len = 0; writes[i].end.len = 0;
writes[i].writeVersion = v; writes[i].writeVersion = v;
#if DEBUG_VERBOSE && !defined(NDEBUG) #if DEBUG_VERBOSE && !defined(NDEBUG)
printf("Write: {%s} -> %d\n", printable(writes[i].begin).c_str(), fprintf(stderr, "Write: {%s} -> %d\n",
int(writes[i].writeVersion)); printable(writes[i].begin).c_str(),
int(writes[i].writeVersion));
#endif #endif
} }
assert(iter == keys.end()); assert(iter == keys.end());
@@ -488,14 +491,14 @@ template <class ConflictSetImpl> struct TestDriver {
} }
{ {
int numReads = arbitrary.bounded(10); int numReads = arbitrary.bounded(10);
int64_t v = writeVersion - arbitrary.bounded(10); int64_t v = std::max<int64_t>(writeVersion - arbitrary.bounded(10), 0);
auto *reads = new (arena) ConflictSet::ReadRange[numReads]; auto *reads = new (arena) ConflictSet::ReadRange[numReads];
auto keys = set<std::string_view>(arena); auto keys = set<std::string_view>(arena);
while (int(keys.size()) < numReads) { while (int(keys.size()) < numReads) {
if (!arbitrary.hasEntropy()) { if (!arbitrary.hasEntropy()) {
return true; return true;
} }
int keyLen = arbitrary.bounded(8); int keyLen = arbitrary.bounded(kMaxKeyLen);
auto *begin = new (arena) uint8_t[keyLen]; auto *begin = new (arena) uint8_t[keyLen];
arbitrary.randomBytes(begin, keyLen); arbitrary.randomBytes(begin, keyLen);
keys.insert(std::string_view((const char *)begin, keyLen)); keys.insert(std::string_view((const char *)begin, keyLen));
@@ -508,8 +511,8 @@ template <class ConflictSetImpl> struct TestDriver {
reads[i].end.len = 0; reads[i].end.len = 0;
reads[i].readVersion = v; reads[i].readVersion = v;
#if DEBUG_VERBOSE && !defined(NDEBUG) #if DEBUG_VERBOSE && !defined(NDEBUG)
printf("Read: {%s} at %d\n", printable(reads[i].begin).c_str(), fprintf(stderr, "Read: {%s} at %d\n", printable(reads[i].begin).c_str(),
int(reads[i].readVersion)); int(reads[i].readVersion));
#endif #endif
} }
assert(iter == keys.end()); assert(iter == keys.end());