diff --git a/VersionedMap.cpp b/VersionedMap.cpp index c7c7551..fc31247 100644 --- a/VersionedMap.cpp +++ b/VersionedMap.cpp @@ -322,33 +322,86 @@ private: uint32_t freeList = 0; }; -auto operator<=>(const VersionedMap::Mutation &lhs, const Node &rhs) { - int cl = std::min(lhs.param1Length, rhs.entry->keyLen); +auto operator<=>(const VersionedMap::Key &lhs, const Node &rhs) { + int cl = std::min(lhs.len, rhs.entry->keyLen); if (cl > 0) { - int c = memcmp(lhs.param1, rhs.entry->getKey(), cl); + int c = memcmp(lhs.p, rhs.entry->getKey(), cl); if (c != 0) { return c <=> 0; } } - return lhs.param1Length <=> rhs.entry->keyLen; + return lhs.len <=> rhs.entry->keyLen; } struct Finger { - void push(uint32_t node) { searchPath[searchPathSize_++] = node; } + void push(uint32_t node, bool dir) { + searchPath[searchPathSize_] = node; + direction[searchPathSize_] = dir; + ++searchPathSize_; + } void pop() { --searchPathSize_; } - uint32_t back() const { + uint32_t backNode() const { assert(searchPathSize_ > 0); return searchPath[searchPathSize_ - 1]; } + bool backDirection() const { + assert(searchPathSize_ > 0); + return direction[searchPathSize_ - 1]; + } uint32_t searchPathSize() const { return searchPathSize_; } + Finger() : searchPathSize_(0) {} + + Finger(const Finger &other) { +#ifndef NDEBUG + memset(searchPath, 0, sizeof(searchPath)); + memset(direction, 0, sizeof(direction)); +#endif + memcpy(searchPath, other.searchPath, + other.searchPathSize_ * sizeof(searchPath[0])); + memcpy(direction, other.direction, + other.searchPathSize_ * sizeof(direction[0])); + searchPathSize_ = other.searchPathSize_; + } + + Finger &operator=(const Finger &other) { +#ifndef NDEBUG + memset(searchPath, 0, sizeof(searchPath)); + memset(direction, 0, sizeof(direction)); +#endif + memcpy(searchPath, other.searchPath, + other.searchPathSize_ * sizeof(searchPath[0])); + memcpy(direction, other.direction, + other.searchPathSize_ * sizeof(direction[0])); + searchPathSize_ = other.searchPathSize_; + return *this; + } + private: uint32_t searchPath[kPathLengthUpperBound]; - int searchPathSize_ = 0; + bool direction[kPathLengthUpperBound]; + int searchPathSize_; }; struct VersionedMap::Impl { + template + void move(Finger &finger, int64_t at, bool direction) { + uint32_t c; + if (finger.backNode() != 0 && + (c = child(finger.backNode(), direction, at)) != 0) { + finger.push(c, direction); + while (auto c = child(finger.backNode(), !direction, at) != 0) { + finger.push(c, !direction); + } + } else { + while (finger.searchPathSize() > 1 && finger.backDirection() == true) { + finger.pop(); + } + finger.pop(); + } + } + template uint32_t child(uint32_t node, bool which, int64_t at) { static_assert(kOrder == std::memory_order_acquire || @@ -436,36 +489,69 @@ struct VersionedMap::Impl { at); } - void insert(const Mutation &m) { + struct Val { + const uint8_t *p; + int len; + }; + + // Infers `val` and `clearTo` if not set + void insert(Key key, std::optional val, std::optional clearTo) { Finger finger; - finger.push(latestRoot); - bool directionStack[kPathLengthUpperBound]; - int directionStackSize = 0; + bool ignored; + finger.push(latestRoot, ignored); + bool inserted; // Initialize finger to the search path of `m` for (;;) { - auto n = finger.back(); + auto n = finger.backNode(); if (n == 0) { + inserted = true; break; } - auto c = m <=> mm.base[n]; + auto c = key <=> mm.base[n]; if (c == 0) { // No duplicates + inserted = false; break; } - finger.push(child(n, c > 0, latestVersion)); - directionStack[directionStackSize++] = c > 0; + finger.push(child(n, c > 0, latestVersion), + c > 0); + } + + // Infer `val` if not set + if (!val.has_value()) { + if (inserted) { + val = {nullptr, -1}; + } else { + auto *entry = mm.base[finger.backNode()].entry; + val = {entry->getVal(), entry->valLen}; + } + } + + // Infer `clearTo` if not set + if (!clearTo.has_value()) { + if (inserted) { + auto copy = finger; + move(copy, latestVersion, true); + if (copy.searchPathSize() == 0) { + clearTo = false; + } else { + clearTo = mm.base[copy.backNode()].entry->clearTo; + } + } else { + clearTo = false; + } } // Prepare new node - uint32_t node = newNode(latestVersion, m.param1, m.param1Length, m.param2, - m.param2Length, false /* TODO set correctly */); - if (finger.back() != 0) { + uint32_t node = + newNode(latestVersion, key.p, key.len, val->p, val->len, *clearTo); + if (!inserted) { auto &n = mm.base[node]; - n.pointer[0] = - child(finger.back(), false, latestVersion); - n.pointer[1] = - child(finger.back(), true, latestVersion); + n.pointer[0] = child(finger.backNode(), false, + latestVersion); + n.pointer[1] = child(finger.backNode(), true, + latestVersion); } // Rotate and propagate up the search path @@ -475,14 +561,15 @@ struct VersionedMap::Impl { latestRoot = node; break; } + const bool direction = finger.backDirection(); finger.pop(); - auto parent = finger.back(); - const bool direction = directionStack[--directionStackSize]; + auto parent = finger.backNode(); parent = update(parent, direction, node, latestVersion); - if (mm.base[node].entry->priority > mm.base[parent].entry->priority) { + if (inserted && + mm.base[node].entry->priority > mm.base[parent].entry->priority) { rotate(parent, latestVersion, !direction); } else { - if (parent == finger.back()) { + if (parent == finger.backNode()) { break; } } @@ -511,6 +598,30 @@ struct VersionedMap::Impl { void printInOrderHelper(int64_t version, uint32_t node); + void addMutations(const Mutation *mutations, int numMutations, + int64_t version) { + assert(latestVersion < version); + latestVersion = version; + latestRoot = roots.roots()[roots.rootCount() - 1]; + // TODO Improve ILP? + for (int i = 0; i < numMutations; ++i) { + const auto &m = mutations[i]; + switch (m.type) { + case Set: { + insert({m.param1, m.param1Len}, {{m.param2, m.param2Len}}, {}); + } break; + case Clear: { + insert({m.param1, m.param1Len}, {{nullptr, -1}}, {}); + // TODO erase (param1, param2) + insert({m.param2, m.param2Len}, {}, true); + } break; + default: // GCOVR_EXCL_LINE + __builtin_unreachable(); // GCOVR_EXCL_LINE + } + } + roots.add(latestRoot, latestVersion); + } + MemManager mm; RootSet roots; // Only meaningful within the callstack of `addMutations` @@ -518,6 +629,31 @@ struct VersionedMap::Impl { int64_t latestVersion = 0; }; +VersionedMap::VersionedMap(int64_t version) + : impl(new(malloc(sizeof(Impl))) Impl()) { + impl->latestVersion = version; +} + +VersionedMap::~VersionedMap() { + if (impl != nullptr) { + impl->~Impl(); + free(impl); + } +} + +VersionedMap::VersionedMap(VersionedMap &&other) noexcept { + impl = std::exchange(other.impl, nullptr); +} +VersionedMap &VersionedMap::operator=(VersionedMap &&other) noexcept { + impl = std::exchange(other.impl, nullptr); + return *this; +} + +void VersionedMap::addMutations(const Mutation *mutations, int numMutations, + int64_t version) { + impl->addMutations(mutations, numMutations, version); +} + // ==================== END IMPLEMENTATION ==================== // GCOVR_EXCL_START @@ -556,20 +692,13 @@ void VersionedMap::Impl::printInOrderHelper(int64_t version, uint32_t node) { int main() { { weaselab::VersionedMap::Impl impl; - impl.latestRoot = impl.roots.roots()[impl.roots.rootCount() - 1]; - impl.latestVersion = 1; - impl.insert(weaselab::VersionedMap::Mutation{ - (const uint8_t *)"c", nullptr, 1, 0, weaselab::VersionedMap::Set}); - impl.insert(weaselab::VersionedMap::Mutation{ - (const uint8_t *)"b", nullptr, 1, 0, weaselab::VersionedMap::Set}); - impl.insert(weaselab::VersionedMap::Mutation{ - (const uint8_t *)"a", nullptr, 1, 0, weaselab::VersionedMap::Set}); - impl.roots.add(impl.latestRoot, impl.latestVersion); - impl.printInOrder(0); + weaselab::VersionedMap::Mutation m[] = { + {(const uint8_t *)"a", nullptr, 1, 0, weaselab::VersionedMap::Set}, + {(const uint8_t *)"b", nullptr, 1, 0, weaselab::VersionedMap::Set}, + {(const uint8_t *)"c", nullptr, 1, 0, weaselab::VersionedMap::Set}, + }; + impl.addMutations(m, sizeof(m) / sizeof(m[0]), 1); impl.printInOrder(1); - impl.printInOrder(2); - impl.printInOrder(3); - impl.setOldestVersion(3); } ankerl::nanobench::Bench bench; @@ -615,30 +744,6 @@ int main() { roots.getThreadSafeHandle().rootForVersion(i - kNumVersions / 2)); }); } - - { - weaselab::VersionedMap::Impl impl; - bench.run("insert fixed entry", [&]() { - ++impl.latestVersion; - impl.latestRoot = impl.roots.roots()[impl.roots.rootCount() - 1]; - impl.insert(weaselab::VersionedMap::Mutation{ - (const uint8_t *)"a", nullptr, 1, 0, weaselab::VersionedMap::Set}); - impl.roots.add(impl.latestRoot, impl.latestVersion); - }); - } - - { - weaselab::VersionedMap::Impl impl; - bench.run("insert fresh entry", [&]() { - ++impl.latestVersion; - impl.latestRoot = impl.roots.roots()[impl.roots.rootCount() - 1]; - auto k = __builtin_bswap64(impl.latestVersion); - impl.insert(weaselab::VersionedMap::Mutation{ - (const uint8_t *)&k, nullptr, sizeof(k), 0, - weaselab::VersionedMap::Set}); - impl.roots.add(impl.latestRoot, impl.latestVersion); - }); - } } #endif diff --git a/include/VersionedMap.h b/include/VersionedMap.h index 80dd768..34aebae 100644 --- a/include/VersionedMap.h +++ b/include/VersionedMap.h @@ -61,8 +61,8 @@ struct VersionedMap { struct Mutation { const uint8_t *param1; const uint8_t *param2; - int param1Length; - int param2Length; + int param1Len; + int param2Len; MutationType type; };