diff --git a/VersionedMap.cpp b/VersionedMap.cpp index 87c36df..bd1b20e 100644 --- a/VersionedMap.cpp +++ b/VersionedMap.cpp @@ -73,15 +73,26 @@ namespace weaselab { constexpr int kPathLengthUpperBound = 96; struct Entry { - int64_t insertVersion; + // If there is a point mutation at key, then pointVersion is its version. + // Otherwise it's negative. + int64_t pointVersion; + // If there is a range mutation ending at key, then rangeVersion is its + // version. Otherwise it's negative. + int64_t rangeVersion; int keyLen; - // Negative if this key is cleared + // Negative if this key is cleared. Only meaningful if this is a point + // mutation. int valLen; mutable int refCount; uint32_t priority; + + // True if the entry is a point mutation. If false, this entry's key should be + // read through to the underlying data structure. + bool pointMutation() const { return pointVersion >= 0; } + // True if mutations in (pred, this) are cleared. If false, (pred, this) // should be read through to the underlying data structure. - bool clearTo; + bool clearTo() const { return rangeVersion >= 0; } // There's an extra zero byte past the end of getKey, used for // reconstructing logical mutations without copies. @@ -102,15 +113,16 @@ struct Entry { } } - static Entry *make(int64_t insertVersion, const uint8_t *key, int keyLen, - const uint8_t *val, int valLen, bool clearTo) { + static Entry *make(int64_t pointVersion, int64_t rangeVersion, + const uint8_t *key, int keyLen, const uint8_t *val, + int valLen) { auto e = (Entry *)malloc(sizeof(Entry) + keyLen + 1 + std::max(valLen, 0)); - e->insertVersion = insertVersion; + e->pointVersion = pointVersion; + e->rangeVersion = rangeVersion; e->keyLen = keyLen; e->valLen = valLen; e->refCount = 1; e->priority = XXH3_64bits(key, keyLen); - e->clearTo = clearTo; if (keyLen > 0) { memcpy((uint8_t *)e->getKey(), key, keyLen); } @@ -525,8 +537,9 @@ struct VersionedMap::Impl { return finger; } - // Infers `val` and `clearTo` if not set - void insert(Key key, std::optional val, std::optional clearTo) { + // If `val` is set, then this is a point mutation at `latestVersion`. + // Otherwise it's the end of a range mutation at `latestVersion`. + void insert(Key key, std::optional val) { Finger finger; bool ignored; finger.push(latestRoot, ignored); @@ -549,34 +562,36 @@ struct VersionedMap::Impl { c > 0); } - // Infer `val` if not set - if (!val.has_value()) { - if (inserted) { - val = {nullptr, -1}; - } else { - auto *entry = mm.base[finger.backNode()].entry; - val = {entry->getVal(), entry->valLen}; - } - } - - // Infer `clearTo` if not set - if (!clearTo.has_value()) { + int64_t pointVersion, rangeVersion; + if (val.has_value()) { + pointVersion = latestVersion; if (inserted) { auto copy = finger; move(copy, latestVersion, true); if (copy.searchPathSize() == 0) { - clearTo = false; + rangeVersion = -1; } else { - clearTo = mm.base[copy.backNode()].entry->clearTo; + rangeVersion = mm.base[copy.backNode()].entry->rangeVersion; } } else { - clearTo = false; + auto *entry = mm.base[finger.backNode()].entry; + rangeVersion = entry->rangeVersion; + } + } else { + rangeVersion = latestVersion; + if (inserted) { + val = {nullptr, -1}; + pointVersion = -1; + } else { + auto *entry = mm.base[finger.backNode()].entry; + val = {entry->getVal(), entry->valLen}; + pointVersion = entry->pointVersion; } } // Prepare new node uint32_t node = - newNode(latestVersion, key.p, key.len, val->p, val->len, *clearTo); + newNode(pointVersion, rangeVersion, key.p, key.len, val->p, val->len); if (!inserted) { auto &n = mm.base[node]; n.pointer[0] = child(finger.backNode(), false, @@ -662,24 +677,30 @@ struct VersionedMap::Impl { finger.setSearchPathSizeUnsafe(oldSize); if (greaterThan) { - while (auto c = child(finger.backNode(), false, - latestVersion) != 0) { + uint32_t c; + while ((c = child(finger.backNode(), false, + latestVersion)) != 0) { finger.push(c, false); } } else { move(finger, latestVersion, true); } + + if (finger.backNode() == 0) { + finger.pop(); + assert(finger.searchPathSize() == 0); + } } - uint32_t newNode(int64_t version, const uint8_t *key, int keyLen, - const uint8_t *val, int valLen, bool clearTo) { + uint32_t newNode(int64_t version, int64_t rangeVersion, const uint8_t *key, + int keyLen, const uint8_t *val, int valLen) { auto result = mm.allocate(); auto &node = mm.base[result]; node.updateVersion = version; node.pointer[0] = 0; node.pointer[1] = 0; node.updated.store(false, std::memory_order_relaxed); - node.entry = Entry::make(version, key, keyLen, val, valLen, clearTo); + node.entry = Entry::make(version, rangeVersion, key, keyLen, val, valLen); return result; } @@ -694,6 +715,7 @@ struct VersionedMap::Impl { void addMutations(const Mutation *mutations, int numMutations, int64_t version) { + // TODO scan to remove mutations older than oldestVersion assert(latestVersion < version); latestVersion = version; latestRoot = roots.roots()[roots.rootCount() - 1]; @@ -702,18 +724,18 @@ struct VersionedMap::Impl { const auto &m = mutations[i]; switch (m.type) { case Set: { - insert({m.param1, m.param1Len}, {{m.param2, m.param2Len}}, {}); + insert({m.param1, m.param1Len}, {{m.param2, m.param2Len}}); } break; case Clear: { - insert({m.param1, m.param1Len}, {{nullptr, -1}}, {}); + insert({m.param1, m.param1Len}, {{nullptr, -1}}); if (m.param2Len > 0) { auto iter = search({m.param1, m.param1Len}, latestVersion); move(iter, latestVersion, true); - while (iter.backNode() != 0 && + while (iter.searchPathSize() > 0 && mm.base[iter.backNode()] < Key{m.param2, m.param2Len}) { remove(iter); } - insert({m.param2, m.param2Len}, {}, true); + insert({m.param2, m.param2Len}, {}); } } break; default: // GCOVR_EXCL_LINE @@ -760,6 +782,8 @@ struct VersionedMap::Iterator::Impl { int64_t version; VersionedMap::Impl *map; int cmp; + // True if this is a point mutation and a range mutation, and we're + // materializing the range mutation instead of the point mutation. bool materializeClearEndingHere = false; }; @@ -801,7 +825,7 @@ VersionedMap::Mutation VersionedMap::Iterator::operator*() const { assert(impl->finger.searchPathSize() != 0); const auto &entry = *impl->map->mm.base[impl->finger.backNode()].entry; if (impl->materializeClearEndingHere) { - assert(entry.clearTo); + assert(entry.pointMutation() && entry.clearTo()); auto prev = *this; --prev; const auto &prevEntry = @@ -819,7 +843,7 @@ VersionedMap::Mutation VersionedMap::Iterator::operator*() const { VersionedMap::Iterator &VersionedMap::Iterator::operator++() { const auto &entry = *impl->map->mm.base[impl->finger.backNode()].entry; if (impl->materializeClearEndingHere) { - assert(entry.clearTo); + assert(entry.pointMutation() && entry.clearTo()); impl->materializeClearEndingHere = false; return *this; } @@ -827,7 +851,8 @@ VersionedMap::Iterator &VersionedMap::Iterator::operator++() { impl->map->move(impl->finger, impl->version, true); impl->materializeClearEndingHere = impl->finger.searchPathSize() > 0 && - impl->map->mm.base[impl->finger.backNode()].entry->clearTo; + impl->map->mm.base[impl->finger.backNode()].entry->pointMutation() && + impl->map->mm.base[impl->finger.backNode()].entry->clearTo(); return *this; } @@ -840,7 +865,8 @@ VersionedMap::Iterator VersionedMap::Iterator::operator++(int) { VersionedMap::Iterator &VersionedMap::Iterator::operator--() { const auto &entry = *impl->map->mm.base[impl->finger.backNode()].entry; - if (entry.clearTo && !impl->materializeClearEndingHere) { + if (entry.pointMutation() && entry.clearTo() && + !impl->materializeClearEndingHere) { impl->materializeClearEndingHere = true; return *this; } @@ -935,7 +961,7 @@ void VersionedMap::Impl::printInOrderHelper(int64_t version, uint32_t node, } else { printf(" "); } - if (mm.base[node].entry->clearTo) { + if (mm.base[node].entry->clearTo()) { printf(" "); } printf("\n"); @@ -1016,50 +1042,6 @@ int main() { } } return 0; - - ankerl::nanobench::Bench bench; - bench.minEpochIterations(10000); - - weaselab::MemManager mm; - bench.run("allocate", [&]() { - auto x = mm.allocate(); - mm.base[x].pointer[0] = 0; - mm.base[x].pointer[1] = 0; - mm.base[x].updated.store(false, std::memory_order_relaxed); - }); - mm.gc(nullptr, 0, 0); - for (int i = 0; i < 10000; ++i) { - auto x = mm.allocate(); - mm.base[x].pointer[0] = 0; - mm.base[x].pointer[1] = 0; - mm.base[x].updated.store(false, std::memory_order_relaxed); - } - auto root = mm.allocate(); - mm.base[root].entry = weaselab::Entry::make(0, nullptr, 0, nullptr, 0, - weaselab::VersionedMap::Set); - mm.base[root].pointer[0] = 0; - mm.base[root].pointer[1] = 0; - mm.base[root].updated.store(false, std::memory_order_relaxed); - bench.run("gc", [&]() { mm.gc(&root, 1, 0); }); - - { - int i = 0; - constexpr int kNumVersions = 1000; - RootSet roots; - for (; i < kNumVersions; i += 2) { - roots.add(i, i); - roots.add(i, i + 1); - } - bench.run("roots - setOldestVersion", [&]() { - roots.add(i, i); - roots.setOldestVersion(i - kNumVersions); - ++i; - }); - bench.run("roots - rootForVersion", [&]() { - bench.doNotOptimizeAway( - roots.getThreadSafeHandle().rootForVersion(i - kNumVersions / 2)); - }); - } } #endif