diff --git a/VersionedMap.cpp b/VersionedMap.cpp index c5a84c9..c28e437 100644 --- a/VersionedMap.cpp +++ b/VersionedMap.cpp @@ -102,8 +102,8 @@ struct Node { uint32_t nextFree; }; Entry *entry; - uint32_t pointers[3]; - bool replacePointer; + uint32_t pointer[3]; + bool replacedPointer; std::atomic updated; }; @@ -224,21 +224,21 @@ struct MemManager { uint32_t p = stack[--stackIndex]; auto &node = base[p]; if (node.updated.load(std::memory_order_relaxed)) { - if (node.pointers[!node.replacePointer] != 0) { - tryPush(node.pointers[!node.replacePointer]); + if (node.pointer[!node.replacedPointer] != 0) { + tryPush(node.pointer[!node.replacedPointer]); } if (oldestVersion < node.updateVersion) { - if (node.pointers[node.replacePointer] != 0) { - tryPush(node.pointers[node.replacePointer]); + if (node.pointer[node.replacedPointer] != 0) { + tryPush(node.pointer[node.replacedPointer]); } } - tryPush(node.pointers[2]); + tryPush(node.pointer[2]); } else { - if (node.pointers[0] != 0) { - tryPush(node.pointers[0]); + if (node.pointer[0] != 0) { + tryPush(node.pointer[0]); } - if (node.pointers[1] != 0) { - tryPush(node.pointers[1]); + if (node.pointer[1] != 0) { + tryPush(node.pointer[1]); } } } @@ -375,6 +375,94 @@ private: struct VersionedMap::Impl { + template + uint32_t child(uint32_t node, bool which, int64_t at) { + auto &n = mm.base[node]; + if (n.updated.load(kOrder) && n.updateVersion <= at && + which == n.replacedPointer) { + return n.pointer[2]; + } else { + return n.pointer[which]; + } + } + + template + uint32_t left(uint32_t node, bool which, int64_t at) { + return child(node, false, at); + } + + template + uint32_t right(uint32_t node, bool which, int64_t at) { + return child(node, true, at); + } + + // Returns the node that results from setting `which` to `child` on `node` + uint32_t update(uint32_t node, int64_t version, bool which, uint32_t child) { + if (this->child(node, which, version) == child) { + return node; + } + auto &n = mm.base[node]; + const bool updated = n.updated.load(std::memory_order_relaxed); + + auto doCopy = [&]() { + uint32_t copy = mm.allocate(); + auto &c = mm.base[copy]; + n.entry->addref(); + c.entry = n.entry; + c.pointer[which] = child; + c.pointer[!which] = n.pointer[!which]; + c.updated.store(false, std::memory_order_relaxed); + c.updateVersion = version; + return copy; + }; + + if (n.updateVersion == version) { + if (updated && n.replacedPointer != which) { + // We can't update n.replacedPointer without introducing a data race + // (unless we packed it into the atomic?) so we copy. pointer[2] becomes + // unreachable, but need to tell the garbage collector. + n.pointer[2] = 0; + return doCopy(); + } else if (updated) { + n.pointer[2] = child; + } else { + n.pointer[which] = child; + } + return node; + } + + if (updated) { + // We already used this node's in-place update + return doCopy(); + } else { + n.updateVersion = version; + n.pointer[2] = child; + n.replacedPointer = which; + n.updated.store(true, std::memory_order_release); // Must be last + return node; + } + } + + void rotate(uint32_t &n, int64_t at, bool right) { + auto l = child(n, !right, at); + n = update( + l, right, + update(n, !right, child(l, right, at), at), + at); + } + + uint32_t newNode(int64_t version, const uint8_t *key, int keyLen, + const uint8_t *val, int valLen, bool clearTo) { + auto result = mm.allocate(); + auto &node = mm.base[result]; + node.updateVersion = version; + node.pointer[0] = 0; + node.pointer[1] = 0; + node.updated.store(false, std::memory_order_relaxed); + node.entry = Entry::make(version, key, keyLen, val, valLen, clearTo); + return result; + } + MemManager mm; RootSet roots; }; @@ -389,22 +477,22 @@ int main() { weaselab::MemManager mm; bench.run("allocate", [&]() { auto x = mm.allocate(); - mm.base[x].pointers[0] = 0; - mm.base[x].pointers[1] = 0; + mm.base[x].pointer[0] = 0; + mm.base[x].pointer[1] = 0; mm.base[x].updated.store(false, std::memory_order_relaxed); }); mm.gc(nullptr, 0, 0); for (int i = 0; i < 10000; ++i) { auto x = mm.allocate(); - mm.base[x].pointers[0] = 0; - mm.base[x].pointers[1] = 0; + mm.base[x].pointer[0] = 0; + mm.base[x].pointer[1] = 0; mm.base[x].updated.store(false, std::memory_order_relaxed); } auto root = mm.allocate(); mm.base[root].entry = weaselab::Entry::make(0, nullptr, 0, nullptr, 0, weaselab::VersionedMap::Set); - mm.base[root].pointers[0] = 0; - mm.base[root].pointers[1] = 0; + mm.base[root].pointer[0] = 0; + mm.base[root].pointer[1] = 0; mm.base[root].updated.store(false, std::memory_order_relaxed); bench.run("gc", [&]() { mm.gc(&root, 1, 0); });