From 5b988efe6f9bad72c5750d476eeea1472a2c5ca7 Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Fri, 15 Mar 2024 16:12:44 -0700 Subject: [PATCH] Consolidate copyChildrenAndKeyFrom implementations --- ConflictSet.cpp | 262 +++++++++++++++++++++++++++++------------------- 1 file changed, 157 insertions(+), 105 deletions(-) diff --git a/ConflictSet.cpp b/ConflictSet.cpp index 2697577..b3405a8 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -81,7 +81,7 @@ struct BitSet { int firstSetGeq(int i) const; // Calls `f` with the index of each bit set in [begin, end) - template void forEachInRange(F f, int begin, int end) { + template void forEachInRange(F f, int begin, int end) const { // See section 3.1 in https://arxiv.org/pdf/1709.07821.pdf for details about // this approach @@ -240,6 +240,9 @@ struct Child { struct Node0 : Node { constexpr static auto kType = Type_Node0; uint8_t *partialKey() { return (uint8_t *)(this + 1); } + + void copyChildrenAndKeyFrom(const Node0 &other); + void copyChildrenAndKeyFrom(const struct Node3 &other); }; struct Node3 : Node { @@ -250,16 +253,11 @@ struct Node3 : Node { Child children[kMaxNodes]; uint8_t *partialKey() { return (uint8_t *)(this + 1); } + void copyChildrenAndKeyFrom(const Node0 &other); void copyChildrenAndKeyFrom(const Node3 &other); + void copyChildrenAndKeyFrom(const struct Node16 &other); }; -inline void Node3::copyChildrenAndKeyFrom(const Node3 &other) { - assert(numChildren == other.numChildren); - assert(partialKeyLen == other.partialKeyLen); - memcpy(index, other.index, - sizeof(*this) - offsetof(Node3, index) + partialKeyLen); -} - struct Node16 : Node { constexpr static auto kType = Type_Node16; constexpr static auto kMaxNodes = 16; @@ -268,9 +266,76 @@ struct Node16 : Node { Child children[kMaxNodes]; uint8_t *partialKey() { return (uint8_t *)(this + 1); } + void copyChildrenAndKeyFrom(const Node3 &other); void copyChildrenAndKeyFrom(const Node16 &other); + void copyChildrenAndKeyFrom(const struct Node48 &other); }; +struct Node48 : Node { + constexpr static auto kType = Type_Node48; + constexpr static auto kMaxNodes = 48; + BitSet bitSet; + int8_t nextFree; + int8_t index[256]; + Child children[kMaxNodes]; + + uint8_t *partialKey() { return (uint8_t *)(this + 1); } + + void copyChildrenAndKeyFrom(const Node16 &other); + void copyChildrenAndKeyFrom(const Node48 &other); + void copyChildrenAndKeyFrom(const struct Node256 &other); +}; + +struct Node256 : Node { + constexpr static auto kType = Type_Node256; + BitSet bitSet; + Child children[256]; + uint8_t *partialKey() { return (uint8_t *)(this + 1); } + void copyChildrenAndKeyFrom(const Node48 &other); + void copyChildrenAndKeyFrom(const Node256 &other); +}; + +inline void Node0::copyChildrenAndKeyFrom(const Node0 &other) { + assert(numChildren == other.numChildren); + assert(partialKeyLen == other.partialKeyLen); + memcpy(partialKey(), &other + 1, partialKeyLen); +} + +inline void Node0::copyChildrenAndKeyFrom(const Node3 &other) { + assert(numChildren == other.numChildren); + assert(partialKeyLen == other.partialKeyLen); + memcpy(partialKey(), &other + 1, partialKeyLen); +} + +inline void Node3::copyChildrenAndKeyFrom(const Node0 &other) { + assert(numChildren == other.numChildren); + assert(partialKeyLen == other.partialKeyLen); + memcpy(partialKey(), &other + 1, partialKeyLen); +} + +inline void Node3::copyChildrenAndKeyFrom(const Node3 &other) { + assert(numChildren == other.numChildren); + assert(partialKeyLen == other.partialKeyLen); + memcpy(index, other.index, + sizeof(*this) - offsetof(Node3, index) + partialKeyLen); +} + +inline void Node3::copyChildrenAndKeyFrom(const Node16 &other) { + assert(numChildren == other.numChildren); + assert(partialKeyLen == other.partialKeyLen); + memcpy(index, other.index, kMaxNodes); + memcpy(children, other.children, kMaxNodes * sizeof(Child)); + memcpy(partialKey(), &other + 1, partialKeyLen); +} + +inline void Node16::copyChildrenAndKeyFrom(const Node3 &other) { + assert(numChildren == other.numChildren); + assert(partialKeyLen == other.partialKeyLen); + memcpy(index, other.index, Node3::kMaxNodes); + memcpy(children, other.children, Node3::kMaxNodes * sizeof(Child)); + memcpy(partialKey(), &other + 1, partialKeyLen); +} + inline void Node16::copyChildrenAndKeyFrom(const Node16 &other) { assert(numChildren == other.numChildren); assert(partialKeyLen == other.partialKeyLen); @@ -281,18 +346,24 @@ inline void Node16::copyChildrenAndKeyFrom(const Node16 &other) { memcpy(partialKey(), &other + 1, partialKeyLen); } -struct Node48 : Node { - constexpr static auto kType = Type_Node48; - constexpr static auto kMaxNodes = 48; - BitSet bitSet; - Child children[kMaxNodes]; - int8_t nextFree; - int8_t index[256]; - uint8_t *partialKey() { return (uint8_t *)(this + 1); } - - void copyChildrenAndKeyFrom(const Node16 &other); - void copyChildrenAndKeyFrom(const struct Node256 &other); -}; +inline void Node16::copyChildrenAndKeyFrom(const Node48 &other) { + assert(numChildren == other.numChildren); + assert(partialKeyLen == other.partialKeyLen); + int i = 0; + other.bitSet.forEachInRange( + [&](int c) { + // Suppress a false positive -Waggressive-loop-optimizations warning + // in gcc. `assume` doesn't work for some reason. + if (!(i < Node16::kMaxNodes)) { + __builtin_unreachable(); // GCOVR_EXCL_LINE + } + index[i] = c; + children[i] = other.children[other.index[c]]; + ++i; + }, + 0, 256); + memcpy(partialKey(), &other + 1, partialKeyLen); +} inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) { assert(numChildren == other.numChildren); @@ -310,16 +381,19 @@ inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) { } } -struct Node256 : Node { - constexpr static auto kType = Type_Node256; - BitSet bitSet; - Child children[256]; - uint8_t *partialKey() { return (uint8_t *)(this + 1); } -}; +inline void Node48::copyChildrenAndKeyFrom(const Node48 &other) { + assert(numChildren == other.numChildren); + assert(partialKeyLen == other.partialKeyLen); + memcpy(&bitSet, &other.bitSet, + offsetof(Node48, children) - offsetof(Node48, bitSet)); + for (int i = 0; i < numChildren; ++i) { + children[i] = other.children[i]; + } + memcpy(partialKey(), &other + 1, partialKeyLen); +} inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) { - memcpy(partialKey(), &other + 1, partialKeyLen); - + memset(index, -1, sizeof(index)); nextFree = other.numChildren; bitSet = other.bitSet; int i = 0; @@ -335,6 +409,25 @@ inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) { ++i; }, 0, 256); + memcpy(partialKey(), &other + 1, partialKeyLen); +} + +inline void Node256::copyChildrenAndKeyFrom(const Node48 &other) { + for (int i = 0; i < 256; ++i) { + children[i].child = nullptr; + } + + bitSet = other.bitSet; + bitSet.forEachInRange( + [&](int c) { children[c] = other.children[other.index[c]]; }, 0, 256); + memcpy(partialKey(), &other + 1, partialKeyLen); +} + +inline void Node256::copyChildrenAndKeyFrom(const Node256 &other) { + bitSet = other.bitSet; + bitSet.forEachInRange([&](int c) { children[c] = other.children[c]; }, 0, + 256); + memcpy(partialKey(), &other + 1, partialKeyLen); } namespace { @@ -736,7 +829,7 @@ Node *&getOrCreateChild(Node *&self, uint8_t index, auto *newSelf = allocators->node3.allocate(self->partialKeyLen); memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin, kNodeCopySize); - memcpy(newSelf->partialKey(), self0->partialKey(), self->partialKeyLen); + newSelf->copyChildrenAndKeyFrom(*self0); allocators->node0.release(self0); self = newSelf; @@ -748,12 +841,7 @@ Node *&getOrCreateChild(Node *&self, uint8_t index, auto *newSelf = allocators->node16.allocate(self->partialKeyLen); memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin, kNodeCopySize); - memcpy(newSelf->partialKey(), self3->partialKey(), self->partialKeyLen); - // TODO replace with memcpy? - for (int i = 0; i < Node3::kMaxNodes; ++i) { - newSelf->index[i] = self3->index[i]; - newSelf->children[i] = self3->children[i]; - } + newSelf->copyChildrenAndKeyFrom(*self3); allocators->node3.release(self3); setChildrenParents(newSelf); self = newSelf; @@ -816,18 +904,9 @@ Node *&getOrCreateChild(Node *&self, uint8_t index, if (self->numChildren == 48) { auto *self48 = static_cast(self); auto *newSelf = allocators->node256.allocate(self->partialKeyLen); - for (int i = 0; i < 256; ++i) { - newSelf->children[i].child = nullptr; - } memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin, kNodeCopySize); - memcpy(newSelf->partialKey(), self48->partialKey(), self->partialKeyLen); - newSelf->bitSet = self48->bitSet; - newSelf->bitSet.forEachInRange( - [&](int i) { - newSelf->children[i] = self48->children[self48->index[i]]; - }, - 0, 256); + newSelf->copyChildrenAndKeyFrom(*self48); allocators->node48.release(self48); setChildrenParents(newSelf); self = newSelf; @@ -891,7 +970,7 @@ void freeAndMakeCapacityAtLeast(Node *&self, int capacity, auto *newSelf = allocators->node0.allocate(capacity); memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin, kNodeCopySize); - memcpy(newSelf->partialKey(), self0->partialKey(), self->partialKeyLen); + newSelf->copyChildrenAndKeyFrom(*self0); getInTree(self, impl) = newSelf; if constexpr (kUseFreeList) { allocators->node0.release(self0); @@ -936,21 +1015,9 @@ void freeAndMakeCapacityAtLeast(Node *&self, int capacity, case Type_Node48: { auto *self48 = (Node48 *)self; auto *newSelf = allocators->node48.allocate(capacity); - memset(newSelf->index, -1, sizeof(newSelf->index)); memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin, kNodeCopySize); - memcpy(newSelf->partialKey(), self48->partialKey(), self->partialKeyLen); - newSelf->bitSet = self48->bitSet; - // TODO check codegen here? - newSelf->nextFree = 0; - newSelf->bitSet.forEachInRange( - [&](int c) { - int index = newSelf->nextFree; - newSelf->index[c] = index; - newSelf->children[index] = self48->children[self48->index[c]]; - ++newSelf->nextFree; - }, - 0, 256); + newSelf->copyChildrenAndKeyFrom(*self48); getInTree(self, impl) = newSelf; setChildrenParents(newSelf); if constexpr (kUseFreeList) { @@ -964,15 +1031,9 @@ void freeAndMakeCapacityAtLeast(Node *&self, int capacity, case Type_Node256: { auto *self256 = (Node256 *)self; auto *newSelf = allocators->node256.allocate(capacity); - for (int i = 0; i < 256; ++i) { - newSelf->children[i].child = nullptr; - } memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin, kNodeCopySize); - memcpy(newSelf->partialKey(), self256->partialKey(), self->partialKeyLen); - newSelf->bitSet = self256->bitSet; - newSelf->bitSet.forEachInRange( - [&](int c) { newSelf->children[c] = self256->children[c]; }, 0, 256); + newSelf->copyChildrenAndKeyFrom(*self256); getInTree(self, impl) = newSelf; setChildrenParents(newSelf); if constexpr (kUseFreeList) { @@ -1019,50 +1080,47 @@ void maybeDownsize(Node *self, NodeAllocators *allocators, auto *newSelf = allocators->node0.allocate(self->partialKeyLen); memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin, kNodeCopySize); - memcpy(newSelf->partialKey(), self3->partialKey(), self->partialKeyLen); - + newSelf->copyChildrenAndKeyFrom(*self3); getInTree(self, impl) = newSelf; allocators->node3.release(self3); - } else if (self->numChildren == 1) { - if (!self->entryPresent) { - auto *child = self3->children[0].child; - int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen; + } else if (self->numChildren == 1 && !self->entryPresent) { + auto *child = self3->children[0].child; + int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen; - if (minCapacity > child->getCapacity()) { - const bool update = child == dontInvalidate; - freeAndMakeCapacityAtLeast(child, minCapacity, - allocators, impl); - if (update) { - dontInvalidate = child; - } + if (minCapacity > child->getCapacity()) { + const bool update = child == dontInvalidate; + freeAndMakeCapacityAtLeast(child, minCapacity, + allocators, impl); + if (update) { + dontInvalidate = child; } + } - // Merge partial key with child + // Merge partial key with child #if DEBUG_VERBOSE && !defined(NDEBUG) - fprintf(stderr, "Merge %s into %s\n", - getSearchPathPrintable(self).c_str(), - getSearchPathPrintable(child).c_str()); + fprintf(stderr, "Merge %s into %s\n", + getSearchPathPrintable(self).c_str(), + getSearchPathPrintable(child).c_str()); #endif - int64_t childMaxVersion = maxVersion(child, impl); + int64_t childMaxVersion = maxVersion(child, impl); - // Construct new partial key for child - memmove(child->partialKey() + self3->partialKeyLen + 1, - child->partialKey(), child->partialKeyLen); - memcpy(child->partialKey(), self3->partialKey(), self->partialKeyLen); - child->partialKey()[self3->partialKeyLen] = self3->index[0]; - child->partialKeyLen += 1 + self3->partialKeyLen; + // Construct new partial key for child + memmove(child->partialKey() + self3->partialKeyLen + 1, + child->partialKey(), child->partialKeyLen); + memcpy(child->partialKey(), self3->partialKey(), self->partialKeyLen); + child->partialKey()[self3->partialKeyLen] = self3->index[0]; + child->partialKeyLen += 1 + self3->partialKeyLen; - child->parent = self->parent; - child->parentsIndex = self->parentsIndex; + child->parent = self->parent; + child->parentsIndex = self->parentsIndex; - // Max versions are stored in the parent, so we need to update it now - // that we have a new parent. - maxVersion(child, impl) = childMaxVersion; + // Max versions are stored in the parent, so we need to update it now + // that we have a new parent. + maxVersion(child, impl) = childMaxVersion; - getInTree(self, impl) = child; - allocators->node3.release(self3); - } + getInTree(self, impl) = child; + allocators->node3.release(self3); } } break; case Type_Node16: @@ -1071,12 +1129,7 @@ void maybeDownsize(Node *self, NodeAllocators *allocators, auto *newSelf = allocators->node3.allocate(self->partialKeyLen); memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin, kNodeCopySize); - memcpy(newSelf->partialKey(), self16->partialKey(), self->partialKeyLen); - static_assert(Node3::kMaxNodes == kMinChildrenNode16 - 1); - for (int i = 0; i < Node3::kMaxNodes; ++i) { - newSelf->index[i] = self16->index[i]; - newSelf->children[i] = self16->children[i]; - } + newSelf->copyChildrenAndKeyFrom(*self16); setChildrenParents(newSelf); getInTree(self, impl) = newSelf; allocators->node16.release(self16); @@ -1114,7 +1167,6 @@ void maybeDownsize(Node *self, NodeAllocators *allocators, if (self->numChildren + int(self->entryPresent) < kMinChildrenNode256) { auto *self256 = (Node256 *)self; auto *newSelf = allocators->node48.allocate(self->partialKeyLen); - memset(newSelf->index, -1, sizeof(newSelf->index)); memcpy((char *)newSelf + kNodeCopyBegin, (char *)self + kNodeCopyBegin, kNodeCopySize); newSelf->copyChildrenAndKeyFrom(*self256);