diff --git a/ConflictSet.cpp b/ConflictSet.cpp index 7aa3e40..491ce01 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -308,6 +308,8 @@ struct Node256 : Node { constexpr static auto kType = Type_Node256; BitSet bitSet; Child children[256]; + // Max of each "page" of 16 children + int64_t maxOfMax[16]; uint8_t *partialKey() { return (uint8_t *)(this + 1); } void copyChildrenAndKeyFrom(const Node48 &other); void copyChildrenAndKeyFrom(const Node256 &other); @@ -457,13 +459,16 @@ inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) { inline void Node256::copyChildrenAndKeyFrom(const Node48 &other) { memcpy((char *)this + kNodeCopyBegin, (char *)&other + kNodeCopyBegin, kNodeCopySize); - memset(children, 0, sizeof(children)); bitSet = other.bitSet; + memset(children, 0, sizeof(children)); + memset(maxOfMax, 0, sizeof(maxOfMax)); bitSet.forEachInRange( [&](int c) { children[c] = other.children[other.index[c]]; assert(children[c].child->parent == &other); children[c].child->parent = this; + maxOfMax[c >> 4] = + std::max(maxOfMax[c >> 4], children[c].childMaxVersion); }, 0, 256); memcpy(partialKey(), &other + 1, partialKeyLen); @@ -481,6 +486,7 @@ inline void Node256::copyChildrenAndKeyFrom(const Node256 &other) { children[c].child->parent = this; }, 0, 256); + memcpy(maxOfMax, other.maxOfMax, sizeof(maxOfMax)); memcpy(partialKey(), &other + 1, partialKeyLen); } @@ -562,9 +568,12 @@ template struct BoundedFreeListAllocator { T *allocate(int partialKeyCapacity) { T *result = allocate_helper(partialKeyCapacity); - if constexpr (std::is_same_v || std::is_same_v) { + if constexpr (!std::is_same_v) { memset(result->children, 0, sizeof(result->children)); } + if constexpr (std::is_same_v) { + memset(result->maxOfMax, 0, sizeof(result->maxOfMax)); + } return result; } @@ -1689,17 +1698,35 @@ bool checkMaxBetweenExclusive(Node *n, int begin, int end, return result; } case Type_Node256: { - bool conflict[256] = {}; auto *self = static_cast(n); - self->bitSet.forEachInRange( - [&](int i) { - conflict[i] = self->children[i].childMaxVersion > readVersion; - }, - begin, end); - // Should be vectorized + // Check the first page + if (self->maxOfMax[begin >> 4] > readVersion) { + bool result = true; + for (int i = 0; i < 16; ++i) { + int j = (begin & ~15) + i; + result &= !((self->children[j].childMaxVersion > readVersion) & + (begin <= j) & (j < end)); + } + if (!result) { + return result; + } + } + // Check the last page + if (end >= 1 && self->maxOfMax[(end - 1) >> 4] > readVersion) { + bool result = true; + for (int i = 0; i < 16; ++i) { + int j = ((end - 1) & ~15) + i; + result &= !((self->children[j].childMaxVersion > readVersion) & + (begin <= j) & (j < end)); + } + if (!result) { + return result; + } + } + // Check inner pages bool result = true; - for (auto c : conflict) { - result &= !c; + for (int i = (begin >> 4) + 1; i < (end - 1) >> 4; ++i) { + result &= self->maxOfMax[i] <= readVersion; } return result; } @@ -2635,6 +2662,7 @@ void setMaxVersion(Node *n, ConflictSet::Impl *impl, int64_t newMax) { auto *n256 = static_cast(n); assert(n256->bitSet.test(index)); n256->children[index].childMaxVersion = newMax; + n256->maxOfMax[index >> 4] = std::max(n256->maxOfMax[index >> 4], newMax); return; } default: // GCOVR_EXCL_LINE