diff --git a/Bench.cpp b/Bench.cpp index 7b05a57..cf04f0d 100644 --- a/Bench.cpp +++ b/Bench.cpp @@ -17,26 +17,26 @@ constexpr int kPrefixLen = 0; constexpr int kMvccWindow = 100000; -std::span makeKey(Arena &arena, int index) { +TrivialSpan makeKey(Arena &arena, int index) { - auto result = - std::span{new (arena) uint8_t[4 + kPrefixLen], 4 + kPrefixLen}; + uint8_t *buf = new (arena) uint8_t[4 + kPrefixLen]; + auto result = TrivialSpan{buf, 4 + kPrefixLen}; index = __builtin_bswap32(index); - memset(result.data(), 0, kPrefixLen); - memcpy(result.data() + kPrefixLen, &index, 4); + memset(buf, 0, kPrefixLen); + memcpy(buf, &index, 4); return result; } -ConflictSet::ReadRange singleton(Arena &arena, std::span key) { - auto r = - std::span(new (arena) uint8_t[key.size() + 1], key.size() + 1); - memcpy(r.data(), key.data(), key.size()); - r[key.size()] = 0; +ConflictSet::ReadRange singleton(Arena &arena, TrivialSpan key) { + uint8_t *buf = new (arena) uint8_t[key.size() + 1]; + auto r = TrivialSpan(buf, key.size() + 1); + memcpy(buf, key.data(), key.size()); + buf[key.size()] = 0; return {{key.data(), int(key.size())}, {r.data(), int(r.size())}, 0}; } -ConflictSet::ReadRange prefixRange(Arena &arena, std::span key) { +ConflictSet::ReadRange prefixRange(Arena &arena, TrivialSpan key) { int index; for (index = key.size() - 1; index >= 0; index--) if ((key[index]) != 255) @@ -48,9 +48,10 @@ ConflictSet::ReadRange prefixRange(Arena &arena, std::span key) { assert(false); } - auto r = std::span(new (arena) uint8_t[index + 1], index + 1); - memcpy(r.data(), key.data(), index + 1); - r[r.size() - 1]++; + uint8_t *buf = new (arena) uint8_t[index + 1]; + auto r = TrivialSpan(buf, index + 1); + memcpy(buf, key.data(), index + 1); + buf[r.size() - 1]++; return {{key.data(), int(key.size())}, {r.data(), int(r.size())}, 0}; } @@ -81,14 +82,7 @@ void benchConflictSet() { ++version; } - // I don't know why std::less didn't work /shrug - struct Less { - bool operator()(const std::span &lhs, - const std::span &rhs) const { - return lhs < rhs; - } - }; - auto points = set, Less>(arena); + auto points = set>(arena); while (points.size() < kOpsPerTx * 2 + 1) { // TODO don't use rand? diff --git a/ConflictSet.cpp b/ConflictSet.cpp index 1e4fb54..27ae53a 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -1213,14 +1213,13 @@ TaggedNodePointer getChild(Node *self, uint8_t index) { } } -TaggedNodePointer *getChildUpdatingMaxVersion(Node0 *, - std::span &, +TaggedNodePointer *getChildUpdatingMaxVersion(Node0 *, TrivialSpan &, InternalVersionT) { return nullptr; } -TaggedNodePointer * -getChildUpdatingMaxVersion(Node3 *self, std::span &remaining, - InternalVersionT maxVersion) { +TaggedNodePointer *getChildUpdatingMaxVersion(Node3 *self, + TrivialSpan &remaining, + InternalVersionT maxVersion) { assert(remaining.size() > 0); int index = remaining.front(); auto key = remaining.subspan(1, remaining.size() - 1); @@ -1241,9 +1240,9 @@ getChildUpdatingMaxVersion(Node3 *self, std::span &remaining, self->childMaxVersion[i] = maxVersion; return &self->children[i]; } -TaggedNodePointer * -getChildUpdatingMaxVersion(Node16 *self, std::span &remaining, - InternalVersionT maxVersion) { +TaggedNodePointer *getChildUpdatingMaxVersion(Node16 *self, + TrivialSpan &remaining, + InternalVersionT maxVersion) { assert(remaining.size() > 0); int index = remaining.front(); auto key = remaining.subspan(1, remaining.size() - 1); @@ -1264,9 +1263,9 @@ getChildUpdatingMaxVersion(Node16 *self, std::span &remaining, self->childMaxVersion[i] = maxVersion; return &self->children[i]; } -TaggedNodePointer * -getChildUpdatingMaxVersion(Node48 *self, std::span &remaining, - InternalVersionT maxVersion) { +TaggedNodePointer *getChildUpdatingMaxVersion(Node48 *self, + TrivialSpan &remaining, + InternalVersionT maxVersion) { assert(remaining.size() > 0); int index = remaining.front(); auto key = remaining.subspan(1, remaining.size() - 1); @@ -1289,9 +1288,9 @@ getChildUpdatingMaxVersion(Node48 *self, std::span &remaining, std::max(self->maxOfMax[i >> Node48::kMaxOfMaxShift], maxVersion); return &self->children[i]; } -TaggedNodePointer * -getChildUpdatingMaxVersion(Node256 *self, std::span &remaining, - InternalVersionT maxVersion) { +TaggedNodePointer *getChildUpdatingMaxVersion(Node256 *self, + TrivialSpan &remaining, + InternalVersionT maxVersion) { assert(remaining.size() > 0); int index = remaining.front(); auto key = remaining.subspan(1, remaining.size() - 1); @@ -1319,9 +1318,9 @@ getChildUpdatingMaxVersion(Node256 *self, std::span &remaining, // If a child of self lies along the search path of remaining, return a pointer // to that child, update max version, and consume the matching prefix bytes from // remaining. Otherwise return nullptr without changing the tree at all. -TaggedNodePointer * -getChildUpdatingMaxVersion(Node *self, std::span &remaining, - InternalVersionT maxVersion) { +TaggedNodePointer *getChildUpdatingMaxVersion(Node *self, + TrivialSpan &remaining, + InternalVersionT maxVersion) { switch (self->getType()) { case Type_Node0: return getChildUpdatingMaxVersion(static_cast(self), remaining, @@ -1560,8 +1559,7 @@ TaggedNodePointer getFirstChildExists(Node *self) { // GCOVR_EXCL_STOP } -void consumePartialKeyFull(TaggedNodePointer &self, - std::span &key, +void consumePartialKeyFull(TaggedNodePointer &self, TrivialSpan &key, InternalVersionT writeVersion, WriteContext *writeContext) { // Handle an existing partial key @@ -1607,7 +1605,7 @@ void consumePartialKeyFull(TaggedNodePointer &self, // Consume any partial key of `self`, and update `self` and // `key` such that `self` is along the search path of `key` inline __attribute__((always_inline)) void -consumePartialKey(TaggedNodePointer &self, std::span &key, +consumePartialKey(TaggedNodePointer &self, TrivialSpan &key, InternalVersionT writeVersion, WriteContext *writeContext) { if (self->partialKeyLen > 0) { consumePartialKeyFull(self, key, writeVersion, writeContext); @@ -1618,8 +1616,7 @@ consumePartialKey(TaggedNodePointer &self, std::span &key, // such that the search path of the result + key is the same as the search path // of self + key before the call. Creates a node if necessary. Updates // `maxVersion` for result. -TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, - std::span &key, +TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key, InternalVersionT newMaxVersion, WriteContext *writeContext) { @@ -2805,7 +2802,7 @@ bool checkMaxBetweenExclusive(Node *n, int begin, int end, } } -Vector getSearchPath(Arena &arena, Node *n) { +TrivialSpan getSearchPath(Arena &arena, Node *n) { assert(n != nullptr); auto result = vector(arena); for (;;) { @@ -2819,7 +2816,7 @@ Vector getSearchPath(Arena &arena, Node *n) { n = n->parent; } std::reverse(result.begin(), result.end()); - return result; + return {result.begin(), result.size()}; } // Return true if the max version among all keys that start with key + [child], @@ -2828,8 +2825,8 @@ Vector getSearchPath(Arena &arena, Node *n) { // Precondition: transitively, no child of n has a search path that's a longer // prefix of key than n template -bool checkRangeStartsWith(NodeT *nTyped, std::span key, - int begin, int end, InternalVersionT readVersion, +bool checkRangeStartsWith(NodeT *nTyped, TrivialSpan key, int begin, int end, + InternalVersionT readVersion, ReadContext *readContext) { Node *n; #if DEBUG_VERBOSE && !defined(NDEBUG) @@ -2921,7 +2918,7 @@ checkMaxBetweenExclusiveImpl(Node256 *n, int begin, int end, // postcondition. Nodes along the search path may be invalidated. Callers must // ensure that the max version of the self argument is updated. [[nodiscard]] TaggedNodePointer *insert(TaggedNodePointer *self, - std::span key, + TrivialSpan key, InternalVersionT writeVersion, WriteContext *writeContext) { @@ -2983,7 +2980,7 @@ void eraseTree(Node *root, WriteContext *writeContext) { } } -void addPointWrite(TaggedNodePointer &root, std::span key, +void addPointWrite(TaggedNodePointer &root, TrivialSpan key, InternalVersionT writeVersion, WriteContext *writeContext) { ++writeContext->accum.point_writes; auto n = *insert(&root, key, writeVersion, writeContext); @@ -3113,9 +3110,8 @@ struct AddedWriteRange { Node *endNode; }; -AddedWriteRange addWriteRange(Node *beginRoot, std::span begin, - Node *endRoot, std::span end, - InternalVersionT writeVersion, +AddedWriteRange addWriteRange(Node *beginRoot, TrivialSpan begin, Node *endRoot, + TrivialSpan end, InternalVersionT writeVersion, WriteContext *writeContext, ConflictSet::Impl *impl) { @@ -3176,9 +3172,9 @@ void eraseInRange(Node *beginNode, Node *endNode, WriteContext *writeContext, fixupMaxVersion(iter, writeContext); } -void addWriteRange(TaggedNodePointer &root, std::span begin, - std::span end, InternalVersionT writeVersion, - WriteContext *writeContext, ConflictSet::Impl *impl) { +void addWriteRange(TaggedNodePointer &root, TrivialSpan begin, TrivialSpan end, + InternalVersionT writeVersion, WriteContext *writeContext, + ConflictSet::Impl *impl) { int lcp = longestCommonPrefix(begin.data(), end.data(), std::min(begin.size(), end.size())); if (lcp == int(begin.size()) && end.size() == begin.size() + 1 && @@ -3195,7 +3191,7 @@ void addWriteRange(TaggedNodePointer &root, std::span begin, eraseInRange(beginNode, endNode, writeContext, impl); } -Node *firstGeqPhysical(Node *n, const std::span key) { +Node *firstGeqPhysical(Node *n, const TrivialSpan key) { auto remaining = key; for (;;) { if (remaining.size() == 0) { @@ -3278,13 +3274,13 @@ struct Job { Node *root, int64_t oldestVersionFullPrecision); Node *n; - std::span begin; + TrivialSpan begin; InternalVersionT maxV; - std::span end; // range read only - std::span remaining; // range read only - Node *child; // range read only - int lcp; // range read only - Node *commonPrefixNode; // range read only + TrivialSpan end; // range read only + TrivialSpan remaining; // range read only + Node *child; // range read only + int lcp; // range read only + Node *commonPrefixNode; // range read only InternalVersionT readVersion; ConflictSet::Result *result; Continuation continuation; @@ -4068,8 +4064,8 @@ PRESERVE_NONE void right_side_iter(Job *job, Context *context) { void Job::init(const ConflictSet::ReadRange *read, ConflictSet::Result *result, Node *root, int64_t oldestVersionFullPrecision) { - auto begin = std::span(read->begin.p, read->begin.len); - auto end = std::span(read->end.p, read->end.len); + auto begin = TrivialSpan(read->begin.p, read->begin.len); + auto end = TrivialSpan(read->end.p, read->end.len); if (read->readVersion < oldestVersionFullPrecision) [[unlikely]] { *result = ConflictSet::TooOld; continuation = complete; @@ -4096,13 +4092,13 @@ typedef PRESERVE_NONE void (*Continuation)(struct Job *, struct Context *); // State relevant to an individual insertion struct Job { - std::span remaining; + TrivialSpan remaining; Node *n; int index; - std::span begin; // Range write only - std::span end; // Range write only - Node *endNode; // Range write only - int commonPrefixLen; // Range write only + TrivialSpan begin; // Range write only + TrivialSpan end; // Range write only + Node *endNode; // Range write only + int commonPrefixLen; // Range write only // State for context switching machinery - not application specific Continuation continuation; @@ -4116,10 +4112,10 @@ struct Job { // path of the original key struct Result { Node *insertionPoint; - std::span remaining; + TrivialSpan remaining; - Node *endInsertionPoint; // Range write only - std::span endRemaining; // Range write only + Node *endInsertionPoint; // Range write only + TrivialSpan endRemaining; // Range write only }; // State relevant to every insertion @@ -4285,10 +4281,10 @@ void Job::init(Context *context, int index) { goto pointWrite; } - begin = std::span(context->writes[index].begin.p, - context->writes[index].begin.len); - end = std::span(context->writes[index].end.p, - context->writes[index].end.len); + begin = TrivialSpan(context->writes[index].begin.p, + context->writes[index].begin.len); + end = + TrivialSpan(context->writes[index].end.p, context->writes[index].end.len); commonPrefixLen = longestCommonPrefix(begin.data(), end.data(), std::min(begin.size(), end.size())); @@ -4297,8 +4293,7 @@ void Job::init(Context *context, int index) { goto pointWrite; } - remaining = - std::span(context->writes[index].begin.p, commonPrefixLen); + remaining = TrivialSpan(context->writes[index].begin.p, commonPrefixLen); if (commonPrefixLen > 0) { // common prefix iter will set endNode @@ -4313,8 +4308,8 @@ void Job::init(Context *context, int index) { return; pointWrite: - remaining = std::span(context->writes[index].begin.p, - context->writes[index].begin.len); + remaining = TrivialSpan(context->writes[index].begin.p, + context->writes[index].begin.len); if (remaining.size() == 0) [[unlikely]] { context->results[index] = {n, remaining, nullptr, {}}; continuation = complete; @@ -4330,7 +4325,7 @@ namespace { // Logically this is the same as performing firstGeq and then checking against // point or range version according to cmp, but this version short circuits as // soon as it can prove that there's no conflict. -bool checkPointRead(Node *n, const std::span key, +bool checkPointRead(Node *n, const TrivialSpan key, InternalVersionT readVersion, ReadContext *readContext) { ++readContext->point_read_accum; #if DEBUG_VERBOSE && !defined(NDEBUG) @@ -4404,7 +4399,7 @@ downLeftSpine: // Logically this is the same as performing firstGeq and then checking against // max version or range version if this prefix doesn't exist, but this version // short circuits as soon as it can prove that there's no conflict. -bool checkPrefixRead(Node *n, const std::span key, +bool checkPrefixRead(Node *n, const TrivialSpan key, InternalVersionT readVersion, ReadContext *readContext) { ++readContext->prefix_read_accum; #if DEBUG_VERBOSE && !defined(NDEBUG) @@ -4478,7 +4473,7 @@ downLeftSpine: // Return true if the max version among all keys that start with key[:prefixLen] // that are >= key is <= readVersion -bool checkRangeLeftSide(Node *n, std::span key, int prefixLen, +bool checkRangeLeftSide(Node *n, TrivialSpan key, int prefixLen, InternalVersionT readVersion, ReadContext *readContext) { auto remaining = key; @@ -4565,7 +4560,7 @@ downLeftSpine: // Return true if the max version among all keys that start with key[:prefixLen] // that are < key is <= readVersion -bool checkRangeRightSide(Node *n, std::span key, int prefixLen, +bool checkRangeRightSide(Node *n, TrivialSpan key, int prefixLen, InternalVersionT readVersion, ReadContext *readContext) { auto remaining = key; @@ -4658,9 +4653,8 @@ downLeftSpine: } return n->entry.rangeVersion <= readVersion; } -bool checkRangeRead(Node *n, std::span begin, - std::span end, InternalVersionT readVersion, - ReadContext *readContext) { +bool checkRangeRead(Node *n, TrivialSpan begin, TrivialSpan end, + InternalVersionT readVersion, ReadContext *readContext) { int lcp = longestCommonPrefix(begin.data(), end.data(), std::min(begin.size(), end.size())); if (lcp == int(begin.size()) && end.size() == begin.size() + 1 && @@ -4746,14 +4740,12 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { bool ok; if (reads[i].end.len == 0) { ok = checkPointRead( - root, - std::span(reads[i].begin.p, reads[i].begin.len), + root, TrivialSpan(reads[i].begin.p, reads[i].begin.len), InternalVersionT(reads[i].readVersion), &context.readContext); } else { ok = checkRangeRead( - root, - std::span(reads[i].begin.p, reads[i].begin.len), - std::span(reads[i].end.p, reads[i].end.len), + root, TrivialSpan(reads[i].begin.p, reads[i].begin.len), + TrivialSpan(reads[i].end.p, reads[i].end.len), InternalVersionT(reads[i].readVersion), &context.readContext); } result[i] = ok ? Commit : Conflict; @@ -4977,8 +4969,8 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { } else { for (int i = 0; i < count; ++i) { const auto &w = writes[i]; - auto begin = std::span(w.begin.p, w.begin.len); - auto end = std::span(w.end.p, w.end.len); + auto begin = TrivialSpan(w.begin.p, w.begin.len); + auto end = TrivialSpan(w.end.p, w.end.len); if (w.end.len > 0) { addWriteRange(root, begin, end, InternalVersionT(writeVersion), &writeContext, this); @@ -5216,7 +5208,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { WriteContext writeContext; Arena removalKeyArena; - std::span removalKey; + TrivialSpan removalKey; int64_t keyUpdates; TaggedNodePointer root; @@ -5363,7 +5355,7 @@ double internal_getMetricValue(const ConflictSet::MetricsV1 *metric) { // GCOVR_EXCL_START -Node *firstGeqLogical(Node *n, const std::span key) { +Node *firstGeqLogical(Node *n, const TrivialSpan key) { auto remaining = key; for (;;) { if (remaining.size() == 0) { @@ -5624,8 +5616,8 @@ void checkParentPointers(Node *node, bool &success) { } Node *firstGeq(Node *n, std::string_view key) { - return firstGeqLogical( - n, std::span((const uint8_t *)key.data(), key.size())); + return firstGeqLogical(n, + TrivialSpan((const uint8_t *)key.data(), key.size())); } #if USE_64_BIT diff --git a/Internal.h b/Internal.h index c81f704..dfc3d70 100644 --- a/Internal.h +++ b/Internal.h @@ -26,9 +26,38 @@ using namespace weaselab; #define DEBUG_VERBOSE 0 #define SHOW_MEMORY 0 -[[nodiscard]] inline auto -operator<=>(const std::span &lhs, - const std::span &rhs) noexcept { +// std::span is not trivially constructible. We want a span that leaves its +// members uninitialized for performance reasons. +struct TrivialSpan { + TrivialSpan() = default; + TrivialSpan(const uint8_t *begin, int len) : begin(begin), len(len) {} + + uint8_t back() const { + assert(len > 0); + return begin[len - 1]; + } + uint8_t front() const { + assert(len > 0); + return begin[0]; + } + uint8_t operator[](int i) const { + assert(0 <= i); + assert(i < len); + return begin[i]; + } + int size() const { return len; } + TrivialSpan subspan(int offset, int len) { return {begin + offset, len}; } + const uint8_t *data() const { return begin; } + +private: + const uint8_t *begin; + int len; +}; + +static_assert(std::is_trivial_v); + +[[nodiscard]] inline auto operator<=>(const TrivialSpan &lhs, + const TrivialSpan &rhs) noexcept { int cl = std::min(lhs.size(), rhs.size()); if (cl > 0) { if (auto c = memcmp(lhs.data(), rhs.data(), cl) <=> 0; c != 0) { @@ -38,7 +67,7 @@ operator<=>(const std::span &lhs, return lhs.size() <=> rhs.size(); } -[[nodiscard]] inline auto operator<=>(const std::span &lhs, +[[nodiscard]] inline auto operator<=>(const TrivialSpan &lhs, const ConflictSet::Key &rhs) noexcept { int cl = std::min(lhs.size(), rhs.len); if (cl > 0) { @@ -46,7 +75,7 @@ operator<=>(const std::span &lhs, return c; } } - return lhs.size() <=> size_t(rhs.len); + return lhs.size() <=> rhs.len; } [[nodiscard]] inline auto operator<=>(const ConflictSet::Key &lhs, @@ -688,10 +717,8 @@ struct TestDriver { arbitrary->randomBytes(begin + prefixLen, keyLen - prefixLen); writes[i].end.len = keyLen; writes[i].end.p = begin; - auto c = - std::span(writes[i].begin.p, - writes[i].begin.len) <=> - std::span(writes[i].end.p, writes[i].end.len); + auto c = TrivialSpan(writes[i].begin.p, writes[i].begin.len) <=> + TrivialSpan(writes[i].end.p, writes[i].end.len); if (c > 0) { using std::swap; swap(writes[i].begin, writes[i].end);