diff --git a/ConflictSet.cpp b/ConflictSet.cpp index 7078e58..299f72c 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -1544,6 +1544,46 @@ void rezero(Node *n, InternalVersionT z) { } } +void mergeWithChild(Node *&self, WriteContext *tls, ConflictSet::Impl *impl, + Node *&dontInvalidate, Node3 *self3) { + assert(!self3->entryPresent); + auto *child = self3->children[0]; + int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen; + + if (minCapacity > child->getCapacity()) { + const bool update = child == dontInvalidate; + freeAndMakeCapacityAtLeast(child, minCapacity, tls, impl, true); + if (update) { + dontInvalidate = child; + } + } + + // Merge partial key with child +#if DEBUG_VERBOSE && !defined(NDEBUG) + fprintf(stderr, "Merge %s into %s\n", getSearchPathPrintable(self).c_str(), + getSearchPathPrintable(child).c_str()); +#endif + + InternalVersionT childMaxVersion = self3->childMaxVersion[0]; + + // Construct new partial key for child + memmove(child->partialKey() + self3->partialKeyLen + 1, child->partialKey(), + child->partialKeyLen); + memcpy(child->partialKey(), self3->partialKey(), self->partialKeyLen); + child->partialKey()[self3->partialKeyLen] = self3->index[0]; + child->partialKeyLen += 1 + self3->partialKeyLen; + + child->parent = self->parent; + child->parentsIndex = self->parentsIndex; + + // Max versions are stored in the parent, so we need to update it now + // that we have a new parent. + setMaxVersion(child, impl, std::max(childMaxVersion, tls->zero)); + + self = child; + tls->release(self3); +} + void maybeDownsize(Node *self, WriteContext *tls, ConflictSet::Impl *impl, Node *&dontInvalidate) { @@ -1562,45 +1602,7 @@ void maybeDownsize(Node *self, WriteContext *tls, ConflictSet::Impl *impl, getInTree(self, impl) = newSelf; tls->release(self3); } else if (self->numChildren == 1 && !self->entryPresent) { - auto *child = self3->children[0]; - int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen; - - if (minCapacity > child->getCapacity()) { - const bool update = child == dontInvalidate; - freeAndMakeCapacityAtLeast(child, minCapacity, tls, impl, true); - if (update) { - dontInvalidate = child; - } - } - - // Merge partial key with child -#if DEBUG_VERBOSE && !defined(NDEBUG) - fprintf(stderr, "Merge %s into %s\n", - getSearchPathPrintable(self).c_str(), - getSearchPathPrintable(child).c_str()); -#endif - - InternalVersionT childMaxVersion = maxVersion(child, impl); - - // Construct new partial key for child - memmove(child->partialKey() + self3->partialKeyLen + 1, - child->partialKey(), child->partialKeyLen); - memcpy(child->partialKey(), self3->partialKey(), self->partialKeyLen); - child->partialKey()[self3->partialKeyLen] = self3->index[0]; - child->partialKeyLen += 1 + self3->partialKeyLen; - - child->parent = self->parent; - child->parentsIndex = self->parentsIndex; - - // Max versions are stored in the parent, so we need to update it now - // that we have a new parent. - setMaxVersion(child, impl, childMaxVersion); - if (child->parent) { - rezero(child->parent, tls->zero); - } - - getInTree(self, impl) = child; - tls->release(self3); + mergeWithChild(getInTree(self, impl), tls, impl, dontInvalidate, self3); } } break; case Type_Node16: @@ -2812,28 +2814,50 @@ Node **insert(Node **self, std::span key, return self; } -void destroyTree(Node *root) { +void eraseTree(Node *root, WriteContext *tls) { Arena arena; auto toFree = vector(arena); toFree.push_back(root); -#if SHOW_MEMORY - for (auto *iter = root; iter != nullptr; iter = nextPhysical(iter)) { - removeNode(iter); - removeKey(iter); - } -#endif - while (toFree.size() > 0) { auto *n = toFree.back(); toFree.pop_back(); - // Add all children to toFree - for (auto c = getChildGeq(n, 0); c != nullptr; - c = getChildGeq(n, c->parentsIndex + 1)) { - assert(c != nullptr); - toFree.push_back(c); + tls->accum.entries_erased += n->entryPresent; + ++tls->accum.nodes_released; + + removeNode(n); + removeKey(n); + + switch (n->getType()) { + case Type_Node0: { + auto *n0 = static_cast(n); + tls->release(n0); + } break; + case Type_Node3: { + auto *n3 = static_cast(n); + toFree.append(std::span(n3->children, n3->numChildren)); + tls->release(n3); + } break; + case Type_Node16: { + auto *n16 = static_cast(n); + toFree.append(std::span(n16->children, n16->numChildren)); + tls->release(n16); + } break; + case Type_Node48: { + auto *n48 = static_cast(n); + toFree.append(std::span(n48->children, n48->numChildren)); + tls->release(n48); + } break; + case Type_Node256: { + auto *n256 = static_cast(n); + auto *out = toFree.unsafePrepareAppend(n256->numChildren).data(); + n256->bitSet.forEachSet([&](int i) { *out++ = n256->children[i]; }); + assert(out == toFree.end()); + tls->release(n256); + } break; + default: // GCOVR_EXCL_LINE + __builtin_unreachable(); // GCOVR_EXCL_LINE } - safe_free(n, n->size()); } } @@ -3070,7 +3094,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { if (oldestExtantVersion < writeVersion - kMaxCorrectVersionWindow) [[unlikely]] { if (writeVersion > newestVersionFullPrecision + kNominalVersionWindow) { - destroyTree(root); + eraseTree(root, &tls); init(writeVersion - kNominalVersionWindow); } @@ -3238,7 +3262,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { initMetrics(); } ~Impl() { - destroyTree(root); + eraseTree(root, &tls); safe_free(metrics, metricsCount * sizeof(metrics[0])); } @@ -3416,6 +3440,7 @@ InternalVersionT exchangeMaxVersion(Node *n, InternalVersionT newMax) { } void setMaxVersion(Node *n, ConflictSet::Impl *impl, InternalVersionT newMax) { + assert(newMax >= InternalVersionT::zero); int index = n->parentsIndex; n = n->parent; if (n == nullptr) { @@ -3926,7 +3951,7 @@ checkMaxVersion(Node *root, Node *node, InternalVersionT oldestVersion, bool success = true; if (node->partialKeyLen > 0) { - fprintf(stderr, "Root cannot have a partial key"); + fprintf(stderr, "Root cannot have a partial key\n"); success = false; } checkParentPointers(node, success); diff --git a/Internal.h b/Internal.h index 10ddf17..e90312a 100644 --- a/Internal.h +++ b/Internal.h @@ -273,6 +273,16 @@ template struct Vector { size_ += slice.size(); } + // Caller must write to the returned slice + std::span unsafePrepareAppend(int appendSize) { + if (size_ + appendSize > capacity) { + grow(std::max(size_ + appendSize, capacity * 2)); + } + auto result = std::span(t + size_, appendSize); + size_ += appendSize; + return result; + } + void push_back(const T &t) { append(std::span(&t, 1)); } T *begin() { return t; } diff --git a/ServerBench.cpp b/ServerBench.cpp index 9237056..5912d4b 100644 --- a/ServerBench.cpp +++ b/ServerBench.cpp @@ -164,6 +164,63 @@ double toSeconds(timeval t) { return double(t.tv_sec) + double(t.tv_usec) * 1e-6; } +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __linux__ +struct PerfCounter { + explicit PerfCounter(int event) { + struct perf_event_attr pe; + + memset(&pe, 0, sizeof(pe)); + pe.type = PERF_TYPE_HARDWARE; + pe.size = sizeof(pe); + pe.config = event; + pe.inherit = 1; + pe.exclude_kernel = 1; + pe.exclude_hv = 1; + + fd = perf_event_open(&pe, 0, -1, -1, 0); + if (fd == -1) { + fprintf(stderr, "Error opening leader %llx\n", pe.config); + exit(EXIT_FAILURE); + } + } + + int64_t total() { + int64_t count; + if (read(fd, &count, sizeof(count)) != sizeof(count)) { + perror("read instructions from perf"); + abort(); + } + return count; + } + + ~PerfCounter() { close(fd); } + +private: + int fd; + static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, + int cpu, int group_fd, unsigned long flags) { + int ret; + + ret = syscall(SYS_perf_event_open, hw_event, pid, cpu, group_fd, flags); + return ret; + } +}; +#else +struct PerfCounter { + explicit PerPerfCounter(int) {} + int64_t total() { return 0; } +}; +#endif + int main(int argc, char **argv) { if (argc != 3) { goto fail; @@ -176,6 +233,8 @@ int main(int argc, char **argv) { int metricsCount; cs.getMetricsV1(&metrics, &metricsCount); + PerfCounter instructions{PERF_COUNT_HW_INSTRUCTIONS}; + PerfCounter cycles{PERF_COUNT_HW_CPU_CYCLES}; auto w = std::thread{workload, &cs}; for (;;) { @@ -203,6 +262,16 @@ int main(int argc, char **argv) { "transactions_total "; body += std::to_string(transactions.load(std::memory_order_relaxed)); body += "\n"; + body += "# HELP instructions_total Total number of instructions\n" + "# TYPE instructions_total counter\n" + "instructions_total "; + body += std::to_string(instructions.total()); + body += "\n"; + body += "# HELP cycles_total Total number of cycles\n" + "# TYPE cycles_total counter\n" + "cycles_total "; + body += std::to_string(cycles.total()); + body += "\n"; for (int i = 0; i < metricsCount; ++i) { body += "# HELP ";