Bring in some of the changes from erase-between branch

ConflictSet.cpp

@@ -1544,6 +1544,46 @@ void rezero(Node *n, InternalVersionT z) {
   }
 }
 
+void mergeWithChild(Node *&self, WriteContext *tls, ConflictSet::Impl *impl,
+                    Node *&dontInvalidate, Node3 *self3) {
+  assert(!self3->entryPresent);
+  auto *child = self3->children[0];
+  int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen;
+
+  if (minCapacity > child->getCapacity()) {
+    const bool update = child == dontInvalidate;
+    freeAndMakeCapacityAtLeast(child, minCapacity, tls, impl, true);
+    if (update) {
+      dontInvalidate = child;
+    }
+  }
+
+  // Merge partial key with child
+#if DEBUG_VERBOSE && !defined(NDEBUG)
+  fprintf(stderr, "Merge %s into %s\n", getSearchPathPrintable(self).c_str(),
+          getSearchPathPrintable(child).c_str());
+#endif
+
+  InternalVersionT childMaxVersion = self3->childMaxVersion[0];
+
+  // Construct new partial key for child
+  memmove(child->partialKey() + self3->partialKeyLen + 1, child->partialKey(),
+          child->partialKeyLen);
+  memcpy(child->partialKey(), self3->partialKey(), self->partialKeyLen);
+  child->partialKey()[self3->partialKeyLen] = self3->index[0];
+  child->partialKeyLen += 1 + self3->partialKeyLen;
+
+  child->parent = self->parent;
+  child->parentsIndex = self->parentsIndex;
+
+  // Max versions are stored in the parent, so we need to update it now
+  // that we have a new parent.
+  setMaxVersion(child, impl, std::max(childMaxVersion, tls->zero));
+
+  self = child;
+  tls->release(self3);
+}
+
 void maybeDownsize(Node *self, WriteContext *tls, ConflictSet::Impl *impl,
                    Node *&dontInvalidate) {
 
@@ -1562,45 +1602,7 @@ void maybeDownsize(Node *self, WriteContext *tls, ConflictSet::Impl *impl,
       getInTree(self, impl) = newSelf;
       tls->release(self3);
     } else if (self->numChildren == 1 && !self->entryPresent) {
-      auto *child = self3->children[0];
+      mergeWithChild(getInTree(self, impl), tls, impl, dontInvalidate, self3);
-      int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen;
-
-      if (minCapacity > child->getCapacity()) {
-        const bool update = child == dontInvalidate;
-        freeAndMakeCapacityAtLeast(child, minCapacity, tls, impl, true);
-        if (update) {
-          dontInvalidate = child;
-        }
-      }
-
-      // Merge partial key with child
-#if DEBUG_VERBOSE && !defined(NDEBUG)
-      fprintf(stderr, "Merge %s into %s\n",
-              getSearchPathPrintable(self).c_str(),
-              getSearchPathPrintable(child).c_str());
-#endif
-
-      InternalVersionT childMaxVersion = maxVersion(child, impl);
-
-      // Construct new partial key for child
-      memmove(child->partialKey() + self3->partialKeyLen + 1,
-              child->partialKey(), child->partialKeyLen);
-      memcpy(child->partialKey(), self3->partialKey(), self->partialKeyLen);
-      child->partialKey()[self3->partialKeyLen] = self3->index[0];
-      child->partialKeyLen += 1 + self3->partialKeyLen;
-
-      child->parent = self->parent;
-      child->parentsIndex = self->parentsIndex;
-
-      // Max versions are stored in the parent, so we need to update it now
-      // that we have a new parent.
-      setMaxVersion(child, impl, childMaxVersion);
-      if (child->parent) {
-        rezero(child->parent, tls->zero);
-      }
-
-      getInTree(self, impl) = child;
-      tls->release(self3);
     }
   } break;
   case Type_Node16:
@@ -2812,28 +2814,50 @@ Node **insert(Node **self, std::span<const uint8_t> key,
   return self;
 }
 
-void destroyTree(Node *root) {
+void eraseTree(Node *root, WriteContext *tls) {
   Arena arena;
   auto toFree = vector<Node *>(arena);
   toFree.push_back(root);
 
-#if SHOW_MEMORY
-  for (auto *iter = root; iter != nullptr; iter = nextPhysical(iter)) {
-    removeNode(iter);
-    removeKey(iter);
-  }
-#endif
-
   while (toFree.size() > 0) {
     auto *n = toFree.back();
     toFree.pop_back();
-    // Add all children to toFree
+    tls->accum.entries_erased += n->entryPresent;
-    for (auto c = getChildGeq(n, 0); c != nullptr;
+    ++tls->accum.nodes_released;
-         c = getChildGeq(n, c->parentsIndex + 1)) {
+
-      assert(c != nullptr);
+    removeNode(n);
-      toFree.push_back(c);
+    removeKey(n);
+
+    switch (n->getType()) {
+    case Type_Node0: {
+      auto *n0 = static_cast<Node0 *>(n);
+      tls->release(n0);
+    } break;
+    case Type_Node3: {
+      auto *n3 = static_cast<Node3 *>(n);
+      toFree.append(std::span<Node *>(n3->children, n3->numChildren));
+      tls->release(n3);
+    } break;
+    case Type_Node16: {
+      auto *n16 = static_cast<Node16 *>(n);
+      toFree.append(std::span<Node *>(n16->children, n16->numChildren));
+      tls->release(n16);
+    } break;
+    case Type_Node48: {
+      auto *n48 = static_cast<Node48 *>(n);
+      toFree.append(std::span<Node *>(n48->children, n48->numChildren));
+      tls->release(n48);
+    } break;
+    case Type_Node256: {
+      auto *n256 = static_cast<Node256 *>(n);
+      auto *out = toFree.unsafePrepareAppend(n256->numChildren).data();
+      n256->bitSet.forEachSet([&](int i) { *out++ = n256->children[i]; });
+      assert(out == toFree.end());
+      tls->release(n256);
+    } break;
+    default:                   // GCOVR_EXCL_LINE
+      __builtin_unreachable(); // GCOVR_EXCL_LINE
     }
-    safe_free(n, n->size());
   }
 }
 
@@ -3070,7 +3094,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
     if (oldestExtantVersion < writeVersion - kMaxCorrectVersionWindow)
         [[unlikely]] {
       if (writeVersion > newestVersionFullPrecision + kNominalVersionWindow) {
-        destroyTree(root);
+        eraseTree(root, &tls);
         init(writeVersion - kNominalVersionWindow);
       }
 
@@ -3238,7 +3262,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
     initMetrics();
   }
   ~Impl() {
-    destroyTree(root);
+    eraseTree(root, &tls);
     safe_free(metrics, metricsCount * sizeof(metrics[0]));
   }
 
@@ -3416,6 +3440,7 @@ InternalVersionT exchangeMaxVersion(Node *n, InternalVersionT newMax) {
 }
 
 void setMaxVersion(Node *n, ConflictSet::Impl *impl, InternalVersionT newMax) {
+  assert(newMax >= InternalVersionT::zero);
   int index = n->parentsIndex;
   n = n->parent;
   if (n == nullptr) {
@@ -3926,7 +3951,7 @@ checkMaxVersion(Node *root, Node *node, InternalVersionT oldestVersion,
   bool success = true;
 
   if (node->partialKeyLen > 0) {
-    fprintf(stderr, "Root cannot have a partial key");
+    fprintf(stderr, "Root cannot have a partial key\n");
     success = false;
   }
   checkParentPointers(node, success);

Internal.h

@@ -273,6 +273,16 @@ template <class T> struct Vector {
     size_ += slice.size();
   }
 
+  // Caller must write to the returned slice
+  std::span<T> unsafePrepareAppend(int appendSize) {
+    if (size_ + appendSize > capacity) {
+      grow(std::max<int>(size_ + appendSize, capacity * 2));
+    }
+    auto result = std::span<T>(t + size_, appendSize);
+    size_ += appendSize;
+    return result;
+  }
+
   void push_back(const T &t) { append(std::span<const T>(&t, 1)); }
 
   T *begin() { return t; }

ServerBench.cpp

@@ -164,6 +164,63 @@ double toSeconds(timeval t) {
   return double(t.tv_sec) + double(t.tv_usec) * 1e-6;
 }
 
+#include <linux/perf_event.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#ifdef __linux__
+struct PerfCounter {
+  explicit PerfCounter(int event) {
+    struct perf_event_attr pe;
+
+    memset(&pe, 0, sizeof(pe));
+    pe.type = PERF_TYPE_HARDWARE;
+    pe.size = sizeof(pe);
+    pe.config = event;
+    pe.inherit = 1;
+    pe.exclude_kernel = 1;
+    pe.exclude_hv = 1;
+
+    fd = perf_event_open(&pe, 0, -1, -1, 0);
+    if (fd == -1) {
+      fprintf(stderr, "Error opening leader %llx\n", pe.config);
+      exit(EXIT_FAILURE);
+    }
+  }
+
+  int64_t total() {
+    int64_t count;
+    if (read(fd, &count, sizeof(count)) != sizeof(count)) {
+      perror("read instructions from perf");
+      abort();
+    }
+    return count;
+  }
+
+  ~PerfCounter() { close(fd); }
+
+private:
+  int fd;
+  static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid,
+                              int cpu, int group_fd, unsigned long flags) {
+    int ret;
+
+    ret = syscall(SYS_perf_event_open, hw_event, pid, cpu, group_fd, flags);
+    return ret;
+  }
+};
+#else
+struct PerfCounter {
+  explicit PerPerfCounter(int) {}
+  int64_t total() { return 0; }
+};
+#endif
+
 int main(int argc, char **argv) {
   if (argc != 3) {
     goto fail;
@@ -176,6 +233,8 @@ int main(int argc, char **argv) {
     int metricsCount;
     cs.getMetricsV1(&metrics, &metricsCount);
 
+    PerfCounter instructions{PERF_COUNT_HW_INSTRUCTIONS};
+    PerfCounter cycles{PERF_COUNT_HW_CPU_CYCLES};
     auto w = std::thread{workload, &cs};
 
     for (;;) {
@@ -203,6 +262,16 @@ int main(int argc, char **argv) {
               "transactions_total ";
       body += std::to_string(transactions.load(std::memory_order_relaxed));
       body += "\n";
+      body += "# HELP instructions_total Total number of instructions\n"
+              "# TYPE instructions_total counter\n"
+              "instructions_total ";
+      body += std::to_string(instructions.total());
+      body += "\n";
+      body += "# HELP cycles_total Total number of cycles\n"
+              "# TYPE cycles_total counter\n"
+              "cycles_total ";
+      body += std::to_string(cycles.total());
+      body += "\n";
 
       for (int i = 0; i < metricsCount; ++i) {
         body += "# HELP ";