Add cycles_total

If numChildren + entryPresent is enough, we don't have to copy even if
it would fit in a smaller node.

If we have to copy, we might as well use the smallest acceptable node
type.

ConflictSet.cpp

@@ -800,7 +800,6 @@ struct WriteContext {
     int64_t nodes_allocated;
     int64_t nodes_released;
     int64_t point_writes;
-    int64_t prefix_writes;
     int64_t range_writes;
     int64_t write_bytes;
   } accum;
@@ -1605,6 +1604,46 @@ void rezero(Node *n, InternalVersionT z) {
   }
 }
 
+void mergeWithChild(Node *&self, WriteContext *tls, ConflictSet::Impl *impl,
+                    Node *&dontInvalidate, Node3 *self3) {
+  assert(!self3->entryPresent);
+  auto *child = self3->children[0];
+  int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen;
+
+  if (minCapacity > child->getCapacity()) {
+    const bool update = child == dontInvalidate;
+    freeAndMakeCapacityAtLeast(child, minCapacity, tls, impl, true);
+    if (update) {
+      dontInvalidate = child;
+    }
+  }
+
+  // Merge partial key with child
+#if DEBUG_VERBOSE && !defined(NDEBUG)
+  fprintf(stderr, "Merge %s into %s\n", getSearchPathPrintable(self).c_str(),
+          getSearchPathPrintable(child).c_str());
+#endif
+
+  InternalVersionT childMaxVersion = self3->childMaxVersion[0];
+
+  // Construct new partial key for child
+  memmove(child->partialKey() + self3->partialKeyLen + 1, child->partialKey(),
+          child->partialKeyLen);
+  memcpy(child->partialKey(), self3->partialKey(), self->partialKeyLen);
+  child->partialKey()[self3->partialKeyLen] = self3->index[0];
+  child->partialKeyLen += 1 + self3->partialKeyLen;
+
+  child->parent = self->parent;
+  child->parentsIndex = self->parentsIndex;
+
+  // Max versions are stored in the parent, so we need to update it now
+  // that we have a new parent.
+  setMaxVersion(child, impl, std::max(childMaxVersion, tls->zero));
+
+  self = child;
+  tls->release(self3);
+}
+
 void maybeDownsize(Node *self, WriteContext *tls, ConflictSet::Impl *impl,
                    Node *&dontInvalidate) {
 
@@ -1624,45 +1663,7 @@ void maybeDownsize(Node *self, WriteContext *tls, ConflictSet::Impl *impl,
       getInTree(self, impl) = newSelf;
       tls->release(self3);
     } else if (self->numChildren == 1 && !self->entryPresent) {
-      auto *child = self3->children[0];
-      int minCapacity = self3->partialKeyLen + 1 + child->partialKeyLen;
-
-      if (minCapacity > child->getCapacity()) {
-        const bool update = child == dontInvalidate;
-        freeAndMakeCapacityAtLeast(child, minCapacity, tls, impl, true);
-        if (update) {
-          dontInvalidate = child;
-        }
-      }
-
-      // Merge partial key with child
-#if DEBUG_VERBOSE && !defined(NDEBUG)
-      fprintf(stderr, "Merge %s into %s\n",
-              getSearchPathPrintable(self).c_str(),
-              getSearchPathPrintable(child).c_str());
-#endif
-
-      InternalVersionT childMaxVersion = maxVersion(child, impl);
-
-      // Construct new partial key for child
-      memmove(child->partialKey() + self3->partialKeyLen + 1,
-              child->partialKey(), child->partialKeyLen);
-      memcpy(child->partialKey(), self3->partialKey(), self->partialKeyLen);
-      child->partialKey()[self3->partialKeyLen] = self3->index[0];
-      child->partialKeyLen += 1 + self3->partialKeyLen;
-
-      child->parent = self->parent;
-      child->parentsIndex = self->parentsIndex;
-
-      // Max versions are stored in the parent, so we need to update it now
-      // that we have a new parent.
-      setMaxVersion(child, impl, childMaxVersion);
-      if (child->parent) {
-        rezero(child->parent, tls->zero);
-      }
-
-      getInTree(self, impl) = child;
-      tls->release(self3);
+      mergeWithChild(getInTree(self, impl), tls, impl, dontInvalidate, self3);
     }
   } break;
   case Type_Node16:
@@ -1828,7 +1829,7 @@ void eraseBetween(Node **inTree, Node16 *n, int begin, int end,
     n->index[i] = n->index[i + count];
   }
 
-  if (n->numChildren > Node3::kMaxNodes) {
+  if (n->numChildren + n->entryPresent >= kMinChildrenNode16) {
     // nop
   } else if (n->numChildren > 0) {
     auto *newNode = tls->allocate<Node3>(n->partialKeyLen);
@@ -1868,7 +1869,7 @@ void eraseBetween(Node **inTree, Node48 *n, int begin, int end,
     --n->numChildren;
   }
 
-  if (n->numChildren > Node16::kMaxNodes) {
+  if (n->numChildren + n->entryPresent >= kMinChildrenNode48) {
     // nop
   } else if (n->numChildren > Node3::kMaxNodes) {
     auto *newNode = tls->allocate<Node16>(n->partialKeyLen);
@@ -1897,7 +1898,7 @@ void eraseBetween(Node **inTree, Node256 *n, int begin, int end,
     n->bitSet.reset(i);
     --n->numChildren;
   }
-  if (n->numChildren > Node48::kMaxNodes) {
+  if (n->numChildren + n->entryPresent >= kMinChildrenNode256) {
     // nop
   } else if (n->numChildren > Node16::kMaxNodes) {
     auto *newNode = tls->allocate<Node48>(n->partialKeyLen);
@@ -1923,7 +1924,9 @@ void eraseBetween(Node **inTree, Node256 *n, int begin, int end,
 }
 
 // Erase all nodes with a search path starting with n + [child],
-// where child in [begin, end).
+// where child in [begin, end). To avoid the need to propagate erases up the
+// search path, the caller must ensure that the result has at least one child or
+// has entryPresent.
 void eraseBetween(Node *&n, int begin, int end, WriteContext *tls) {
 #if DEBUG_VERBOSE && !defined(NDEBUG)
   fprintf(stderr, "eraseBetween: %s + [%d,%d)\n",
@@ -1947,6 +1950,7 @@ void eraseBetween(Node *&n, int begin, int end, WriteContext *tls) {
   default:                   // GCOVR_EXCL_LINE
     __builtin_unreachable(); // GCOVR_EXCL_LINE
   }
+  assert(n->numChildren > 0 || n->entryPresent);
 }
 
 // Precondition: self is not the root. May invalidate nodes along the search
@@ -3188,55 +3192,6 @@ void fixupMaxVersion(Node *node, ConflictSet::Impl *impl, WriteContext *tls) {
   setMaxVersion(node, impl, max);
 }
 
-void addPrefixWrite(Node *&root, std::span<const uint8_t> begin,
-                    std::span<const uint8_t> end, InternalVersionT writeVersion,
-                    WriteContext *tls, ConflictSet::Impl *impl) {
-  ++tls->accum.prefix_writes;
-  int lcp = begin.size() - 1;
-
-  Node **useAsRoot =
-      insert(&root, begin.subspan(0, lcp), writeVersion, tls, impl);
-
-  auto *beginNode =
-      *insert(useAsRoot, begin.subspan(lcp, 1), writeVersion, tls, impl);
-
-  const bool insertedBegin = !beginNode->entryPresent;
-
-  addKey(beginNode);
-  beginNode->entryPresent = true;
-
-  if (insertedBegin) {
-    ++tls->accum.entries_inserted;
-    auto *p = nextLogical(beginNode);
-    beginNode->entry.rangeVersion =
-        p == nullptr ? tls->zero : std::max(p->entry.rangeVersion, tls->zero);
-    beginNode->entry.pointVersion = writeVersion;
-  }
-  assert(writeVersion >= beginNode->entry.pointVersion);
-  beginNode->entry.pointVersion = writeVersion;
-
-  auto *endNode =
-      *insert(useAsRoot, end.subspan(lcp, 1), writeVersion, tls, impl);
-
-  const bool insertedEnd = !endNode->entryPresent;
-
-  addKey(endNode);
-  endNode->entryPresent = true;
-
-  if (insertedEnd) {
-    ++tls->accum.entries_inserted;
-    auto *p = nextLogical(endNode);
-    endNode->entry.pointVersion =
-        p == nullptr ? tls->zero : std::max(p->entry.rangeVersion, tls->zero);
-  }
-  endNode->entry.rangeVersion = writeVersion;
-
-  eraseBetween(getInTree(beginNode, impl), 0, 256, tls);
-
-  // Inserting end trashed endNode's maxVersion. Fix that
-  fixupMaxVersion(endNode, impl, tls);
-}
-
 void addWriteRange(Node *&root, std::span<const uint8_t> begin,
                    std::span<const uint8_t> end, InternalVersionT writeVersion,
                    WriteContext *tls, ConflictSet::Impl *impl) {
@@ -3247,68 +3202,98 @@ void addWriteRange(Node *&root, std::span<const uint8_t> begin,
       end.back() == 0) {
     return addPointWrite(root, begin, writeVersion, tls, impl);
   }
-  if (lcp == int(begin.size() - 1) && end.size() == begin.size() &&
-      int(begin.back()) + 1 == int(end.back())) {
-    return addPrefixWrite(root, begin, end, writeVersion, tls, impl);
-  }
   ++tls->accum.range_writes;
   const bool beginIsPrefix = lcp == int(begin.size());
 
   Node **useAsRoot =
       insert(&root, begin.subspan(0, lcp), writeVersion, tls, impl);
+  begin = begin.subspan(lcp, begin.size() - lcp);
+  end = end.subspan(lcp, end.size() - lcp);
 
-  int consumed = lcp;
-
-  begin = begin.subspan(consumed, begin.size() - consumed);
-  end = end.subspan(consumed, end.size() - consumed);
-
-  auto *beginNode = *insert(useAsRoot, begin, writeVersion, tls, impl);
-
-  const bool insertedBegin = !beginNode->entryPresent;
-
-  addKey(beginNode);
-  beginNode->entryPresent = true;
-
-  if (insertedBegin) {
-    ++tls->accum.entries_inserted;
-    auto *p = nextLogical(beginNode);
-    beginNode->entry.rangeVersion =
-        p == nullptr ? tls->zero : std::max(p->entry.rangeVersion, tls->zero);
+  if (beginIsPrefix) {
+    auto *beginNode = *useAsRoot;
+    addKey(beginNode);
+    if (!beginNode->entryPresent) {
+      ++tls->accum.entries_inserted;
+      auto *p = nextLogical(beginNode);
+      beginNode->entry.rangeVersion =
+          p == nullptr ? tls->zero : std::max(p->entry.rangeVersion, tls->zero);
+      beginNode->entryPresent = true;
+    }
     beginNode->entry.pointVersion = writeVersion;
-  }
-  assert(writeVersion >= beginNode->entry.pointVersion);
-  beginNode->entry.pointVersion = writeVersion;
 
-  auto *endNode = *insert(useAsRoot, end, writeVersion, tls, impl);
-
-  const bool insertedEnd = !endNode->entryPresent;
-
-  addKey(endNode);
-  endNode->entryPresent = true;
-
-  if (insertedEnd) {
-    ++tls->accum.entries_inserted;
-    auto *p = nextLogical(endNode);
-    endNode->entry.pointVersion =
-        p == nullptr ? tls->zero : std::max(p->entry.rangeVersion, tls->zero);
-    if (beginIsPrefix) {
+    auto *endNode = *insert(useAsRoot, end, writeVersion, tls, impl);
+    addKey(endNode);
+    if (!endNode->entryPresent) {
+      ++tls->accum.entries_inserted;
+      auto *p = nextLogical(endNode);
+      endNode->entry.pointVersion =
+          p == nullptr ? tls->zero : std::max(p->entry.rangeVersion, tls->zero);
       // beginNode may have been invalidated when inserting end
       beginNode = *useAsRoot;
-      assert(beginNode->entryPresent);
+      endNode->entryPresent = true;
     }
-  }
-  endNode->entry.rangeVersion = writeVersion;
+    endNode->entry.rangeVersion = writeVersion;
+
+    for (beginNode = nextLogical(beginNode); beginNode != endNode;
+         beginNode = erase(beginNode, tls, impl, /*logical*/ true, endNode)) {
+    }
+
+    // Inserting end trashed endNode's maxVersion. Fix that
+    fixupMaxVersion(endNode, impl, tls);
+  } else /*!beginIsPrefix*/ {
+    auto *beginNode = *insert(useAsRoot, begin, writeVersion, tls, impl);
+    addKey(beginNode);
+    if (!beginNode->entryPresent) {
+      ++tls->accum.entries_inserted;
+      auto *p = nextLogical(beginNode);
+      beginNode->entry.rangeVersion =
+          p == nullptr ? tls->zero : std::max(p->entry.rangeVersion, tls->zero);
+      beginNode->entryPresent = true;
+    }
+    beginNode->entry.pointVersion = writeVersion;
+
+    auto *endNode = *insert(useAsRoot, end, writeVersion, tls, impl);
+    addKey(endNode);
+    if (!endNode->entryPresent) {
+      ++tls->accum.entries_inserted;
+      auto *p = nextLogical(endNode);
+      endNode->entry.pointVersion =
+          p == nullptr ? tls->zero : std::max(p->entry.rangeVersion, tls->zero);
+      endNode->entryPresent = true;
+    }
+    endNode->entry.rangeVersion = writeVersion;
 
-  if (!beginIsPrefix) {
     eraseBetween(*useAsRoot, begin[0] + 1, end[0], tls);
-  }
 
-  for (beginNode = nextLogical(beginNode); beginNode != endNode;
-       beginNode = erase(beginNode, tls, impl, /*logical*/ true, endNode)) {
-  }
+    // // Erase along left spine
+    // for (auto [n, key] = std::make_tuple(useAsRoot, begin);;) {
+    //   auto before = key;
+    //   auto **child = &getOrCreateChild(*n, key, writeVersion, tls);
+    //   if (key.size() > 0) {
+    //     eraseBetween(*child, int(key[0]) + 1, 256, tls);
+    //     if ((*child)->numChildren == 1 && !(*child)->entryPresent) {
+    //       Node *dummy = nullptr;
+    //       mergeWithChild(*child, tls, impl, dummy,
+    //                      static_cast<Node3 *>(*child));
+    //       key = before;
+    //       continue;
+    //     }
+    //   } else {
+    //     eraseBetween(*child, 0, 256, tls);
+    //     beginNode = *child;
+    //     break;
+    //   }
+    //   n = child;
+    // }
 
-  // Inserting end trashed endNode's maxVersion. Fix that
-  fixupMaxVersion(endNode, impl, tls);
+    for (beginNode = nextLogical(beginNode); beginNode != endNode;
+         beginNode = erase(beginNode, tls, impl, /*logical*/ true, endNode)) {
+    }
+
+    // Inserting end trashed endNode's maxVersion. Fix that
+    fixupMaxVersion(endNode, impl, tls);
+  }
 }
 
 Node *firstGeqPhysical(Node *n, const std::span<const uint8_t> key) {
@@ -3457,7 +3442,6 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
 
     memory_bytes.set(totalBytes);
     point_writes_total.add(tls.accum.point_writes);
-    prefix_writes_total.add(tls.accum.prefix_writes);
     range_writes_total.add(tls.accum.range_writes);
     nodes_allocated_total.add(tls.accum.nodes_allocated);
     nodes_released_total.add(tls.accum.nodes_released);
@@ -3656,7 +3640,6 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
           "Total number of checks where the result is \"too old\"");
   COUNTER(check_bytes_total, "Total number of key bytes checked");
   COUNTER(point_writes_total, "Total number of point writes");
-  COUNTER(prefix_writes_total, "Total number of prefix writes");
   COUNTER(range_writes_total, "Total number of range writes");
   GAUGE(memory_bytes, "Total number of bytes in use");
   COUNTER(nodes_allocated_total,
@@ -3764,6 +3747,7 @@ InternalVersionT exchangeMaxVersion(Node *n, InternalVersionT newMax) {
 }
 
 void setMaxVersion(Node *n, ConflictSet::Impl *impl, InternalVersionT newMax) {
+  assert(newMax >= InternalVersionT::zero);
   int index = n->parentsIndex;
   n = n->parent;
   if (n == nullptr) {
@@ -4274,7 +4258,7 @@ checkMaxVersion(Node *root, Node *node, InternalVersionT oldestVersion,
   bool success = true;
 
   if (node->partialKeyLen > 0) {
-    fprintf(stderr, "Root cannot have a partial key");
+    fprintf(stderr, "Root cannot have a partial key\n");
     success = false;
   }
   checkParentPointers(node, success);

ServerBench.cpp

@@ -164,6 +164,63 @@ double toSeconds(timeval t) {
   return double(t.tv_sec) + double(t.tv_usec) * 1e-6;
 }
 
+#include <linux/perf_event.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#ifdef __linux__
+struct PerfCounter {
+  explicit PerfCounter(int event) {
+    struct perf_event_attr pe;
+
+    memset(&pe, 0, sizeof(pe));
+    pe.type = PERF_TYPE_HARDWARE;
+    pe.size = sizeof(pe);
+    pe.config = event;
+    pe.inherit = 1;
+    pe.exclude_kernel = 1;
+    pe.exclude_hv = 1;
+
+    fd = perf_event_open(&pe, 0, -1, -1, 0);
+    if (fd == -1) {
+      fprintf(stderr, "Error opening leader %llx\n", pe.config);
+      exit(EXIT_FAILURE);
+    }
+  }
+
+  int64_t total() {
+    int64_t count;
+    if (read(fd, &count, sizeof(count)) != sizeof(count)) {
+      perror("read instructions from perf");
+      abort();
+    }
+    return count;
+  }
+
+  ~PerfCounter() { close(fd); }
+
+private:
+  int fd;
+  static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid,
+                              int cpu, int group_fd, unsigned long flags) {
+    int ret;
+
+    ret = syscall(SYS_perf_event_open, hw_event, pid, cpu, group_fd, flags);
+    return ret;
+  }
+};
+#else
+struct PerfCounter {
+  explicit PerPerfCounter(int) {}
+  int64_t total() { return 0; }
+};
+#endif
+
 int main(int argc, char **argv) {
   if (argc != 3) {
     goto fail;
@@ -176,6 +233,8 @@ int main(int argc, char **argv) {
     int metricsCount;
     cs.getMetricsV1(&metrics, &metricsCount);
 
+    PerfCounter instructions{PERF_COUNT_HW_INSTRUCTIONS};
+    PerfCounter cycles{PERF_COUNT_HW_CPU_CYCLES};
     auto w = std::thread{workload, &cs};
 
     for (;;) {
@@ -203,6 +262,16 @@ int main(int argc, char **argv) {
               "transactions_total ";
       body += std::to_string(transactions.load(std::memory_order_relaxed));
       body += "\n";
+      body += "# HELP instructions_total Total number of instructions\n"
+              "# TYPE instructions_total counter\n"
+              "instructions_total ";
+      body += std::to_string(instructions.total());
+      body += "\n";
+      body += "# HELP cycles_total Total number of cycles\n"
+              "# TYPE cycles_total counter\n"
+              "cycles_total ";
+      body += std::to_string(cycles.total());
+      body += "\n";
 
       for (int i = 0; i < metricsCount; ++i) {
         body += "# HELP ";