Fix some grammar in a comment

Once we know the type, for Node3 and higher we know the first child
exists anyway

ConflictSet.cpp

@@ -345,8 +345,8 @@ struct Node3 : Node {
   // Sorted
   uint8_t index[kMaxNodes];
 
-  TaggedNodePointer children[kMaxNodes];
   InternalVersionT childMaxVersion[kMaxNodes];
+  TaggedNodePointer children[kMaxNodes];
 
   uint8_t *partialKey() {
     assert(!releaseDeferred);
@@ -960,8 +960,7 @@ private:
   NodeAllocator<Node256> node256;
 };
 
-int getNodeIndex(Node3 *self, uint8_t index) {
-  Node3 *n = (Node3 *)self;
+int getNodeIndex(Node3 *n, uint8_t index) {
   assume(n->numChildren >= 1);
   assume(n->numChildren <= 3);
   for (int i = 0; i < n->numChildren; ++i) {
@@ -972,8 +971,7 @@ int getNodeIndex(Node3 *self, uint8_t index) {
   return -1;
 }
 
-int getNodeIndexExists(Node3 *self, uint8_t index) {
-  Node3 *n = (Node3 *)self;
+int getNodeIndexExists(Node3 *n, uint8_t index) {
   assume(n->numChildren >= 1);
   assume(n->numChildren <= 3);
   for (int i = 0; i < n->numChildren; ++i) {
@@ -1274,33 +1272,32 @@ TaggedNodePointer getChild(Node *self, uint8_t index) {
 struct ChildAndMaxVersion {
   TaggedNodePointer child;
   InternalVersionT maxVersion;
+  static ChildAndMaxVersion empty() {
+    ChildAndMaxVersion result;
+    result.child = nullptr;
+    return result;
+  }
 };
 
 ChildAndMaxVersion getChildAndMaxVersion(Node0 *, uint8_t) { return {}; }
 ChildAndMaxVersion getChildAndMaxVersion(Node3 *self, uint8_t index) {
   int i = getNodeIndex(self, index);
   if (i < 0) {
-    ChildAndMaxVersion result;
-    result.child = nullptr;
-    return result;
+    return ChildAndMaxVersion::empty();
   }
   return {self->children[i], self->childMaxVersion[i]};
 }
 ChildAndMaxVersion getChildAndMaxVersion(Node16 *self, uint8_t index) {
   int i = getNodeIndex(self, index);
   if (i < 0) {
-    ChildAndMaxVersion result;
-    result.child = nullptr;
-    return result;
+    return ChildAndMaxVersion::empty();
   }
   return {self->children[i], self->childMaxVersion[i]};
 }
 ChildAndMaxVersion getChildAndMaxVersion(Node48 *self, uint8_t index) {
   int i = self->index[index];
   if (i < 0) {
-    ChildAndMaxVersion result;
-    result.child = nullptr;
-    return result;
+    return ChildAndMaxVersion::empty();
   }
   return {self->children[i], self->childMaxVersion[i]};
 }
@@ -1383,17 +1380,11 @@ TaggedNodePointer getChildGeq(Node16 *self, int child) {
 
 TaggedNodePointer getChildGeq(Node48 *self, int child) {
   int c = self->bitSet.firstSetGeq(child);
-  if (c < 0) {
-    return nullptr;
-  }
-  return self->children[self->index[c]];
+  return c < 0 ? nullptr : self->children[self->index[c]];
 }
 TaggedNodePointer getChildGeq(Node256 *self, int child) {
   int c = self->bitSet.firstSetGeq(child);
-  if (c < 0) {
-    return nullptr;
-  }
-  return self->children[c];
+  return c < 0 ? nullptr : self->children[c];
 }
 
 TaggedNodePointer getChildGeq(Node *self, int child) {
@@ -1413,22 +1404,25 @@ TaggedNodePointer getChildGeq(Node *self, int child) {
   }
 }
 
-Node *getFirstChild(Node0 *) { return nullptr; }
-Node *getFirstChild(Node3 *self) {
-  return self->numChildren == 0 ? nullptr : self->children[0];
+TaggedNodePointer getFirstChild(Node0 *) { return nullptr; }
+TaggedNodePointer getFirstChild(Node3 *self) {
+  // Improves scan performance
+  __builtin_prefetch(self->children[1]);
+  return self->children[0];
 }
-Node *getFirstChild(Node16 *self) {
-  return self->numChildren == 0 ? nullptr : self->children[0];
+TaggedNodePointer getFirstChild(Node16 *self) {
+  // Improves scan performance
+  __builtin_prefetch(self->children[1]);
+  return self->children[0];
 }
-Node *getFirstChild(Node48 *self) {
-  int index = self->index[self->bitSet.firstSetGeq(0)];
-  return index < 0 ? nullptr : self->children[index];
+TaggedNodePointer getFirstChild(Node48 *self) {
+  return self->children[self->index[self->bitSet.firstSetGeq(0)]];
 }
-Node *getFirstChild(Node256 *self) {
+TaggedNodePointer getFirstChild(Node256 *self) {
   return self->children[self->bitSet.firstSetGeq(0)];
 }
 
-Node *getFirstChild(Node *self) {
+TaggedNodePointer getFirstChild(Node *self) {
   // Only require that the node-specific overloads are covered
   // GCOVR_EXCL_START
   switch (self->getType()) {
@@ -1448,46 +1442,6 @@ Node *getFirstChild(Node *self) {
   // GCOVR_EXCL_STOP
 }
 
-// Precondition: self has a child
-TaggedNodePointer getFirstChildExists(Node3 *self) {
-  assert(self->numChildren > 0);
-  return self->children[0];
-}
-// Precondition: self has a child
-TaggedNodePointer getFirstChildExists(Node16 *self) {
-  assert(self->numChildren > 0);
-  return self->children[0];
-}
-// Precondition: self has a child
-TaggedNodePointer getFirstChildExists(Node48 *self) {
-  return self->children[self->index[self->bitSet.firstSetGeq(0)]];
-}
-// Precondition: self has a child
-TaggedNodePointer getFirstChildExists(Node256 *self) {
-  return self->children[self->bitSet.firstSetGeq(0)];
-}
-
-// Precondition: self has a child
-TaggedNodePointer getFirstChildExists(Node *self) {
-  // Only require that the node-specific overloads are covered
-  // GCOVR_EXCL_START
-  switch (self->getType()) {
-  case Type_Node0:
-    __builtin_unreachable();
-  case Type_Node3:
-    return getFirstChildExists(static_cast<Node3 *>(self));
-  case Type_Node16:
-    return getFirstChildExists(static_cast<Node16 *>(self));
-  case Type_Node48:
-    return getFirstChildExists(static_cast<Node48 *>(self));
-  case Type_Node256:
-    return getFirstChildExists(static_cast<Node256 *>(self));
-  default:
-    __builtin_unreachable();
-  }
-  // GCOVR_EXCL_STOP
-}
-
 // self must not be the root
 void maybeDecreaseCapacity(Node *&self, WriteContext *writeContext,
                            ConflictSet::Impl *impl);
@@ -1741,7 +1695,7 @@ TaggedNodePointer &getOrCreateChild(TaggedNodePointer &self, TrivialSpan &key,
 }
 
 Node *nextPhysical(Node *node) {
-  auto nextChild = getFirstChild(node);
+  Node *nextChild = getFirstChild(node);
   if (nextChild != nullptr) {
     return nextChild;
   }
@@ -1759,7 +1713,7 @@ Node *nextPhysical(Node *node) {
 }
 
 Node *nextLogical(Node *node) {
-  auto nextChild = getFirstChild(node);
+  Node *nextChild = getFirstChild(node);
   if (nextChild != nullptr) {
     node = nextChild;
     goto downLeftSpine;
@@ -1777,7 +1731,7 @@ Node *nextLogical(Node *node) {
     }
   }
 downLeftSpine:
-  for (; !node->entryPresent; node = getFirstChildExists(node)) {
+  for (; !node->entryPresent; node = getFirstChild(node)) {
   }
   return node;
 }
@@ -1947,7 +1901,7 @@ void mergeWithChild(TaggedNodePointer &self, WriteContext *writeContext,
   child->parentsIndex = self->parentsIndex;
 
   // Max versions are stored in the parent, so we need to update it now
-  // that we have a new parent. Safe we call since the root never has a partial
+  // that we have a new parent. Safe to call since the root never has a partial
   // key.
   setMaxVersion(child, std::max(childMaxVersion, writeContext->zero));
 
@@ -2825,7 +2779,7 @@ bool checkRangeStartsWith(NodeT *nTyped, TrivialSpan key, int begin, int end,
   __builtin_unreachable(); // GCOVR_EXCL_LINE
 
 downLeftSpine:
-  for (; !n->entryPresent; n = getFirstChildExists(n)) {
+  for (; !n->entryPresent; n = getFirstChild(n)) {
   }
   return n->entry.rangeVersion <= readVersion;
 }
@@ -3282,7 +3236,7 @@ PRESERVE_NONE void down_left_spine(Job *job, Context *context) {
     job->setResult(n->entry.rangeVersion <= job->readVersion);
     MUSTTAIL return complete(job, context);
   }
-  auto child = getFirstChildExists(n);
+  auto child = getFirstChild(n);
   job->n = child;
   __builtin_prefetch(job->n);
   job->continuation = downLeftSpineTable[child.getType()];
@@ -3377,7 +3331,7 @@ template <class NodeT> void iter(Job *job, Context *context) {
       job->setResult(n->entry.pointVersion <= job->readVersion);
       MUSTTAIL return complete(job, context);
     }
-    auto c = getFirstChildExists(n);
+    auto c = getFirstChild(n);
     job->n = c;
     job->continuation = downLeftSpineTable[c.getType()];
     __builtin_prefetch(job->n);
@@ -3901,7 +3855,7 @@ void left_side_down_left_spine(Job *job, Context *context) {
     }
     MUSTTAIL return done_left_side_iter(job, context);
   }
-  auto c = getFirstChildExists(n);
+  auto c = getFirstChild(n);
   job->n = c;
   job->continuation = leftSideDownLeftSpineTable[c.getType()];
   __builtin_prefetch(job->n);
@@ -4564,7 +4518,7 @@ bool checkPointRead(Node *n, const TrivialSpan key,
       if (n->entryPresent) {
         return n->entry.pointVersion <= readVersion;
       }
-      n = getFirstChildExists(n);
+      n = getFirstChild(n);
       goto downLeftSpine;
     }
 
@@ -4618,7 +4572,7 @@ bool checkPointRead(Node *n, const TrivialSpan key,
     }
   }
 downLeftSpine:
-  for (; !n->entryPresent; n = getFirstChildExists(n)) {
+  for (; !n->entryPresent; n = getFirstChild(n)) {
   }
   return n->entry.rangeVersion <= readVersion;
 }
@@ -4693,7 +4647,7 @@ bool checkPrefixRead(Node *n, const TrivialSpan key,
     }
   }
 downLeftSpine:
-  for (; !n->entryPresent; n = getFirstChildExists(n)) {
+  for (; !n->entryPresent; n = getFirstChild(n)) {
   }
   return n->entry.rangeVersion <= readVersion;
 }
@@ -4780,7 +4734,7 @@ bool checkRangeLeftSide(Node *n, TrivialSpan key, int prefixLen,
     }
   }
 downLeftSpine:
-  for (; !n->entryPresent; n = getFirstChildExists(n)) {
+  for (; !n->entryPresent; n = getFirstChild(n)) {
   }
   return n->entry.rangeVersion <= readVersion;
 }
@@ -4876,7 +4830,7 @@ backtrack:
     }
   }
 downLeftSpine:
-  for (; !n->entryPresent; n = getFirstChildExists(n)) {
+  for (; !n->entryPresent; n = getFirstChild(n)) {
   }
   return n->entry.rangeVersion <= readVersion;
 }
@@ -5649,7 +5603,7 @@ Node *firstGeqLogical(Node *n, const TrivialSpan key) {
       if (n->entryPresent) {
         return n;
       }
-      n = getFirstChildExists(n);
+      n = getFirstChild(n);
       goto downLeftSpine;
     }
 
@@ -5694,7 +5648,7 @@ Node *firstGeqLogical(Node *n, const TrivialSpan key) {
     }
   }
 downLeftSpine:
-  for (; !n->entryPresent; n = getFirstChildExists(n)) {
+  for (; !n->entryPresent; n = getFirstChild(n)) {
   }
   return n;
 }

Internal.h

@@ -368,23 +368,6 @@ template <class T, class C = std::less<T>> auto set(Arena &arena) {
   return Set<T, C>(ArenaAlloc<T>(&arena));
 }
 
-template <class T> struct MyHash;
-
-template <class T> struct MyHash<T *> {
-  size_t operator()(const T *t) const noexcept {
-    size_t result;
-    memcpy(&result, &t, sizeof(result));
-    return result;
-  }
-};
-
-template <class T>
-using HashSet =
-    std::unordered_set<T, MyHash<T>, std::equal_to<T>, ArenaAlloc<T>>;
-template <class T> auto hashSet(Arena &arena) {
-  return HashSet<T>(ArenaAlloc<T>(&arena));
-}
-
 template <class T, class U>
 bool operator==(const ArenaAlloc<T> &lhs, const ArenaAlloc<U> &rhs) {
   return lhs.arena == rhs.arena;

README.md

@@ -26,15 +26,15 @@ Compiler is `Ubuntu clang version 20.0.0 (++20241029082144+7544d3af0e28-1~exp1~2
 
 |               ns/op |                op/s |    err% |          ins/op |          cyc/op |    IPC |         bra/op |   miss% |     total | benchmark
 |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
-|               12.63 |       79,186,868.18 |    1.4% |          241.61 |           64.76 |  3.731 |          31.64 |    0.8% |      0.01 | `point reads`
-|               14.48 |       69,078,073.40 |    0.3% |          292.42 |           74.69 |  3.915 |          41.49 |    0.5% |      0.01 | `prefix reads`
-|               34.37 |       29,094,694.11 |    0.2% |          759.53 |          179.77 |  4.225 |         100.38 |    0.2% |      0.01 | `range reads`
-|               19.34 |       51,713,896.36 |    0.7% |          369.70 |          101.81 |  3.631 |          47.88 |    0.6% |      0.01 | `point writes`
-|               39.16 |       25,538,968.61 |    0.2% |          653.16 |          206.77 |  3.159 |          89.62 |    0.8% |      0.01 | `prefix writes`
-|               40.58 |       24,642,681.12 |    4.7% |          718.44 |          216.44 |  3.319 |          99.28 |    0.6% |      0.01 | `range writes`
-|               78.77 |       12,694,520.69 |    3.8% |        1,395.55 |          421.73 |  3.309 |         249.81 |    0.1% |      0.01 | `monotonic increasing point writes`
-|          287,760.50 |            3,475.11 |    0.5% |    3,929,266.50 |    1,550,225.50 |  2.535 |     639,064.00 |    0.0% |      0.01 | `worst case for radix tree`
-|              104.76 |        9,545,250.65 |    3.1% |        2,000.00 |          552.82 |  3.618 |         342.00 |    0.0% |      0.01 | `create and destroy`
+|               12.83 |       77,947,334.88 |    0.8% |          247.13 |           63.80 |  3.873 |          32.64 |    0.6% |      0.01 | `point reads`
+|               14.73 |       67,908,470.74 |    0.1% |          299.99 |           73.66 |  4.073 |          42.50 |    0.5% |      0.01 | `prefix reads`
+|               35.63 |       28,066,165.78 |    0.1% |          782.70 |          178.49 |  4.385 |         106.65 |    0.2% |      0.01 | `range reads`
+|               20.00 |       49,993,123.62 |    0.1% |          376.83 |          100.50 |  3.749 |          50.05 |    0.5% |      0.01 | `point writes`
+|               38.04 |       26,287,266.49 |    0.1% |          665.86 |          191.21 |  3.482 |         100.41 |    0.4% |      0.01 | `prefix writes`
+|               40.48 |       24,703,557.31 |    1.3% |          732.80 |          204.36 |  3.586 |         111.26 |    0.2% |      0.01 | `range writes`
+|               81.01 |       12,343,591.64 |    1.4% |        1,551.57 |          409.23 |  3.791 |         292.66 |    0.1% |      0.01 | `monotonic increasing point writes`
+|          315,672.00 |            3,167.85 |    1.7% |    4,043,066.00 |    1,590,315.00 |  2.542 |     714,828.00 |    0.1% |      0.01 | `worst case for radix tree`
+|              114.81 |        8,710,164.86 |    0.7% |        2,178.00 |          578.69 |  3.764 |         345.00 |    0.0% |      0.01 | `create and destroy`
 
 # "Real data" test
 
@@ -49,7 +49,7 @@ Check: 4.39702 seconds, 370.83 MB/s, Add: 4.50025 seconds, 124.583 MB/s, Gc rati
 ## radix tree
 
 ```
-Check: 0.987757 seconds, 1650.76 MB/s, Add: 1.24815 seconds, 449.186 MB/s, Gc ratio: 41.4675%, Peak idle memory: 2.02872e+06
+Check: 0.975666 seconds, 1728.24 MB/s, Add: 1.19751 seconds, 480.444 MB/s, Gc ratio: 36.8478%, Peak idle memory: 2.39447e+06
 ```
 
 ## hash table

ServerBench.cpp

@@ -138,6 +138,7 @@ void tupleAppend(std::string &output, int64_t value) {
 
 void tupleAppend(std::string &output, std::string_view value) {
   output.push_back('\x02');
+  if (memchr(value.data(), '\x00', value.size()) != nullptr) {
     for (auto c : value) {
       if (c == '\x00') {
         output.push_back('\x00');
@@ -146,6 +147,9 @@ void tupleAppend(std::string &output, std::string_view value) {
         output.push_back(c);
       }
     }
+  } else {
+    output.insert(output.end(), value.begin(), value.end());
+  }
   output.push_back('\x00');
 }
 
@@ -189,7 +193,7 @@ struct Transaction {
     }
     reads.push_back({{(const uint8_t *)keys[0].data(), int(keys[0].size())},
                      {(const uint8_t *)keys[1].data(), int(keys[1].size())},
-                     version - kWindowSize});
+                     version - std::min(10, kWindowSize)});
     static_assert(kNumReadKeysPerTx >= 3);
     for (int i = 2; i < kNumReadKeysPerTx; ++i) {
       reads.push_back({{(const uint8_t *)keys[i].data(), int(keys[i].size())},
@@ -412,19 +416,18 @@ int main(int argc, char **argv) {
     }
 #endif
 
-    TxQueue<std::unique_ptr<Transaction>> queue{10};
+    TxQueue<Transaction> queue{10};
 
     auto workloadThread = std::thread{[&]() {
       for (int64_t version = kWindowSize;;
            ++version, transactions.fetch_add(1, std::memory_order_relaxed)) {
-        auto tx = std::make_unique<Transaction>(version);
-        queue.push(std::move(tx));
+        queue.push(Transaction(version));
       }
     }};
 
     auto resolverThread = std::thread{[&]() {
       for (;;) {
-        auto tx = queue.pop()->get();
+        auto tx = queue.pop();
         resolver.resolve(tx->reads.data(), tx->reads.size(), tx->writes.data(),
                          tx->writes.size(), tx->version, tx->oldestVersion);
       }