Remove unnecessary branch for interleaved range writes

Do it in a simpler, more robust/type safe way

Bench.cpp

@@ -57,6 +57,7 @@ ConflictSet::ReadRange prefixRange(Arena &arena, TrivialSpan key) {
 
 void benchConflictSet() {
   ankerl::nanobench::Bench bench;
+  bench.minEpochIterations(10000);
   ConflictSet cs{0};
 
   bench.batch(kOpsPerTx);

CMakeLists.txt

@@ -69,7 +69,8 @@ if(HAS_FULL_RELRO)
 endif()
 cmake_pop_check_state()
 
-if(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64)
+if(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL
+                                              arm64)
   add_compile_options(-mbranch-protection=standard)
 else()
   add_compile_options(-fcf-protection)
@@ -101,8 +102,6 @@ option(DISABLE_TSAN "Disable TSAN" OFF)
 # https://valgrind.org/docs/manual/manual-core-adv.html#manual-core-adv.clientreq
 include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/valgrind)
 
-add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-Wno-invalid-offsetof>)
-
 if(APPLE)
   add_link_options(-Wl,-dead_strip)
 else()

Internal.h

@@ -18,7 +18,6 @@ using namespace weaselab;
 #include <span>
 #include <string>
 #include <thread>
-#include <unordered_set>
 #include <utility>
 
 #include <callgrind.h>
@@ -368,23 +367,6 @@ template <class T, class C = std::less<T>> auto set(Arena &arena) {
   return Set<T, C>(ArenaAlloc<T>(&arena));
 }
 
-template <class T> struct MyHash;
-
-template <class T> struct MyHash<T *> {
-  size_t operator()(const T *t) const noexcept {
-    size_t result;
-    memcpy(&result, &t, sizeof(result));
-    return result;
-  }
-};
-
-template <class T>
-using HashSet =
-    std::unordered_set<T, MyHash<T>, std::equal_to<T>, ArenaAlloc<T>>;
-template <class T> auto hashSet(Arena &arena) {
-  return HashSet<T>(ArenaAlloc<T>(&arena));
-}
-
 template <class T, class U>
 bool operator==(const ArenaAlloc<T> &lhs, const ArenaAlloc<U> &rhs) {
   return lhs.arena == rhs.arena;

README.md

@@ -4,7 +4,14 @@ Intended as an alternative to FoundationDB's skip list.
 
 Hardware for all benchmarks is an AMD Ryzen 9 7900 with (2x32GB) 5600MT/s CL28-34-34-89 1.35V RAM.
 
-Compiler is `Ubuntu clang version 20.0.0 (++20241029082144+7544d3af0e28-1~exp1~20241029082307.506)`.
+```
+$ clang++ --version
+
+Ubuntu clang version 20.0.0 (++20241120082228+86734c857724-1~exp1~20241120202359.554)
+Target: x86_64-pc-linux-gnu
+Thread model: posix
+InstalledDir: /usr/lib/llvm-20/bin
+```
 
 # Microbenchmark
 
@@ -12,44 +19,45 @@ Compiler is `Ubuntu clang version 20.0.0 (++20241029082144+7544d3af0e28-1~exp1~2
 
 |               ns/op |                op/s |    err% |          ins/op |          cyc/op |    IPC |         bra/op |   miss% |     total | benchmark
 |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
-|              159.65 |        6,263,576.52 |    1.6% |        2,972.36 |          820.37 |  3.623 |         504.59 |    0.0% |      0.01 | `point reads`
-|              156.32 |        6,397,320.65 |    0.7% |        2,913.62 |          806.87 |  3.611 |         490.19 |    0.0% |      0.01 | `prefix reads`
-|              229.18 |        4,363,293.65 |    1.2% |        3,541.05 |        1,219.75 |  2.903 |         629.33 |    0.0% |      0.01 | `range reads`
-|              363.37 |        2,752,026.30 |    0.3% |        5,273.63 |        1,951.54 |  2.702 |         851.66 |    1.7% |      0.01 | `point writes`
-|              364.99 |        2,739,787.02 |    0.3% |        5,250.92 |        1,958.54 |  2.681 |         839.24 |    1.7% |      0.01 | `prefix writes`
-|              242.26 |        4,127,796.58 |    2.9% |        3,117.33 |        1,304.41 |  2.390 |         541.07 |    2.8% |      0.02 | `range writes`
-|              562.48 |        1,777,855.27 |    0.8% |        7,305.21 |        3,034.34 |  2.408 |       1,329.30 |    1.3% |      0.01 | `monotonic increasing point writes`
-|          122,688.57 |            8,150.72 |    0.7% |      798,766.00 |      666,842.00 |  1.198 |     144,584.50 |    0.1% |      0.01 | `worst case for radix tree`
-|               41.71 |       23,976,459.34 |    1.7% |          885.00 |          219.17 |  4.038 |         132.00 |    0.0% |      0.01 | `create and destroy`
+|              161.29 |        6,200,056.17 |    0.1% |        3,014.03 |          831.04 |  3.627 |         504.59 |    0.0% |      1.93 | `point reads`
+|              158.32 |        6,316,160.64 |    0.1% |        2,954.16 |          815.80 |  3.621 |         490.17 |    0.0% |      1.89 | `prefix reads`
+|              237.39 |        4,212,409.50 |    0.2% |        3,592.41 |        1,233.96 |  2.911 |         629.31 |    0.0% |      2.84 | `range reads`
+|              442.11 |        2,261,878.94 |    0.0% |        4,450.57 |        2,314.25 |  1.923 |         707.92 |    2.1% |      5.28 | `point writes`
+|              439.89 |        2,273,308.53 |    0.1% |        4,410.22 |        2,302.29 |  1.916 |         694.74 |    2.1% |      5.25 | `prefix writes`
+|              290.96 |        3,436,936.78 |    0.0% |        2,315.38 |        1,528.68 |  1.515 |         396.69 |    3.3% |      3.49 | `range writes`
+|              476.93 |        2,096,762.02 |    0.6% |        6,999.33 |        2,484.94 |  2.817 |       1,251.73 |    1.3% |      0.06 | `monotonic increasing point writes`
+|          131,736.57 |            7,590.91 |    1.1% |      807,444.50 |      704,941.71 |  1.145 |     144,584.60 |    0.9% |      0.01 | `worst case for radix tree`
+|               45.50 |       21,978,369.95 |    1.1% |          902.00 |          232.36 |  3.882 |         132.00 |    0.0% |      0.01 | `create and destroy`
 
 ## Radix tree (this implementation)
 
+
 |               ns/op |                op/s |    err% |          ins/op |          cyc/op |    IPC |         bra/op |   miss% |     total | benchmark
 |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
-|               12.63 |       79,186,868.18 |    1.4% |          241.61 |           64.76 |  3.731 |          31.64 |    0.8% |      0.01 | `point reads`
-|               14.48 |       69,078,073.40 |    0.3% |          292.42 |           74.69 |  3.915 |          41.49 |    0.5% |      0.01 | `prefix reads`
-|               34.37 |       29,094,694.11 |    0.2% |          759.53 |          179.77 |  4.225 |         100.38 |    0.2% |      0.01 | `range reads`
-|               19.34 |       51,713,896.36 |    0.7% |          369.70 |          101.81 |  3.631 |          47.88 |    0.6% |      0.01 | `point writes`
-|               39.16 |       25,538,968.61 |    0.2% |          653.16 |          206.77 |  3.159 |          89.62 |    0.8% |      0.01 | `prefix writes`
-|               40.58 |       24,642,681.12 |    4.7% |          718.44 |          216.44 |  3.319 |          99.28 |    0.6% |      0.01 | `range writes`
-|               78.77 |       12,694,520.69 |    3.8% |        1,395.55 |          421.73 |  3.309 |         249.81 |    0.1% |      0.01 | `monotonic increasing point writes`
-|          287,760.50 |            3,475.11 |    0.5% |    3,929,266.50 |    1,550,225.50 |  2.535 |     639,064.00 |    0.0% |      0.01 | `worst case for radix tree`
-|              104.76 |        9,545,250.65 |    3.1% |        2,000.00 |          552.82 |  3.618 |         342.00 |    0.0% |      0.01 | `create and destroy`
+|               12.36 |       80,885,626.43 |    0.2% |          243.56 |           63.62 |  3.828 |          31.07 |    0.6% |      0.15 | `point reads`
+|               14.18 |       70,502,196.81 |    0.1% |          297.72 |           73.13 |  4.071 |          40.31 |    0.5% |      0.17 | `prefix reads`
+|               33.44 |       29,901,623.04 |    0.1% |          767.90 |          172.42 |  4.454 |         101.32 |    0.2% |      0.40 | `range reads`
+|               19.48 |       51,342,564.70 |    0.3% |          374.45 |          100.43 |  3.728 |          48.92 |    0.5% |      0.23 | `point writes`
+|               37.46 |       26,694,471.44 |    0.1% |          672.00 |          193.14 |  3.479 |         101.28 |    0.3% |      0.45 | `prefix writes`
+|               38.78 |       25,784,784.34 |    0.0% |          738.26 |          199.93 |  3.693 |         111.59 |    0.1% |      0.47 | `range writes`
+|               76.05 |       13,148,995.74 |    0.7% |        1,450.77 |          397.16 |  3.653 |         275.72 |    0.0% |      0.01 | `monotonic increasing point writes`
+|          286,920.33 |            3,485.29 |    0.4% |    4,117,948.00 |    1,521,352.00 |  2.707 |     714,833.00 |    0.1% |      0.01 | `worst case for radix tree`
+|               95.66 |       10,453,798.72 |    0.5% |        1,986.00 |          495.04 |  4.012 |         315.00 |    0.0% |      0.01 | `create and destroy`
 
 # "Real data" test
 
-Point queries only, best of three runs. Gc ratio is the ratio of time spent doing garbage collection to time spent adding writes or doing garbage collection. Lower is better.
+Point queries only. Gc ratio is the ratio of time spent doing garbage collection to time spent adding writes or doing garbage collection. Lower is better.
 
 ## skip list
 
 ```
-Check: 4.39702 seconds, 370.83 MB/s, Add: 4.50025 seconds, 124.583 MB/s, Gc ratio: 29.1333%, Peak idle memory: 5.51852e+06
+Check: 4.53508 seconds, 371.81 MB/s, Add: 3.81222 seconds, 150.919 MB/s, Gc ratio: 33.66%, Peak idle memory: 5.61007e+06
 ```
 
 ## radix tree
 
 ```
-Check: 0.987757 seconds, 1650.76 MB/s, Add: 1.24815 seconds, 449.186 MB/s, Gc ratio: 41.4675%, Peak idle memory: 2.02872e+06
+Check: 0.957735 seconds, 1760.6 MB/s, Add: 1.19942 seconds, 479.678 MB/s, Gc ratio: 38.6069%, Peak idle memory: 2.05667e+06
 ```
 
 ## hash table
@@ -57,5 +65,6 @@ Check: 0.987757 seconds, 1650.76 MB/s, Add: 1.24815 seconds, 449.186 MB/s, Gc ra
 (The hash table implementation doesn't work on range queries, and its purpose is to provide an idea of how fast point queries can be)
 
 ```
-Check: 0.84256 seconds, 1935.23 MB/s, Add: 0.697204 seconds, 804.146 MB/s, Gc ratio: 35.4091%
+Check: 0.804598 seconds, 2095.69 MB/s, Add: 0.671221 seconds, 857.147 MB/s, Gc ratio: 35.0034%, Peak idle memory: 0
 ```
+

RealDataBench.cpp

@@ -133,10 +133,10 @@ int main(int argc, const char **argv) {
   int metricsCount;
   cs.getMetricsV1(&metrics, &metricsCount);
   for (int i = 0; i < metricsCount; ++i) {
-    printf("# HELP %s %s\n", metrics[i].name, metrics[i].help);
-    printf("# TYPE %s %s\n", metrics[i].name,
-           metrics[i].type == metrics[i].Counter ? "counter" : "gauge");
-    printf("%s %g\n", metrics[i].name, metrics[i].getValue());
+    fprintf(stderr, "# HELP %s %s\n", metrics[i].name, metrics[i].help);
+    fprintf(stderr, "# TYPE %s %s\n", metrics[i].name,
+            metrics[i].type == metrics[i].Counter ? "counter" : "gauge");
+    fprintf(stderr, "%s %g\n", metrics[i].name, metrics[i].getValue());
   }
 
   printf("Check: %g seconds, %g MB/s, Add: %g seconds, %g MB/s, Gc ratio: "

ServerBench.cpp

@@ -138,13 +138,17 @@ void tupleAppend(std::string &output, int64_t value) {
 
 void tupleAppend(std::string &output, std::string_view value) {
   output.push_back('\x02');
-  for (auto c : value) {
-    if (c == '\x00') {
-      output.push_back('\x00');
-      output.push_back('\xff');
-    } else {
-      output.push_back(c);
+  if (memchr(value.data(), '\x00', value.size()) != nullptr) {
+    for (auto c : value) {
+      if (c == '\x00') {
+        output.push_back('\x00');
+        output.push_back('\xff');
+      } else {
+        output.push_back(c);
+      }
     }
+  } else {
+    output.insert(output.end(), value.begin(), value.end());
   }
   output.push_back('\x00');
 }
@@ -157,8 +161,8 @@ template <class... Ts> std::string tupleKey(const Ts &...ts) {
 
 constexpr int kTotalKeyRange = 1'000'000'000;
 constexpr int kWindowSize = 1'000'000;
-constexpr int kNumReadKeysPerTx = 10;
-constexpr int kNumWriteKeysPerTx = 5;
+constexpr int kNumReadKeysPerTx = 5;
+constexpr int kNumWriteKeysPerTx = 10;
 
 struct Transaction {
   std::vector<std::string> keys;
@@ -189,7 +193,7 @@ struct Transaction {
     }
     reads.push_back({{(const uint8_t *)keys[0].data(), int(keys[0].size())},
                      {(const uint8_t *)keys[1].data(), int(keys[1].size())},
-                     version - kWindowSize});
+                     version - std::min(10, kWindowSize)});
     static_assert(kNumReadKeysPerTx >= 3);
     for (int i = 2; i < kNumReadKeysPerTx; ++i) {
       reads.push_back({{(const uint8_t *)keys[i].data(), int(keys[i].size())},
@@ -412,19 +416,18 @@ int main(int argc, char **argv) {
     }
 #endif
 
-    TxQueue<std::unique_ptr<Transaction>> queue{10};
+    TxQueue<Transaction> queue{10};
 
     auto workloadThread = std::thread{[&]() {
       for (int64_t version = kWindowSize;;
            ++version, transactions.fetch_add(1, std::memory_order_relaxed)) {
-        auto tx = std::make_unique<Transaction>(version);
-        queue.push(std::move(tx));
+        queue.push(Transaction(version));
       }
     }};
 
     auto resolverThread = std::thread{[&]() {
       for (;;) {
-        auto tx = queue.pop()->get();
+        auto tx = queue.pop();
         resolver.resolve(tx->reads.data(), tx->reads.size(), tx->writes.data(),
                          tx->writes.size(), tx->version, tx->oldestVersion);
       }

test_conflict_set.py

@@ -1,3 +1,4 @@
+import struct
 from conflict_set import *
 
 
@@ -164,6 +165,16 @@ def test_fixup_256():
         cs.check(read(0, bytes([1]), bytes([2])))
 
 
+def test_large_removal_buffer():
+    with DebugConflictSet() as cs:
+        for i in range(1000):
+            # create extra gc work
+            for j in range(100):
+                cs.addWrites(1000 + i)
+            cs.addWrites(1000 + i, write(struct.pack(">l", i) + bytes([0] * 100000)))
+            cs.setOldestVersion(i)
+
+
 if __name__ == "__main__":
     # budget "pytest" for ctest integration without pulling in a dependency. You can of course still use pytest in local development.
     import argparse

update-readme.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+# Disable frequency scaling
+for i in $(seq "$(nproc)") ; do sudo cat /sys/devices/system/cpu/cpu$((i-1))/cpufreq/scaling_max_freq | sudo tee /sys/devices/system/cpu/cpu$((i-1))/cpufreq/scaling_min_freq >/dev/null ; done
+
+cmake -S. -Bbuild -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang++ -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_CXX_FLAGS=-DNVALGRIND > /dev/null
+cmake --build build > /dev/null
+
+cat << 'EOF'
+A data structure for optimistic concurrency control on ranges of bitwise-lexicographically-ordered keys.
+
+Intended as an alternative to FoundationDB's skip list.
+
+Hardware for all benchmarks is an AMD Ryzen 9 7900 with (2x32GB) 5600MT/s CL28-34-34-89 1.35V RAM.
+
+```
+$ clang++ --version
+
+EOF
+
+clang++ --version
+
+cat << 'EOF'
+```
+
+# Microbenchmark
+
+## Skip list
+EOF
+
+LD_LIBRARY_PATH=build/skip_list build/conflict_set_bench
+
+cat << 'EOF'
+
+## Radix tree (this implementation)
+
+EOF
+
+LD_LIBRARY_PATH=build/radix_tree build/conflict_set_bench
+
+cat << 'EOF'
+
+# "Real data" test
+
+Point queries only. Gc ratio is the ratio of time spent doing garbage collection to time spent adding writes or doing garbage collection. Lower is better.
+
+## skip list
+
+```
+EOF
+
+LD_LIBRARY_PATH=build/skip_list build/real_data_bench ~/Downloads/quotes_2008-08.txt
+
+cat << 'EOF'
+```
+
+## radix tree
+
+```
+EOF
+
+LD_LIBRARY_PATH=build/radix_tree build/real_data_bench ~/Downloads/quotes_2008-08.txt
+
+cat << 'EOF'
+```
+
+## hash table
+
+(The hash table implementation doesn't work on range queries, and its purpose is to provide an idea of how fast point queries can be)
+
+```
+EOF
+
+LD_LIBRARY_PATH=build/hash_table build/real_data_bench ~/Downloads/quotes_2008-08.txt
+
+cat << 'EOF'
+```
+
+EOF