Remove unnecessary branch for interleaved range writes

Do it in a simpler, more robust/type safe way

Bench.cpp

@@ -57,6 +57,7 @@ ConflictSet::ReadRange prefixRange(Arena &arena, TrivialSpan key) {
 
 void benchConflictSet() {
   ankerl::nanobench::Bench bench;
+  bench.minEpochIterations(10000);
   ConflictSet cs{0};
 
   bench.batch(kOpsPerTx);

CMakeLists.txt

@@ -69,7 +69,8 @@ if(HAS_FULL_RELRO)
 endif()
 cmake_pop_check_state()
 
-if(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64)
+if(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL
+                                              arm64)
   add_compile_options(-mbranch-protection=standard)
 else()
   add_compile_options(-fcf-protection)
@@ -101,8 +102,6 @@ option(DISABLE_TSAN "Disable TSAN" OFF)
 # https://valgrind.org/docs/manual/manual-core-adv.html#manual-core-adv.clientreq
 include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/valgrind)
 
-add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-Wno-invalid-offsetof>)
-
 if(APPLE)
   add_link_options(-Wl,-dead_strip)
 else()

Internal.h

@@ -18,7 +18,6 @@ using namespace weaselab;
 #include <span>
 #include <string>
 #include <thread>
-#include <unordered_set>
 #include <utility>
 
 #include <callgrind.h>

README.md

@@ -4,7 +4,14 @@ Intended as an alternative to FoundationDB's skip list.
 
 Hardware for all benchmarks is an AMD Ryzen 9 7900 with (2x32GB) 5600MT/s CL28-34-34-89 1.35V RAM.
 
-Compiler is `Ubuntu clang version 20.0.0 (++20241029082144+7544d3af0e28-1~exp1~20241029082307.506)`.
+```
+$ clang++ --version
+
+Ubuntu clang version 20.0.0 (++20241120082228+86734c857724-1~exp1~20241120202359.554)
+Target: x86_64-pc-linux-gnu
+Thread model: posix
+InstalledDir: /usr/lib/llvm-20/bin
+```
 
 # Microbenchmark
 
@@ -12,44 +19,45 @@ Compiler is `Ubuntu clang version 20.0.0 (++20241029082144+7544d3af0e28-1~exp1~2
 
 |               ns/op |                op/s |    err% |          ins/op |          cyc/op |    IPC |         bra/op |   miss% |     total | benchmark
 |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
-|              159.65 |        6,263,576.52 |    1.6% |        2,972.36 |          820.37 |  3.623 |         504.59 |    0.0% |      0.01 | `point reads`
-|              156.32 |        6,397,320.65 |    0.7% |        2,913.62 |          806.87 |  3.611 |         490.19 |    0.0% |      0.01 | `prefix reads`
-|              229.18 |        4,363,293.65 |    1.2% |        3,541.05 |        1,219.75 |  2.903 |         629.33 |    0.0% |      0.01 | `range reads`
-|              363.37 |        2,752,026.30 |    0.3% |        5,273.63 |        1,951.54 |  2.702 |         851.66 |    1.7% |      0.01 | `point writes`
-|              364.99 |        2,739,787.02 |    0.3% |        5,250.92 |        1,958.54 |  2.681 |         839.24 |    1.7% |      0.01 | `prefix writes`
-|              242.26 |        4,127,796.58 |    2.9% |        3,117.33 |        1,304.41 |  2.390 |         541.07 |    2.8% |      0.02 | `range writes`
-|              562.48 |        1,777,855.27 |    0.8% |        7,305.21 |        3,034.34 |  2.408 |       1,329.30 |    1.3% |      0.01 | `monotonic increasing point writes`
-|          122,688.57 |            8,150.72 |    0.7% |      798,766.00 |      666,842.00 |  1.198 |     144,584.50 |    0.1% |      0.01 | `worst case for radix tree`
-|               41.71 |       23,976,459.34 |    1.7% |          885.00 |          219.17 |  4.038 |         132.00 |    0.0% |      0.01 | `create and destroy`
+|              161.29 |        6,200,056.17 |    0.1% |        3,014.03 |          831.04 |  3.627 |         504.59 |    0.0% |      1.93 | `point reads`
+|              158.32 |        6,316,160.64 |    0.1% |        2,954.16 |          815.80 |  3.621 |         490.17 |    0.0% |      1.89 | `prefix reads`
+|              237.39 |        4,212,409.50 |    0.2% |        3,592.41 |        1,233.96 |  2.911 |         629.31 |    0.0% |      2.84 | `range reads`
+|              442.11 |        2,261,878.94 |    0.0% |        4,450.57 |        2,314.25 |  1.923 |         707.92 |    2.1% |      5.28 | `point writes`
+|              439.89 |        2,273,308.53 |    0.1% |        4,410.22 |        2,302.29 |  1.916 |         694.74 |    2.1% |      5.25 | `prefix writes`
+|              290.96 |        3,436,936.78 |    0.0% |        2,315.38 |        1,528.68 |  1.515 |         396.69 |    3.3% |      3.49 | `range writes`
+|              476.93 |        2,096,762.02 |    0.6% |        6,999.33 |        2,484.94 |  2.817 |       1,251.73 |    1.3% |      0.06 | `monotonic increasing point writes`
+|          131,736.57 |            7,590.91 |    1.1% |      807,444.50 |      704,941.71 |  1.145 |     144,584.60 |    0.9% |      0.01 | `worst case for radix tree`
+|               45.50 |       21,978,369.95 |    1.1% |          902.00 |          232.36 |  3.882 |         132.00 |    0.0% |      0.01 | `create and destroy`
 
 ## Radix tree (this implementation)
 
+
 |               ns/op |                op/s |    err% |          ins/op |          cyc/op |    IPC |         bra/op |   miss% |     total | benchmark
 |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
-|               12.83 |       77,947,334.88 |    0.8% |          247.13 |           63.80 |  3.873 |          32.64 |    0.6% |      0.01 | `point reads`
-|               14.73 |       67,908,470.74 |    0.1% |          299.99 |           73.66 |  4.073 |          42.50 |    0.5% |      0.01 | `prefix reads`
-|               35.63 |       28,066,165.78 |    0.1% |          782.70 |          178.49 |  4.385 |         106.65 |    0.2% |      0.01 | `range reads`
-|               20.00 |       49,993,123.62 |    0.1% |          376.83 |          100.50 |  3.749 |          50.05 |    0.5% |      0.01 | `point writes`
-|               38.04 |       26,287,266.49 |    0.1% |          665.86 |          191.21 |  3.482 |         100.41 |    0.4% |      0.01 | `prefix writes`
-|               40.48 |       24,703,557.31 |    1.3% |          732.80 |          204.36 |  3.586 |         111.26 |    0.2% |      0.01 | `range writes`
-|               81.01 |       12,343,591.64 |    1.4% |        1,551.57 |          409.23 |  3.791 |         292.66 |    0.1% |      0.01 | `monotonic increasing point writes`
-|          315,672.00 |            3,167.85 |    1.7% |    4,043,066.00 |    1,590,315.00 |  2.542 |     714,828.00 |    0.1% |      0.01 | `worst case for radix tree`
-|              114.81 |        8,710,164.86 |    0.7% |        2,178.00 |          578.69 |  3.764 |         345.00 |    0.0% |      0.01 | `create and destroy`
+|               12.36 |       80,885,626.43 |    0.2% |          243.56 |           63.62 |  3.828 |          31.07 |    0.6% |      0.15 | `point reads`
+|               14.18 |       70,502,196.81 |    0.1% |          297.72 |           73.13 |  4.071 |          40.31 |    0.5% |      0.17 | `prefix reads`
+|               33.44 |       29,901,623.04 |    0.1% |          767.90 |          172.42 |  4.454 |         101.32 |    0.2% |      0.40 | `range reads`
+|               19.48 |       51,342,564.70 |    0.3% |          374.45 |          100.43 |  3.728 |          48.92 |    0.5% |      0.23 | `point writes`
+|               37.46 |       26,694,471.44 |    0.1% |          672.00 |          193.14 |  3.479 |         101.28 |    0.3% |      0.45 | `prefix writes`
+|               38.78 |       25,784,784.34 |    0.0% |          738.26 |          199.93 |  3.693 |         111.59 |    0.1% |      0.47 | `range writes`
+|               76.05 |       13,148,995.74 |    0.7% |        1,450.77 |          397.16 |  3.653 |         275.72 |    0.0% |      0.01 | `monotonic increasing point writes`
+|          286,920.33 |            3,485.29 |    0.4% |    4,117,948.00 |    1,521,352.00 |  2.707 |     714,833.00 |    0.1% |      0.01 | `worst case for radix tree`
+|               95.66 |       10,453,798.72 |    0.5% |        1,986.00 |          495.04 |  4.012 |         315.00 |    0.0% |      0.01 | `create and destroy`
 
 # "Real data" test
 
-Point queries only, best of three runs. Gc ratio is the ratio of time spent doing garbage collection to time spent adding writes or doing garbage collection. Lower is better.
+Point queries only. Gc ratio is the ratio of time spent doing garbage collection to time spent adding writes or doing garbage collection. Lower is better.
 
 ## skip list
 
 ```
-Check: 4.39702 seconds, 370.83 MB/s, Add: 4.50025 seconds, 124.583 MB/s, Gc ratio: 29.1333%, Peak idle memory: 5.51852e+06
+Check: 4.53508 seconds, 371.81 MB/s, Add: 3.81222 seconds, 150.919 MB/s, Gc ratio: 33.66%, Peak idle memory: 5.61007e+06
 ```
 
 ## radix tree
 
 ```
-Check: 0.975666 seconds, 1728.24 MB/s, Add: 1.19751 seconds, 480.444 MB/s, Gc ratio: 36.8478%, Peak idle memory: 2.39447e+06
+Check: 0.957735 seconds, 1760.6 MB/s, Add: 1.19942 seconds, 479.678 MB/s, Gc ratio: 38.6069%, Peak idle memory: 2.05667e+06
 ```
 
 ## hash table
@@ -57,5 +65,6 @@ Check: 0.975666 seconds, 1728.24 MB/s, Add: 1.19751 seconds, 480.444 MB/s, Gc ra
 (The hash table implementation doesn't work on range queries, and its purpose is to provide an idea of how fast point queries can be)
 
 ```
-Check: 0.84256 seconds, 1935.23 MB/s, Add: 0.697204 seconds, 804.146 MB/s, Gc ratio: 35.4091%
+Check: 0.804598 seconds, 2095.69 MB/s, Add: 0.671221 seconds, 857.147 MB/s, Gc ratio: 35.0034%, Peak idle memory: 0
 ```
+

RealDataBench.cpp

@@ -133,10 +133,10 @@ int main(int argc, const char **argv) {
   int metricsCount;
   cs.getMetricsV1(&metrics, &metricsCount);
   for (int i = 0; i < metricsCount; ++i) {
-    printf("# HELP %s %s\n", metrics[i].name, metrics[i].help);
-    printf("# TYPE %s %s\n", metrics[i].name,
-           metrics[i].type == metrics[i].Counter ? "counter" : "gauge");
-    printf("%s %g\n", metrics[i].name, metrics[i].getValue());
+    fprintf(stderr, "# HELP %s %s\n", metrics[i].name, metrics[i].help);
+    fprintf(stderr, "# TYPE %s %s\n", metrics[i].name,
+            metrics[i].type == metrics[i].Counter ? "counter" : "gauge");
+    fprintf(stderr, "%s %g\n", metrics[i].name, metrics[i].getValue());
   }
 
   printf("Check: %g seconds, %g MB/s, Add: %g seconds, %g MB/s, Gc ratio: "

ServerBench.cpp

@@ -161,8 +161,8 @@ template <class... Ts> std::string tupleKey(const Ts &...ts) {
 
 constexpr int kTotalKeyRange = 1'000'000'000;
 constexpr int kWindowSize = 1'000'000;
-constexpr int kNumReadKeysPerTx = 10;
-constexpr int kNumWriteKeysPerTx = 5;
+constexpr int kNumReadKeysPerTx = 5;
+constexpr int kNumWriteKeysPerTx = 10;
 
 struct Transaction {
   std::vector<std::string> keys;

update-readme.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+# Disable frequency scaling
+for i in $(seq "$(nproc)") ; do sudo cat /sys/devices/system/cpu/cpu$((i-1))/cpufreq/scaling_max_freq | sudo tee /sys/devices/system/cpu/cpu$((i-1))/cpufreq/scaling_min_freq >/dev/null ; done
+
+cmake -S. -Bbuild -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang++ -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_CXX_FLAGS=-DNVALGRIND > /dev/null
+cmake --build build > /dev/null
+
+cat << 'EOF'
+A data structure for optimistic concurrency control on ranges of bitwise-lexicographically-ordered keys.
+
+Intended as an alternative to FoundationDB's skip list.
+
+Hardware for all benchmarks is an AMD Ryzen 9 7900 with (2x32GB) 5600MT/s CL28-34-34-89 1.35V RAM.
+
+```
+$ clang++ --version
+
+EOF
+
+clang++ --version
+
+cat << 'EOF'
+```
+
+# Microbenchmark
+
+## Skip list
+EOF
+
+LD_LIBRARY_PATH=build/skip_list build/conflict_set_bench
+
+cat << 'EOF'
+
+## Radix tree (this implementation)
+
+EOF
+
+LD_LIBRARY_PATH=build/radix_tree build/conflict_set_bench
+
+cat << 'EOF'
+
+# "Real data" test
+
+Point queries only. Gc ratio is the ratio of time spent doing garbage collection to time spent adding writes or doing garbage collection. Lower is better.
+
+## skip list
+
+```
+EOF
+
+LD_LIBRARY_PATH=build/skip_list build/real_data_bench ~/Downloads/quotes_2008-08.txt
+
+cat << 'EOF'
+```
+
+## radix tree
+
+```
+EOF
+
+LD_LIBRARY_PATH=build/radix_tree build/real_data_bench ~/Downloads/quotes_2008-08.txt
+
+cat << 'EOF'
+```
+
+## hash table
+
+(The hash table implementation doesn't work on range queries, and its purpose is to provide an idea of how fast point queries can be)
+
+```
+EOF
+
+LD_LIBRARY_PATH=build/hash_table build/real_data_bench ~/Downloads/quotes_2008-08.txt
+
+cat << 'EOF'
+```
+
+EOF