From 3b4271dac1703a6a8d33220ab25fe42e772eca15 Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Tue, 27 Oct 2026 13:44:11 -0400 Subject: [PATCH 1/5] Update readme --- README.md | 42 +++++++++++++++++++++--------------------- update-readme.sh | 3 ++- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 0d4b93c..54073d9 100644 --- a/README.md +++ b/README.md @@ -19,30 +19,30 @@ InstalledDir: /usr/lib/llvm-21/bin | ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:---------- -| 164.29 | 6,086,873.38 | 0.0% | 3,107.03 | 604.19 | 5.142 | 558.59 | 0.0% | 1.96 | `point reads` -| 161.05 | 6,209,395.38 | 0.1% | 3,036.76 | 592.21 | 5.128 | 539.35 | 0.0% | 1.93 | `prefix reads` -| 239.55 | 4,174,539.38 | 0.1% | 3,722.71 | 880.68 | 4.227 | 692.00 | 0.0% | 2.86 | `range reads` -| 354.75 | 2,818,919.14 | 0.7% | 4,523.64 | 1,304.75 | 3.467 | 720.22 | 2.0% | 4.23 | `point writes` -| 345.32 | 2,895,878.47 | 0.1% | 4,484.57 | 1,270.31 | 3.530 | 705.00 | 1.8% | 4.12 | `prefix writes` -| 193.48 | 5,168,547.42 | 0.1% | 2,224.10 | 711.72 | 3.125 | 377.17 | 3.3% | 2.32 | `range writes` -| 404.89 | 2,469,777.50 | 2.4% | 6,855.96 | 1,489.70 | 4.602 | 1,227.82 | 1.3% | 0.05 | `monotonic increasing point writes` -| 134,231.80 | 7,449.80 | 1.9% | 812,045.25 | 495,770.40 | 1.638 | 151,246.50 | 0.9% | 0.01 | `worst case for radix tree` -| 37.80 | 26,454,311.17 | 0.4% | 701.00 | 139.14 | 5.038 | 102.00 | 0.0% | 0.01 | `create and destroy` +| 165.35 | 6,047,750.85 | 0.1% | 3,107.03 | 608.25 | 5.108 | 558.59 | 0.0% | 1.98 | `point reads` +| 161.75 | 6,182,478.30 | 0.2% | 3,036.76 | 594.75 | 5.106 | 539.35 | 0.0% | 1.94 | `prefix reads` +| 250.88 | 3,985,953.69 | 0.4% | 3,722.71 | 922.20 | 4.037 | 692.00 | 0.0% | 2.98 | `range reads` +| 358.34 | 2,790,617.48 | 0.2% | 4,523.64 | 1,317.11 | 3.435 | 720.22 | 2.1% | 4.28 | `point writes` +| 349.16 | 2,864,048.29 | 0.9% | 4,484.57 | 1,279.36 | 3.505 | 705.00 | 1.9% | 4.16 | `prefix writes` +| 192.06 | 5,206,715.53 | 0.0% | 2,224.10 | 706.55 | 3.148 | 377.17 | 3.3% | 2.30 | `range writes` +| 408.59 | 2,447,415.85 | 0.9% | 6,855.96 | 1,503.60 | 4.560 | 1,227.82 | 1.4% | 0.05 | `monotonic increasing point writes` +| 136,536.00 | 7,324.08 | 2.0% | 812,045.00 | 501,448.67 | 1.619 | 151,246.43 | 0.9% | 0.01 | `worst case for radix tree` +| 37.84 | 26,426,456.12 | 0.8% | 701.00 | 139.27 | 5.034 | 102.00 | 0.0% | 0.01 | `create and destroy` ## Radix tree (this implementation) | ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:---------- -| 12.89 | 77,565,115.56 | 0.1% | 244.55 | 47.43 | 5.155 | 34.21 | 0.6% | 0.15 | `point reads` -| 15.11 | 66,162,047.76 | 0.1% | 297.79 | 55.60 | 5.356 | 43.23 | 0.4% | 0.18 | `prefix reads` -| 36.29 | 27,559,358.29 | 0.1% | 783.16 | 133.44 | 5.869 | 109.52 | 0.2% | 0.43 | `range reads` -| 20.53 | 48,719,405.55 | 0.1% | 381.81 | 75.51 | 5.057 | 51.04 | 0.5% | 0.25 | `point writes` -| 39.37 | 25,402,042.40 | 0.1% | 685.00 | 144.83 | 4.730 | 106.72 | 0.3% | 0.47 | `prefix writes` -| 43.78 | 22,843,841.63 | 0.1% | 800.40 | 161.06 | 4.970 | 127.36 | 0.1% | 0.53 | `range writes` -| 78.37 | 12,760,008.75 | 1.0% | 1,452.61 | 288.24 | 5.040 | 278.69 | 0.1% | 0.01 | `monotonic increasing point writes` -| 322,885.50 | 3,097.07 | 1.5% | 4,362,382.00 | 1,183,852.00 | 3.685 | 765,301.00 | 0.1% | 0.01 | `worst case for radix tree` -| 99.99 | 10,000,718.79 | 0.4% | 1,775.00 | 367.93 | 4.824 | 288.00 | 0.0% | 0.01 | `create and destroy` +| 12.44 | 80,368,377.55 | 0.2% | 237.48 | 45.74 | 5.192 | 31.23 | 0.8% | 0.15 | `point reads` +| 14.59 | 68,537,989.62 | 0.6% | 291.66 | 53.69 | 5.432 | 40.16 | 0.5% | 0.17 | `prefix reads` +| 33.91 | 29,487,389.82 | 0.1% | 759.22 | 124.80 | 6.084 | 101.27 | 0.2% | 0.41 | `range reads` +| 19.54 | 51,172,096.42 | 0.0% | 369.51 | 71.92 | 5.138 | 47.92 | 0.5% | 0.23 | `point writes` +| 37.87 | 26,402,961.91 | 0.1% | 662.02 | 139.34 | 4.751 | 100.26 | 0.4% | 0.45 | `prefix writes` +| 39.11 | 25,569,264.79 | 0.1% | 725.90 | 143.87 | 5.046 | 111.09 | 0.1% | 0.48 | `range writes` +| 77.70 | 12,869,910.98 | 0.5% | 1,430.41 | 286.41 | 4.994 | 271.78 | 0.1% | 0.01 | `monotonic increasing point writes` +| 319,309.00 | 3,131.76 | 0.5% | 4,181,645.50 | 1,178,098.50 | 3.549 | 728,511.00 | 0.1% | 0.01 | `worst case for radix tree` +| 97.89 | 10,215,764.97 | 0.3% | 1,775.00 | 360.36 | 4.926 | 288.00 | 0.0% | 0.01 | `create and destroy` # "Real data" test @@ -51,13 +51,13 @@ Point queries only. Gc ratio is the ratio of time spent doing garbage collection ## skip list ``` -Check: 4.62967 seconds, 352.195 MB/s, Add: 3.34177 seconds, 167.771 MB/s, Gc ratio: 37.9399%, Peak idle memory: 5.51852e+06 +Check: 4.7186 seconds, 345.557 MB/s, Add: 3.3305 seconds, 168.339 MB/s, Gc ratio: 38.6509%, Peak idle memory: 5.51852e+06 ``` ## radix tree ``` -Check: 1.00477 seconds, 1622.8 MB/s, Add: 1.21142 seconds, 462.808 MB/s, Gc ratio: 39.4716%, Peak idle memory: 2.0226e+06 +Check: 0.992421 seconds, 1643 MB/s, Add: 1.21864 seconds, 460.066 MB/s, Gc ratio: 38.8025%, Peak idle memory: 2.0226e+06 ``` ## hash table @@ -65,6 +65,6 @@ Check: 1.00477 seconds, 1622.8 MB/s, Add: 1.21142 seconds, 462.808 MB/s, Gc rati (The hash table implementation doesn't work on range queries, and its purpose is to provide an idea of how fast point queries can be) ``` -Check: 0.854254 seconds, 1908.74 MB/s, Add: 0.632626 seconds, 886.232 MB/s, Gc ratio: 41.0827%, Peak idle memory: 0 +Check: 0.855247 seconds, 1906.52 MB/s, Add: 0.63138 seconds, 887.98 MB/s, Gc ratio: 41.2278%, Peak idle memory: 0 ``` diff --git a/update-readme.sh b/update-readme.sh index 99a27ae..afe2d06 100755 --- a/update-readme.sh +++ b/update-readme.sh @@ -1,7 +1,8 @@ #!/bin/bash # Disable frequency scaling -for i in $(seq "$(nproc)") ; do sudo cat /sys/devices/system/cpu/cpu$((i-1))/cpufreq/scaling_max_freq | sudo tee /sys/devices/system/cpu/cpu$((i-1))/cpufreq/scaling_min_freq >/dev/null ; done +# shellcheck disable=SC2002 +for i in $(seq "$(nproc)") ; do cat /sys/devices/system/cpu/cpu$((i-1))/cpufreq/scaling_max_freq | sudo tee /sys/devices/system/cpu/cpu$((i-1))/cpufreq/scaling_min_freq >/dev/null ; done cmake -S. -Bbuild -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang++ -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_CXX_FLAGS=-DNVALGRIND > /dev/null cmake --build build > /dev/null -- 2.43.0 From dabddbcb56e6f1f11d01995a8ef72e418f4f905d Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Tue, 27 Oct 2026 13:54:30 -0400 Subject: [PATCH 2/5] Use individual comparison operators instead of operator<=> This gives better codegen --- ConflictSet.cpp | 33 ++++++++++++++++++++++++++++++--- README.md | 42 +++++++++++++++++++++--------------------- 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/ConflictSet.cpp b/ConflictSet.cpp index 553fb11..8d6c2eb 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -116,13 +116,40 @@ struct InternalVersionT { constexpr InternalVersionT() = default; constexpr explicit InternalVersionT(int64_t value) : value(value) {} constexpr int64_t toInt64() const { return value; } // GCOVR_EXCL_LINE - constexpr auto operator<=>(const InternalVersionT &rhs) const { + constexpr bool operator<(const InternalVersionT &rhs) const { #if USE_64_BIT - return value <=> rhs.value; + return value < rhs.value; #else // Maintains ordering after overflow, as long as the full-precision versions // are within `kMaxCorrectVersionWindow` of eachother. - return int32_t(value - rhs.value) <=> 0; + return int32_t(value - rhs.value) < 0; +#endif + } + constexpr bool operator<=(const InternalVersionT &rhs) const { +#if USE_64_BIT + return value <= rhs.value; +#else + // Maintains ordering after overflow, as long as the full-precision versions + // are within `kMaxCorrectVersionWindow` of eachother. + return int32_t(value - rhs.value) <= 0; +#endif + } + constexpr bool operator>=(const InternalVersionT &rhs) const { +#if USE_64_BIT + return value >= rhs.value; +#else + // Maintains ordering after overflow, as long as the full-precision versions + // are within `kMaxCorrectVersionWindow` of eachother. + return int32_t(value - rhs.value) >= 0; +#endif + } + constexpr bool operator>(const InternalVersionT &rhs) const { +#if USE_64_BIT + return value > rhs.value; +#else + // Maintains ordering after overflow, as long as the full-precision versions + // are within `kMaxCorrectVersionWindow` of eachother. + return int32_t(value - rhs.value) > 0; #endif } constexpr bool operator==(const InternalVersionT &) const = default; diff --git a/README.md b/README.md index 54073d9..b650154 100644 --- a/README.md +++ b/README.md @@ -19,30 +19,30 @@ InstalledDir: /usr/lib/llvm-21/bin | ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:---------- -| 165.35 | 6,047,750.85 | 0.1% | 3,107.03 | 608.25 | 5.108 | 558.59 | 0.0% | 1.98 | `point reads` -| 161.75 | 6,182,478.30 | 0.2% | 3,036.76 | 594.75 | 5.106 | 539.35 | 0.0% | 1.94 | `prefix reads` -| 250.88 | 3,985,953.69 | 0.4% | 3,722.71 | 922.20 | 4.037 | 692.00 | 0.0% | 2.98 | `range reads` -| 358.34 | 2,790,617.48 | 0.2% | 4,523.64 | 1,317.11 | 3.435 | 720.22 | 2.1% | 4.28 | `point writes` -| 349.16 | 2,864,048.29 | 0.9% | 4,484.57 | 1,279.36 | 3.505 | 705.00 | 1.9% | 4.16 | `prefix writes` -| 192.06 | 5,206,715.53 | 0.0% | 2,224.10 | 706.55 | 3.148 | 377.17 | 3.3% | 2.30 | `range writes` -| 408.59 | 2,447,415.85 | 0.9% | 6,855.96 | 1,503.60 | 4.560 | 1,227.82 | 1.4% | 0.05 | `monotonic increasing point writes` -| 136,536.00 | 7,324.08 | 2.0% | 812,045.00 | 501,448.67 | 1.619 | 151,246.43 | 0.9% | 0.01 | `worst case for radix tree` -| 37.84 | 26,426,456.12 | 0.8% | 701.00 | 139.27 | 5.034 | 102.00 | 0.0% | 0.01 | `create and destroy` +| 168.06 | 5,950,172.07 | 0.0% | 3,107.03 | 615.58 | 5.047 | 558.59 | 0.0% | 2.01 | `point reads` +| 164.93 | 6,063,229.09 | 0.1% | 3,036.76 | 604.27 | 5.025 | 539.35 | 0.0% | 1.97 | `prefix reads` +| 242.64 | 4,121,298.49 | 0.1% | 3,722.71 | 891.62 | 4.175 | 692.00 | 0.0% | 2.90 | `range reads` +| 361.34 | 2,767,480.63 | 0.0% | 4,523.64 | 1,328.61 | 3.405 | 720.22 | 2.1% | 4.31 | `point writes` +| 349.52 | 2,861,056.44 | 0.1% | 4,484.57 | 1,279.55 | 3.505 | 705.00 | 1.9% | 4.19 | `prefix writes` +| 194.68 | 5,136,536.22 | 0.1% | 2,224.10 | 712.83 | 3.120 | 377.17 | 3.3% | 2.33 | `range writes` +| 416.48 | 2,401,076.73 | 1.0% | 6,855.96 | 1,522.78 | 4.502 | 1,227.82 | 1.4% | 0.05 | `monotonic increasing point writes` +| 131,614.80 | 7,597.93 | 1.0% | 812,044.75 | 484,589.00 | 1.676 | 151,246.50 | 0.1% | 0.01 | `worst case for radix tree` +| 38.24 | 26,148,048.33 | 1.5% | 701.00 | 140.19 | 5.001 | 102.00 | 0.0% | 0.01 | `create and destroy` ## Radix tree (this implementation) | ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:---------- -| 12.44 | 80,368,377.55 | 0.2% | 237.48 | 45.74 | 5.192 | 31.23 | 0.8% | 0.15 | `point reads` -| 14.59 | 68,537,989.62 | 0.6% | 291.66 | 53.69 | 5.432 | 40.16 | 0.5% | 0.17 | `prefix reads` -| 33.91 | 29,487,389.82 | 0.1% | 759.22 | 124.80 | 6.084 | 101.27 | 0.2% | 0.41 | `range reads` -| 19.54 | 51,172,096.42 | 0.0% | 369.51 | 71.92 | 5.138 | 47.92 | 0.5% | 0.23 | `point writes` -| 37.87 | 26,402,961.91 | 0.1% | 662.02 | 139.34 | 4.751 | 100.26 | 0.4% | 0.45 | `prefix writes` -| 39.11 | 25,569,264.79 | 0.1% | 725.90 | 143.87 | 5.046 | 111.09 | 0.1% | 0.48 | `range writes` -| 77.70 | 12,869,910.98 | 0.5% | 1,430.41 | 286.41 | 4.994 | 271.78 | 0.1% | 0.01 | `monotonic increasing point writes` -| 319,309.00 | 3,131.76 | 0.5% | 4,181,645.50 | 1,178,098.50 | 3.549 | 728,511.00 | 0.1% | 0.01 | `worst case for radix tree` -| 97.89 | 10,215,764.97 | 0.3% | 1,775.00 | 360.36 | 4.926 | 288.00 | 0.0% | 0.01 | `create and destroy` +| 12.15 | 82,299,987.93 | 0.3% | 233.49 | 44.55 | 5.241 | 31.23 | 0.7% | 0.15 | `point reads` +| 14.31 | 69,893,290.91 | 0.2% | 283.68 | 52.40 | 5.414 | 40.16 | 0.4% | 0.17 | `prefix reads` +| 33.44 | 29,901,162.28 | 0.2% | 714.70 | 122.88 | 5.816 | 100.97 | 0.2% | 0.40 | `range reads` +| 19.73 | 50,684,375.53 | 0.1% | 369.51 | 72.61 | 5.089 | 47.92 | 0.5% | 0.24 | `point writes` +| 38.10 | 26,248,730.55 | 0.0% | 662.02 | 140.17 | 4.723 | 100.26 | 0.3% | 0.45 | `prefix writes` +| 39.76 | 25,152,283.94 | 0.0% | 725.90 | 146.25 | 4.964 | 111.09 | 0.1% | 0.48 | `range writes` +| 80.73 | 12,387,533.79 | 4.3% | 1,424.45 | 296.99 | 4.796 | 271.47 | 0.1% | 0.01 | `monotonic increasing point writes` +| 310,683.00 | 3,218.71 | 2.9% | 3,866,495.00 | 1,131,201.00 | 3.418 | 713,374.00 | 0.1% | 0.01 | `worst case for radix tree` +| 101.32 | 9,869,495.84 | 0.9% | 1,775.00 | 372.75 | 4.762 | 288.00 | 0.0% | 0.01 | `create and destroy` # "Real data" test @@ -51,13 +51,13 @@ Point queries only. Gc ratio is the ratio of time spent doing garbage collection ## skip list ``` -Check: 4.7186 seconds, 345.557 MB/s, Add: 3.3305 seconds, 168.339 MB/s, Gc ratio: 38.6509%, Peak idle memory: 5.51852e+06 +Check: 4.65794 seconds, 350.058 MB/s, Add: 3.30748 seconds, 169.511 MB/s, Gc ratio: 38.2502%, Peak idle memory: 5.51852e+06 ``` ## radix tree ``` -Check: 0.992421 seconds, 1643 MB/s, Add: 1.21864 seconds, 460.066 MB/s, Gc ratio: 38.8025%, Peak idle memory: 2.0226e+06 +Check: 0.984726 seconds, 1655.84 MB/s, Add: 1.22965 seconds, 455.946 MB/s, Gc ratio: 38.8292%, Peak idle memory: 2.0226e+06 ``` ## hash table @@ -65,6 +65,6 @@ Check: 0.992421 seconds, 1643 MB/s, Add: 1.21864 seconds, 460.066 MB/s, Gc ratio (The hash table implementation doesn't work on range queries, and its purpose is to provide an idea of how fast point queries can be) ``` -Check: 0.855247 seconds, 1906.52 MB/s, Add: 0.63138 seconds, 887.98 MB/s, Gc ratio: 41.2278%, Peak idle memory: 0 +Check: 0.864098 seconds, 1886.99 MB/s, Add: 0.636882 seconds, 880.31 MB/s, Gc ratio: 41.0631%, Peak idle memory: 0 ``` -- 2.43.0 From d71d963d9ad1c046c40ecaad002b58846cfb411a Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Tue, 27 Oct 2026 14:06:58 -0400 Subject: [PATCH 3/5] Improve codegen for checkMaxBetweenExclusiveImpl(Node3 $ ninja -C build && nm -C -S build/CMakeFiles/conflict-set-object.dir/ConflictSet.cpp.o | grep 'checkMaxBetweenExclusive(Node3' ... 0000000000000000 0000000000000529 T checkMaxBetweenExclusive(Node3*, int, int, InternalVersionT, ReadContext*) $ git stash pop ... $ ninja -C build && nm -C -S build/CMakeFiles/conflict-set-object.dir/ConflictSet.cpp.o | grep 'checkMaxBetweenExclusive(Node3' ... 0000000000000000 0000000000000106 T checkMaxBetweenExclusive(Node3*, int, int, InternalVersionT, ReadContext*) --- ConflictSet.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ConflictSet.cpp b/ConflictSet.cpp index 8d6c2eb..5898ec3 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -2307,6 +2307,8 @@ bool checkMaxBetweenExclusiveImpl(Node3 *n, int begin, int end, assert(!(begin == -1 && end == 256)); auto *self = static_cast(n); + assume(1 <= self->numChildren); + assume(self->numChildren <= 3); ++begin; -- 2.43.0 From 84bf5707f9bb0e7f79ba80e4ccb3c6a6be7f265e Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Tue, 27 Oct 2026 14:21:58 -0400 Subject: [PATCH 4/5] Skip IFUNC call for Node3 in fixupMaxVersion --- ConflictSet.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ConflictSet.cpp b/ConflictSet.cpp index 5898ec3..33736c0 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -2936,9 +2936,13 @@ void fixupMaxVersion(Node *node, WriteContext *writeContext) { break; case Type_Node3: { auto *self3 = static_cast(node); - max = std::max(max, - horizontalMaxUpTo16(self3->childMaxVersion, - writeContext->zero, self3->numChildren)); + assume(1 <= self3->numChildren); + assume(self3->numChildren <= 3); + InternalVersionT m = self3->childMaxVersion[0]; + for (int i = 1; i < self3->numChildren; ++i) { + m = std::max(m, self3->childMaxVersion[i]); + } + max = std::max(max, m); } break; case Type_Node16: { auto *self16 = static_cast(node); -- 2.43.0 From 23d092ab6957ed6a47452a41c3ccc3657ee40c56 Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Tue, 27 Oct 2026 15:09:58 -0400 Subject: [PATCH 5/5] Avoid IFUNC for short prefixes in LCP --- LongestCommonPrefix.h | 74 +++++++++++++++++++++++++++++++++++++++++-- README.md | 42 ++++++++++++------------ 2 files changed, 92 insertions(+), 24 deletions(-) diff --git a/LongestCommonPrefix.h b/LongestCommonPrefix.h index 98da7c2..27d3edc 100644 --- a/LongestCommonPrefix.h +++ b/LongestCommonPrefix.h @@ -96,7 +96,7 @@ inline int firstNeqStride(const uint8_t *ap, const uint8_t *bp) { // GCOVR_EXCL_START #if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__) __attribute__((target("avx512f,avx512bw"))) inline int -longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) { +longestCommonPrefixLong(const uint8_t *ap, const uint8_t *bp, int cl) { int i = 0; int end = cl & ~63; while (i < end) { @@ -123,7 +123,7 @@ __attribute__((target("default"))) // GCOVR_EXCL_STOP inline int -longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) { +longestCommonPrefixLong(const uint8_t *ap, const uint8_t *bp, int cl) { if (!(cl >= 0)) { __builtin_unreachable(); // GCOVR_EXCL_LINE } @@ -182,4 +182,72 @@ longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) { } return i; -} \ No newline at end of file +} +// Use in-bounds native-word loads for short common lengths. The second load +// overlaps the first; because the first load matched, any mismatch found in +// the overlap is impossible and the returned absolute index is still correct. +// Precondition: cl <= 2 * sizeof(uint64_t) +__attribute__((always_inline)) static inline int +longestCommonPrefixShort(const uint8_t *ap, const uint8_t *bp, int cl) { + assert(cl <= 2 * int(sizeof(uint64_t))); + if (cl >= 8) { + uint64_t a0; + uint64_t b0; + memcpy(&a0, ap, sizeof(a0)); + memcpy(&b0, bp, sizeof(b0)); + if (uint64_t mismatched = a0 ^ b0) { + return std::countr_zero(mismatched) / 8; + } + if (cl == 8) { + return 8; + } + + uint64_t a1; + uint64_t b1; + memcpy(&a1, ap + cl - 8, sizeof(a1)); + memcpy(&b1, bp + cl - 8, sizeof(b1)); + if (uint64_t mismatched = a1 ^ b1) { + return cl - 8 + std::countr_zero(mismatched) / 8; + } + return cl; + } + + if (cl >= 4) { + uint32_t a0; + uint32_t b0; + memcpy(&a0, ap, sizeof(a0)); + memcpy(&b0, bp, sizeof(b0)); + if (uint32_t mismatched = a0 ^ b0) { + return std::countr_zero(mismatched) / 8; + } + if (cl == 4) { + return 4; + } + + uint32_t a1; + uint32_t b1; + memcpy(&a1, ap + cl - 4, sizeof(a1)); + memcpy(&b1, bp + cl - 4, sizeof(b1)); + if (uint32_t mismatched = a1 ^ b1) { + return cl - 4 + std::countr_zero(mismatched) / 8; + } + return cl; + } + + int i = 0; + while (i < cl && ap[i] == bp[i]) { + ++i; + } + return i; +} + +__attribute__((always_inline)) static inline int +longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) { + if (!(cl >= 0)) { + __builtin_unreachable(); // GCOVR_EXCL_LINE + } + if (cl <= 2 * int(sizeof(uint64_t))) { + return longestCommonPrefixShort(ap, bp, cl); + } + return longestCommonPrefixLong(ap, bp, cl); +} diff --git a/README.md b/README.md index b650154..21a2c85 100644 --- a/README.md +++ b/README.md @@ -19,30 +19,30 @@ InstalledDir: /usr/lib/llvm-21/bin | ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:---------- -| 168.06 | 5,950,172.07 | 0.0% | 3,107.03 | 615.58 | 5.047 | 558.59 | 0.0% | 2.01 | `point reads` -| 164.93 | 6,063,229.09 | 0.1% | 3,036.76 | 604.27 | 5.025 | 539.35 | 0.0% | 1.97 | `prefix reads` -| 242.64 | 4,121,298.49 | 0.1% | 3,722.71 | 891.62 | 4.175 | 692.00 | 0.0% | 2.90 | `range reads` -| 361.34 | 2,767,480.63 | 0.0% | 4,523.64 | 1,328.61 | 3.405 | 720.22 | 2.1% | 4.31 | `point writes` -| 349.52 | 2,861,056.44 | 0.1% | 4,484.57 | 1,279.55 | 3.505 | 705.00 | 1.9% | 4.19 | `prefix writes` -| 194.68 | 5,136,536.22 | 0.1% | 2,224.10 | 712.83 | 3.120 | 377.17 | 3.3% | 2.33 | `range writes` -| 416.48 | 2,401,076.73 | 1.0% | 6,855.96 | 1,522.78 | 4.502 | 1,227.82 | 1.4% | 0.05 | `monotonic increasing point writes` -| 131,614.80 | 7,597.93 | 1.0% | 812,044.75 | 484,589.00 | 1.676 | 151,246.50 | 0.1% | 0.01 | `worst case for radix tree` -| 38.24 | 26,148,048.33 | 1.5% | 701.00 | 140.19 | 5.001 | 102.00 | 0.0% | 0.01 | `create and destroy` +| 165.93 | 6,026,520.04 | 0.1% | 3,107.03 | 610.42 | 5.090 | 558.59 | 0.0% | 1.98 | `point reads` +| 163.08 | 6,132,046.57 | 0.1% | 3,036.76 | 599.68 | 5.064 | 539.35 | 0.0% | 1.95 | `prefix reads` +| 250.26 | 3,995,834.21 | 0.1% | 3,722.71 | 920.25 | 4.045 | 692.00 | 0.0% | 2.99 | `range reads` +| 359.64 | 2,780,581.63 | 0.1% | 4,523.64 | 1,322.63 | 3.420 | 720.22 | 2.1% | 4.30 | `point writes` +| 358.20 | 2,791,726.93 | 0.1% | 4,484.57 | 1,317.23 | 3.405 | 705.00 | 2.0% | 4.28 | `prefix writes` +| 193.91 | 5,157,081.29 | 0.1% | 2,224.10 | 713.33 | 3.118 | 377.17 | 3.3% | 2.32 | `range writes` +| 422.03 | 2,369,513.38 | 1.1% | 6,855.96 | 1,551.55 | 4.419 | 1,227.82 | 1.4% | 0.05 | `monotonic increasing point writes` +| 142,467.00 | 7,019.17 | 3.8% | 812,047.33 | 508,535.40 | 1.597 | 151,246.50 | 1.0% | 0.01 | `worst case for radix tree` +| 37.66 | 26,554,217.74 | 0.8% | 701.00 | 138.80 | 5.050 | 102.00 | 0.0% | 0.01 | `create and destroy` ## Radix tree (this implementation) | ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:---------- -| 12.15 | 82,299,987.93 | 0.3% | 233.49 | 44.55 | 5.241 | 31.23 | 0.7% | 0.15 | `point reads` -| 14.31 | 69,893,290.91 | 0.2% | 283.68 | 52.40 | 5.414 | 40.16 | 0.4% | 0.17 | `prefix reads` -| 33.44 | 29,901,162.28 | 0.2% | 714.70 | 122.88 | 5.816 | 100.97 | 0.2% | 0.40 | `range reads` -| 19.73 | 50,684,375.53 | 0.1% | 369.51 | 72.61 | 5.089 | 47.92 | 0.5% | 0.24 | `point writes` -| 38.10 | 26,248,730.55 | 0.0% | 662.02 | 140.17 | 4.723 | 100.26 | 0.3% | 0.45 | `prefix writes` -| 39.76 | 25,152,283.94 | 0.0% | 725.90 | 146.25 | 4.964 | 111.09 | 0.1% | 0.48 | `range writes` -| 80.73 | 12,387,533.79 | 4.3% | 1,424.45 | 296.99 | 4.796 | 271.47 | 0.1% | 0.01 | `monotonic increasing point writes` -| 310,683.00 | 3,218.71 | 2.9% | 3,866,495.00 | 1,131,201.00 | 3.418 | 713,374.00 | 0.1% | 0.01 | `worst case for radix tree` -| 101.32 | 9,869,495.84 | 0.9% | 1,775.00 | 372.75 | 4.762 | 288.00 | 0.0% | 0.01 | `create and destroy` +| 12.25 | 81,664,669.29 | 0.4% | 233.49 | 44.94 | 5.196 | 31.23 | 0.7% | 0.15 | `point reads` +| 13.35 | 74,903,950.49 | 0.1% | 271.75 | 49.10 | 5.534 | 40.21 | 0.4% | 0.16 | `prefix reads` +| 31.37 | 31,874,583.62 | 0.1% | 700.39 | 115.43 | 6.067 | 100.83 | 0.2% | 0.37 | `range reads` +| 19.55 | 51,153,057.80 | 0.1% | 366.51 | 71.90 | 5.098 | 47.92 | 0.5% | 0.23 | `point writes` +| 36.92 | 27,088,344.17 | 0.1% | 645.11 | 135.72 | 4.753 | 100.31 | 0.4% | 0.44 | `prefix writes` +| 36.84 | 27,143,553.08 | 0.1% | 691.04 | 135.42 | 5.103 | 114.01 | 0.1% | 0.45 | `range writes` +| 74.62 | 13,400,467.72 | 0.9% | 1,418.37 | 274.20 | 5.173 | 273.39 | 0.1% | 0.01 | `monotonic increasing point writes` +| 352,671.00 | 2,835.50 | 15.3% | 3,687,535.00 | 1,287,119.00 | 2.865 | 712,030.00 | 0.1% | 0.01 | :wavy_dash: `worst case for radix tree` (Unstable with ~1.0 iters. Increase `minEpochIterations` to e.g. 10) +| 96.63 | 10,348,729.42 | 1.2% | 1,766.00 | 355.53 | 4.967 | 283.00 | 0.0% | 0.01 | `create and destroy` # "Real data" test @@ -51,13 +51,13 @@ Point queries only. Gc ratio is the ratio of time spent doing garbage collection ## skip list ``` -Check: 4.65794 seconds, 350.058 MB/s, Add: 3.30748 seconds, 169.511 MB/s, Gc ratio: 38.2502%, Peak idle memory: 5.51852e+06 +Check: 4.69875 seconds, 347.017 MB/s, Add: 3.39475 seconds, 165.153 MB/s, Gc ratio: 38.7414%, Peak idle memory: 5.51852e+06 ``` ## radix tree ``` -Check: 0.984726 seconds, 1655.84 MB/s, Add: 1.22965 seconds, 455.946 MB/s, Gc ratio: 38.8292%, Peak idle memory: 2.0226e+06 +Check: 0.976745 seconds, 1669.37 MB/s, Add: 1.20464 seconds, 465.411 MB/s, Gc ratio: 39.4158%, Peak idle memory: 2.0226e+06 ``` ## hash table @@ -65,6 +65,6 @@ Check: 0.984726 seconds, 1655.84 MB/s, Add: 1.22965 seconds, 455.946 MB/s, Gc ra (The hash table implementation doesn't work on range queries, and its purpose is to provide an idea of how fast point queries can be) ``` -Check: 0.864098 seconds, 1886.99 MB/s, Add: 0.636882 seconds, 880.31 MB/s, Gc ratio: 41.0631%, Peak idle memory: 0 +Check: 0.864589 seconds, 1885.92 MB/s, Add: 0.634604 seconds, 883.47 MB/s, Gc ratio: 41.184%, Peak idle memory: 0 ``` -- 2.43.0