diff --git a/LongestCommonPrefix.h b/LongestCommonPrefix.h index 98da7c2..27d3edc 100644 --- a/LongestCommonPrefix.h +++ b/LongestCommonPrefix.h @@ -96,7 +96,7 @@ inline int firstNeqStride(const uint8_t *ap, const uint8_t *bp) { // GCOVR_EXCL_START #if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__) __attribute__((target("avx512f,avx512bw"))) inline int -longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) { +longestCommonPrefixLong(const uint8_t *ap, const uint8_t *bp, int cl) { int i = 0; int end = cl & ~63; while (i < end) { @@ -123,7 +123,7 @@ __attribute__((target("default"))) // GCOVR_EXCL_STOP inline int -longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) { +longestCommonPrefixLong(const uint8_t *ap, const uint8_t *bp, int cl) { if (!(cl >= 0)) { __builtin_unreachable(); // GCOVR_EXCL_LINE } @@ -182,4 +182,72 @@ longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) { } return i; -} \ No newline at end of file +} +// Use in-bounds native-word loads for short common lengths. The second load +// overlaps the first; because the first load matched, any mismatch found in +// the overlap is impossible and the returned absolute index is still correct. +// Precondition: cl <= 2 * sizeof(uint64_t) +__attribute__((always_inline)) static inline int +longestCommonPrefixShort(const uint8_t *ap, const uint8_t *bp, int cl) { + assert(cl <= 2 * int(sizeof(uint64_t))); + if (cl >= 8) { + uint64_t a0; + uint64_t b0; + memcpy(&a0, ap, sizeof(a0)); + memcpy(&b0, bp, sizeof(b0)); + if (uint64_t mismatched = a0 ^ b0) { + return std::countr_zero(mismatched) / 8; + } + if (cl == 8) { + return 8; + } + + uint64_t a1; + uint64_t b1; + memcpy(&a1, ap + cl - 8, sizeof(a1)); + memcpy(&b1, bp + cl - 8, sizeof(b1)); + if (uint64_t mismatched = a1 ^ b1) { + return cl - 8 + std::countr_zero(mismatched) / 8; + } + return cl; + } + + if (cl >= 4) { + uint32_t a0; + uint32_t b0; + memcpy(&a0, ap, sizeof(a0)); + memcpy(&b0, bp, sizeof(b0)); + if (uint32_t mismatched = a0 ^ b0) { + return std::countr_zero(mismatched) / 8; + } + if (cl == 4) { + return 4; + } + + uint32_t a1; + uint32_t b1; + memcpy(&a1, ap + cl - 4, sizeof(a1)); + memcpy(&b1, bp + cl - 4, sizeof(b1)); + if (uint32_t mismatched = a1 ^ b1) { + return cl - 4 + std::countr_zero(mismatched) / 8; + } + return cl; + } + + int i = 0; + while (i < cl && ap[i] == bp[i]) { + ++i; + } + return i; +} + +__attribute__((always_inline)) static inline int +longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) { + if (!(cl >= 0)) { + __builtin_unreachable(); // GCOVR_EXCL_LINE + } + if (cl <= 2 * int(sizeof(uint64_t))) { + return longestCommonPrefixShort(ap, bp, cl); + } + return longestCommonPrefixLong(ap, bp, cl); +} diff --git a/README.md b/README.md index b650154..21a2c85 100644 --- a/README.md +++ b/README.md @@ -19,30 +19,30 @@ InstalledDir: /usr/lib/llvm-21/bin | ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:---------- -| 168.06 | 5,950,172.07 | 0.0% | 3,107.03 | 615.58 | 5.047 | 558.59 | 0.0% | 2.01 | `point reads` -| 164.93 | 6,063,229.09 | 0.1% | 3,036.76 | 604.27 | 5.025 | 539.35 | 0.0% | 1.97 | `prefix reads` -| 242.64 | 4,121,298.49 | 0.1% | 3,722.71 | 891.62 | 4.175 | 692.00 | 0.0% | 2.90 | `range reads` -| 361.34 | 2,767,480.63 | 0.0% | 4,523.64 | 1,328.61 | 3.405 | 720.22 | 2.1% | 4.31 | `point writes` -| 349.52 | 2,861,056.44 | 0.1% | 4,484.57 | 1,279.55 | 3.505 | 705.00 | 1.9% | 4.19 | `prefix writes` -| 194.68 | 5,136,536.22 | 0.1% | 2,224.10 | 712.83 | 3.120 | 377.17 | 3.3% | 2.33 | `range writes` -| 416.48 | 2,401,076.73 | 1.0% | 6,855.96 | 1,522.78 | 4.502 | 1,227.82 | 1.4% | 0.05 | `monotonic increasing point writes` -| 131,614.80 | 7,597.93 | 1.0% | 812,044.75 | 484,589.00 | 1.676 | 151,246.50 | 0.1% | 0.01 | `worst case for radix tree` -| 38.24 | 26,148,048.33 | 1.5% | 701.00 | 140.19 | 5.001 | 102.00 | 0.0% | 0.01 | `create and destroy` +| 165.93 | 6,026,520.04 | 0.1% | 3,107.03 | 610.42 | 5.090 | 558.59 | 0.0% | 1.98 | `point reads` +| 163.08 | 6,132,046.57 | 0.1% | 3,036.76 | 599.68 | 5.064 | 539.35 | 0.0% | 1.95 | `prefix reads` +| 250.26 | 3,995,834.21 | 0.1% | 3,722.71 | 920.25 | 4.045 | 692.00 | 0.0% | 2.99 | `range reads` +| 359.64 | 2,780,581.63 | 0.1% | 4,523.64 | 1,322.63 | 3.420 | 720.22 | 2.1% | 4.30 | `point writes` +| 358.20 | 2,791,726.93 | 0.1% | 4,484.57 | 1,317.23 | 3.405 | 705.00 | 2.0% | 4.28 | `prefix writes` +| 193.91 | 5,157,081.29 | 0.1% | 2,224.10 | 713.33 | 3.118 | 377.17 | 3.3% | 2.32 | `range writes` +| 422.03 | 2,369,513.38 | 1.1% | 6,855.96 | 1,551.55 | 4.419 | 1,227.82 | 1.4% | 0.05 | `monotonic increasing point writes` +| 142,467.00 | 7,019.17 | 3.8% | 812,047.33 | 508,535.40 | 1.597 | 151,246.50 | 1.0% | 0.01 | `worst case for radix tree` +| 37.66 | 26,554,217.74 | 0.8% | 701.00 | 138.80 | 5.050 | 102.00 | 0.0% | 0.01 | `create and destroy` ## Radix tree (this implementation) | ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:---------- -| 12.15 | 82,299,987.93 | 0.3% | 233.49 | 44.55 | 5.241 | 31.23 | 0.7% | 0.15 | `point reads` -| 14.31 | 69,893,290.91 | 0.2% | 283.68 | 52.40 | 5.414 | 40.16 | 0.4% | 0.17 | `prefix reads` -| 33.44 | 29,901,162.28 | 0.2% | 714.70 | 122.88 | 5.816 | 100.97 | 0.2% | 0.40 | `range reads` -| 19.73 | 50,684,375.53 | 0.1% | 369.51 | 72.61 | 5.089 | 47.92 | 0.5% | 0.24 | `point writes` -| 38.10 | 26,248,730.55 | 0.0% | 662.02 | 140.17 | 4.723 | 100.26 | 0.3% | 0.45 | `prefix writes` -| 39.76 | 25,152,283.94 | 0.0% | 725.90 | 146.25 | 4.964 | 111.09 | 0.1% | 0.48 | `range writes` -| 80.73 | 12,387,533.79 | 4.3% | 1,424.45 | 296.99 | 4.796 | 271.47 | 0.1% | 0.01 | `monotonic increasing point writes` -| 310,683.00 | 3,218.71 | 2.9% | 3,866,495.00 | 1,131,201.00 | 3.418 | 713,374.00 | 0.1% | 0.01 | `worst case for radix tree` -| 101.32 | 9,869,495.84 | 0.9% | 1,775.00 | 372.75 | 4.762 | 288.00 | 0.0% | 0.01 | `create and destroy` +| 12.25 | 81,664,669.29 | 0.4% | 233.49 | 44.94 | 5.196 | 31.23 | 0.7% | 0.15 | `point reads` +| 13.35 | 74,903,950.49 | 0.1% | 271.75 | 49.10 | 5.534 | 40.21 | 0.4% | 0.16 | `prefix reads` +| 31.37 | 31,874,583.62 | 0.1% | 700.39 | 115.43 | 6.067 | 100.83 | 0.2% | 0.37 | `range reads` +| 19.55 | 51,153,057.80 | 0.1% | 366.51 | 71.90 | 5.098 | 47.92 | 0.5% | 0.23 | `point writes` +| 36.92 | 27,088,344.17 | 0.1% | 645.11 | 135.72 | 4.753 | 100.31 | 0.4% | 0.44 | `prefix writes` +| 36.84 | 27,143,553.08 | 0.1% | 691.04 | 135.42 | 5.103 | 114.01 | 0.1% | 0.45 | `range writes` +| 74.62 | 13,400,467.72 | 0.9% | 1,418.37 | 274.20 | 5.173 | 273.39 | 0.1% | 0.01 | `monotonic increasing point writes` +| 352,671.00 | 2,835.50 | 15.3% | 3,687,535.00 | 1,287,119.00 | 2.865 | 712,030.00 | 0.1% | 0.01 | :wavy_dash: `worst case for radix tree` (Unstable with ~1.0 iters. Increase `minEpochIterations` to e.g. 10) +| 96.63 | 10,348,729.42 | 1.2% | 1,766.00 | 355.53 | 4.967 | 283.00 | 0.0% | 0.01 | `create and destroy` # "Real data" test @@ -51,13 +51,13 @@ Point queries only. Gc ratio is the ratio of time spent doing garbage collection ## skip list ``` -Check: 4.65794 seconds, 350.058 MB/s, Add: 3.30748 seconds, 169.511 MB/s, Gc ratio: 38.2502%, Peak idle memory: 5.51852e+06 +Check: 4.69875 seconds, 347.017 MB/s, Add: 3.39475 seconds, 165.153 MB/s, Gc ratio: 38.7414%, Peak idle memory: 5.51852e+06 ``` ## radix tree ``` -Check: 0.984726 seconds, 1655.84 MB/s, Add: 1.22965 seconds, 455.946 MB/s, Gc ratio: 38.8292%, Peak idle memory: 2.0226e+06 +Check: 0.976745 seconds, 1669.37 MB/s, Add: 1.20464 seconds, 465.411 MB/s, Gc ratio: 39.4158%, Peak idle memory: 2.0226e+06 ``` ## hash table @@ -65,6 +65,6 @@ Check: 0.984726 seconds, 1655.84 MB/s, Add: 1.22965 seconds, 455.946 MB/s, Gc ra (The hash table implementation doesn't work on range queries, and its purpose is to provide an idea of how fast point queries can be) ``` -Check: 0.864098 seconds, 1886.99 MB/s, Add: 0.636882 seconds, 880.31 MB/s, Gc ratio: 41.0631%, Peak idle memory: 0 +Check: 0.864589 seconds, 1885.92 MB/s, Add: 0.634604 seconds, 883.47 MB/s, Gc ratio: 41.184%, Peak idle memory: 0 ```