Merge pull request 'Improve perf/codegen' (#94) from perf into main

Reviewed-on: #94

ConflictSet.cpp

@@ -118,13 +118,40 @@ struct InternalVersionT {
   constexpr InternalVersionT() = default;
   constexpr explicit InternalVersionT(int64_t value) : value(value) {}
   constexpr int64_t toInt64() const { return value; } // GCOVR_EXCL_LINE
-  constexpr auto operator<=>(const InternalVersionT &rhs) const {
+  constexpr bool operator<(const InternalVersionT &rhs) const {
 #if USE_64_BIT
-    return value <=> rhs.value;
+    return value < rhs.value;
 #else
     // Maintains ordering after overflow, as long as the full-precision versions
     // are within `kMaxCorrectVersionWindow` of eachother.
-    return int32_t(value - rhs.value) <=> 0;
+    return int32_t(value - rhs.value) < 0;
+#endif
+  }
+  constexpr bool operator<=(const InternalVersionT &rhs) const {
+#if USE_64_BIT
+    return value <= rhs.value;
+#else
+    // Maintains ordering after overflow, as long as the full-precision versions
+    // are within `kMaxCorrectVersionWindow` of eachother.
+    return int32_t(value - rhs.value) <= 0;
+#endif
+  }
+  constexpr bool operator>=(const InternalVersionT &rhs) const {
+#if USE_64_BIT
+    return value >= rhs.value;
+#else
+    // Maintains ordering after overflow, as long as the full-precision versions
+    // are within `kMaxCorrectVersionWindow` of eachother.
+    return int32_t(value - rhs.value) >= 0;
+#endif
+  }
+  constexpr bool operator>(const InternalVersionT &rhs) const {
+#if USE_64_BIT
+    return value > rhs.value;
+#else
+    // Maintains ordering after overflow, as long as the full-precision versions
+    // are within `kMaxCorrectVersionWindow` of eachother.
+    return int32_t(value - rhs.value) > 0;
 #endif
   }
   constexpr bool operator==(const InternalVersionT &) const = default;
@@ -2284,6 +2311,8 @@ bool checkMaxBetweenExclusiveImpl(Node3 *n, int begin, int end,
   assert(!(begin == -1 && end == 256));
 
   auto *self = static_cast<Node3 *>(n);
+  assume(1 <= self->numChildren);
+  assume(self->numChildren <= 3);
 
   ++begin;
 
@@ -2957,9 +2986,13 @@ void fixupMaxVersion(Node *node, WriteContext *writeContext) {
     break;
   case Type_Node3: {
     auto *self3 = static_cast<Node3 *>(node);
-    max = std::max(max,
-                   horizontalMaxUpTo16(self3->childMaxVersion,
-                                       writeContext->zero, self3->numChildren));
+    assume(1 <= self3->numChildren);
+    assume(self3->numChildren <= 3);
+    InternalVersionT m = self3->childMaxVersion[0];
+    for (int i = 1; i < self3->numChildren; ++i) {
+      m = std::max(m, self3->childMaxVersion[i]);
+    }
+    max = std::max(max, m);
   } break;
   case Type_Node16: {
     auto *self16 = static_cast<Node16 *>(node);

LongestCommonPrefix.h

@@ -96,7 +96,7 @@ inline int firstNeqStride(const uint8_t *ap, const uint8_t *bp) {
 // GCOVR_EXCL_START
 #if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
 __attribute__((target("avx512f,avx512bw"))) inline int
-longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
+longestCommonPrefixLong(const uint8_t *ap, const uint8_t *bp, int cl) {
   int i = 0;
   int end = cl & ~63;
   while (i < end) {
@@ -123,7 +123,7 @@ __attribute__((target("default")))
 // GCOVR_EXCL_STOP
 
 inline int
-longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
+longestCommonPrefixLong(const uint8_t *ap, const uint8_t *bp, int cl) {
   if (!(cl >= 0)) {
     __builtin_unreachable(); // GCOVR_EXCL_LINE
   }
@@ -182,4 +182,72 @@ longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
   }
 
   return i;
-}
+}
+// Use in-bounds native-word loads for short common lengths. The second load
+// overlaps the first; because the first load matched, any mismatch found in
+// the overlap is impossible and the returned absolute index is still correct.
+// Precondition: cl <= 2 * sizeof(uint64_t)
+__attribute__((always_inline)) static inline int
+longestCommonPrefixShort(const uint8_t *ap, const uint8_t *bp, int cl) {
+  assert(cl <= 2 * int(sizeof(uint64_t)));
+  if (cl >= 8) {
+    uint64_t a0;
+    uint64_t b0;
+    memcpy(&a0, ap, sizeof(a0));
+    memcpy(&b0, bp, sizeof(b0));
+    if (uint64_t mismatched = a0 ^ b0) {
+      return std::countr_zero(mismatched) / 8;
+    }
+    if (cl == 8) {
+      return 8;
+    }
+
+    uint64_t a1;
+    uint64_t b1;
+    memcpy(&a1, ap + cl - 8, sizeof(a1));
+    memcpy(&b1, bp + cl - 8, sizeof(b1));
+    if (uint64_t mismatched = a1 ^ b1) {
+      return cl - 8 + std::countr_zero(mismatched) / 8;
+    }
+    return cl;
+  }
+
+  if (cl >= 4) {
+    uint32_t a0;
+    uint32_t b0;
+    memcpy(&a0, ap, sizeof(a0));
+    memcpy(&b0, bp, sizeof(b0));
+    if (uint32_t mismatched = a0 ^ b0) {
+      return std::countr_zero(mismatched) / 8;
+    }
+    if (cl == 4) {
+      return 4;
+    }
+
+    uint32_t a1;
+    uint32_t b1;
+    memcpy(&a1, ap + cl - 4, sizeof(a1));
+    memcpy(&b1, bp + cl - 4, sizeof(b1));
+    if (uint32_t mismatched = a1 ^ b1) {
+      return cl - 4 + std::countr_zero(mismatched) / 8;
+    }
+    return cl;
+  }
+
+  int i = 0;
+  while (i < cl && ap[i] == bp[i]) {
+    ++i;
+  }
+  return i;
+}
+
+__attribute__((always_inline)) static inline int
+longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
+  if (!(cl >= 0)) {
+    __builtin_unreachable(); // GCOVR_EXCL_LINE
+  }
+  if (cl <= 2 * int(sizeof(uint64_t))) {
+    return longestCommonPrefixShort(ap, bp, cl);
+  }
+  return longestCommonPrefixLong(ap, bp, cl);
+}

README.md

@@ -19,30 +19,30 @@ InstalledDir: /usr/lib/llvm-21/bin
 
 |               ns/op |                op/s |    err% |          ins/op |          cyc/op |    IPC |         bra/op |   miss% |     total | benchmark
 |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
-|              164.29 |        6,086,873.38 |    0.0% |        3,107.03 |          604.19 |  5.142 |         558.59 |    0.0% |      1.96 | `point reads`
-|              161.05 |        6,209,395.38 |    0.1% |        3,036.76 |          592.21 |  5.128 |         539.35 |    0.0% |      1.93 | `prefix reads`
-|              239.55 |        4,174,539.38 |    0.1% |        3,722.71 |          880.68 |  4.227 |         692.00 |    0.0% |      2.86 | `range reads`
-|              354.75 |        2,818,919.14 |    0.7% |        4,523.64 |        1,304.75 |  3.467 |         720.22 |    2.0% |      4.23 | `point writes`
-|              345.32 |        2,895,878.47 |    0.1% |        4,484.57 |        1,270.31 |  3.530 |         705.00 |    1.8% |      4.12 | `prefix writes`
-|              193.48 |        5,168,547.42 |    0.1% |        2,224.10 |          711.72 |  3.125 |         377.17 |    3.3% |      2.32 | `range writes`
-|              404.89 |        2,469,777.50 |    2.4% |        6,855.96 |        1,489.70 |  4.602 |       1,227.82 |    1.3% |      0.05 | `monotonic increasing point writes`
-|          134,231.80 |            7,449.80 |    1.9% |      812,045.25 |      495,770.40 |  1.638 |     151,246.50 |    0.9% |      0.01 | `worst case for radix tree`
-|               37.80 |       26,454,311.17 |    0.4% |          701.00 |          139.14 |  5.038 |         102.00 |    0.0% |      0.01 | `create and destroy`
+|              165.93 |        6,026,520.04 |    0.1% |        3,107.03 |          610.42 |  5.090 |         558.59 |    0.0% |      1.98 | `point reads`
+|              163.08 |        6,132,046.57 |    0.1% |        3,036.76 |          599.68 |  5.064 |         539.35 |    0.0% |      1.95 | `prefix reads`
+|              250.26 |        3,995,834.21 |    0.1% |        3,722.71 |          920.25 |  4.045 |         692.00 |    0.0% |      2.99 | `range reads`
+|              359.64 |        2,780,581.63 |    0.1% |        4,523.64 |        1,322.63 |  3.420 |         720.22 |    2.1% |      4.30 | `point writes`
+|              358.20 |        2,791,726.93 |    0.1% |        4,484.57 |        1,317.23 |  3.405 |         705.00 |    2.0% |      4.28 | `prefix writes`
+|              193.91 |        5,157,081.29 |    0.1% |        2,224.10 |          713.33 |  3.118 |         377.17 |    3.3% |      2.32 | `range writes`
+|              422.03 |        2,369,513.38 |    1.1% |        6,855.96 |        1,551.55 |  4.419 |       1,227.82 |    1.4% |      0.05 | `monotonic increasing point writes`
+|          142,467.00 |            7,019.17 |    3.8% |      812,047.33 |      508,535.40 |  1.597 |     151,246.50 |    1.0% |      0.01 | `worst case for radix tree`
+|               37.66 |       26,554,217.74 |    0.8% |          701.00 |          138.80 |  5.050 |         102.00 |    0.0% |      0.01 | `create and destroy`
 
 ## Radix tree (this implementation)
 
 
 |               ns/op |                op/s |    err% |          ins/op |          cyc/op |    IPC |         bra/op |   miss% |     total | benchmark
 |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
-|               12.89 |       77,565,115.56 |    0.1% |          244.55 |           47.43 |  5.155 |          34.21 |    0.6% |      0.15 | `point reads`
-|               15.11 |       66,162,047.76 |    0.1% |          297.79 |           55.60 |  5.356 |          43.23 |    0.4% |      0.18 | `prefix reads`
-|               36.29 |       27,559,358.29 |    0.1% |          783.16 |          133.44 |  5.869 |         109.52 |    0.2% |      0.43 | `range reads`
-|               20.53 |       48,719,405.55 |    0.1% |          381.81 |           75.51 |  5.057 |          51.04 |    0.5% |      0.25 | `point writes`
-|               39.37 |       25,402,042.40 |    0.1% |          685.00 |          144.83 |  4.730 |         106.72 |    0.3% |      0.47 | `prefix writes`
-|               43.78 |       22,843,841.63 |    0.1% |          800.40 |          161.06 |  4.970 |         127.36 |    0.1% |      0.53 | `range writes`
-|               78.37 |       12,760,008.75 |    1.0% |        1,452.61 |          288.24 |  5.040 |         278.69 |    0.1% |      0.01 | `monotonic increasing point writes`
-|          322,885.50 |            3,097.07 |    1.5% |    4,362,382.00 |    1,183,852.00 |  3.685 |     765,301.00 |    0.1% |      0.01 | `worst case for radix tree`
-|               99.99 |       10,000,718.79 |    0.4% |        1,775.00 |          367.93 |  4.824 |         288.00 |    0.0% |      0.01 | `create and destroy`
+|               12.25 |       81,664,669.29 |    0.4% |          233.49 |           44.94 |  5.196 |          31.23 |    0.7% |      0.15 | `point reads`
+|               13.35 |       74,903,950.49 |    0.1% |          271.75 |           49.10 |  5.534 |          40.21 |    0.4% |      0.16 | `prefix reads`
+|               31.37 |       31,874,583.62 |    0.1% |          700.39 |          115.43 |  6.067 |         100.83 |    0.2% |      0.37 | `range reads`
+|               19.55 |       51,153,057.80 |    0.1% |          366.51 |           71.90 |  5.098 |          47.92 |    0.5% |      0.23 | `point writes`
+|               36.92 |       27,088,344.17 |    0.1% |          645.11 |          135.72 |  4.753 |         100.31 |    0.4% |      0.44 | `prefix writes`
+|               36.84 |       27,143,553.08 |    0.1% |          691.04 |          135.42 |  5.103 |         114.01 |    0.1% |      0.45 | `range writes`
+|               74.62 |       13,400,467.72 |    0.9% |        1,418.37 |          274.20 |  5.173 |         273.39 |    0.1% |      0.01 | `monotonic increasing point writes`
+|          352,671.00 |            2,835.50 |   15.3% |    3,687,535.00 |    1,287,119.00 |  2.865 |     712,030.00 |    0.1% |      0.01 | :wavy_dash: `worst case for radix tree` (Unstable with ~1.0 iters. Increase `minEpochIterations` to e.g. 10)
+|               96.63 |       10,348,729.42 |    1.2% |        1,766.00 |          355.53 |  4.967 |         283.00 |    0.0% |      0.01 | `create and destroy`
 
 # "Real data" test
 
@@ -51,13 +51,13 @@ Point queries only. Gc ratio is the ratio of time spent doing garbage collection
 ## skip list
 
 ```
-Check: 4.62967 seconds, 352.195 MB/s, Add: 3.34177 seconds, 167.771 MB/s, Gc ratio: 37.9399%, Peak idle memory: 5.51852e+06
+Check: 4.69875 seconds, 347.017 MB/s, Add: 3.39475 seconds, 165.153 MB/s, Gc ratio: 38.7414%, Peak idle memory: 5.51852e+06
 ```
 
 ## radix tree
 
 ```
-Check: 1.00477 seconds, 1622.8 MB/s, Add: 1.21142 seconds, 462.808 MB/s, Gc ratio: 39.4716%, Peak idle memory: 2.0226e+06
+Check: 0.976745 seconds, 1669.37 MB/s, Add: 1.20464 seconds, 465.411 MB/s, Gc ratio: 39.4158%, Peak idle memory: 2.0226e+06
 ```
 
 ## hash table
@@ -65,6 +65,6 @@ Check: 1.00477 seconds, 1622.8 MB/s, Add: 1.21142 seconds, 462.808 MB/s, Gc rati
 (The hash table implementation doesn't work on range queries, and its purpose is to provide an idea of how fast point queries can be)
 
 ```
-Check: 0.854254 seconds, 1908.74 MB/s, Add: 0.632626 seconds, 886.232 MB/s, Gc ratio: 41.0827%, Peak idle memory: 0
+Check: 0.864589 seconds, 1885.92 MB/s, Add: 0.634604 seconds, 883.47 MB/s, Gc ratio: 41.184%, Peak idle memory: 0
 ```
 

update-readme.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 
 # Disable frequency scaling
-for i in $(seq "$(nproc)") ; do sudo cat /sys/devices/system/cpu/cpu$((i-1))/cpufreq/scaling_max_freq | sudo tee /sys/devices/system/cpu/cpu$((i-1))/cpufreq/scaling_min_freq >/dev/null ; done
+# shellcheck disable=SC2002
+for i in $(seq "$(nproc)") ; do cat /sys/devices/system/cpu/cpu$((i-1))/cpufreq/scaling_max_freq | sudo tee /sys/devices/system/cpu/cpu$((i-1))/cpufreq/scaling_min_freq >/dev/null ; done
 
 cmake -S. -Bbuild -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang++ -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_CXX_FLAGS=-DNVALGRIND > /dev/null
 cmake --build build > /dev/null