Avoid IFUNC for short prefixes in LCP

LongestCommonPrefix.h

@@ -96,7 +96,7 @@ inline int firstNeqStride(const uint8_t *ap, const uint8_t *bp) {
 // GCOVR_EXCL_START
 #if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
 __attribute__((target("avx512f,avx512bw"))) inline int
-longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
+longestCommonPrefixLong(const uint8_t *ap, const uint8_t *bp, int cl) {
   int i = 0;
   int end = cl & ~63;
   while (i < end) {
@@ -123,7 +123,7 @@ __attribute__((target("default")))
 // GCOVR_EXCL_STOP
 
 inline int
-longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
+longestCommonPrefixLong(const uint8_t *ap, const uint8_t *bp, int cl) {
   if (!(cl >= 0)) {
     __builtin_unreachable(); // GCOVR_EXCL_LINE
   }
@@ -183,3 +183,71 @@ longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
 
   return i;
 }
+// Use in-bounds native-word loads for short common lengths. The second load
+// overlaps the first; because the first load matched, any mismatch found in
+// the overlap is impossible and the returned absolute index is still correct.
+// Precondition: cl <= 2 * sizeof(uint64_t)
+__attribute__((always_inline)) static inline int
+longestCommonPrefixShort(const uint8_t *ap, const uint8_t *bp, int cl) {
+  assert(cl <= 2 * int(sizeof(uint64_t)));
+  if (cl >= 8) {
+    uint64_t a0;
+    uint64_t b0;
+    memcpy(&a0, ap, sizeof(a0));
+    memcpy(&b0, bp, sizeof(b0));
+    if (uint64_t mismatched = a0 ^ b0) {
+      return std::countr_zero(mismatched) / 8;
+    }
+    if (cl == 8) {
+      return 8;
+    }
+
+    uint64_t a1;
+    uint64_t b1;
+    memcpy(&a1, ap + cl - 8, sizeof(a1));
+    memcpy(&b1, bp + cl - 8, sizeof(b1));
+    if (uint64_t mismatched = a1 ^ b1) {
+      return cl - 8 + std::countr_zero(mismatched) / 8;
+    }
+    return cl;
+  }
+
+  if (cl >= 4) {
+    uint32_t a0;
+    uint32_t b0;
+    memcpy(&a0, ap, sizeof(a0));
+    memcpy(&b0, bp, sizeof(b0));
+    if (uint32_t mismatched = a0 ^ b0) {
+      return std::countr_zero(mismatched) / 8;
+    }
+    if (cl == 4) {
+      return 4;
+    }
+
+    uint32_t a1;
+    uint32_t b1;
+    memcpy(&a1, ap + cl - 4, sizeof(a1));
+    memcpy(&b1, bp + cl - 4, sizeof(b1));
+    if (uint32_t mismatched = a1 ^ b1) {
+      return cl - 4 + std::countr_zero(mismatched) / 8;
+    }
+    return cl;
+  }
+
+  int i = 0;
+  while (i < cl && ap[i] == bp[i]) {
+    ++i;
+  }
+  return i;
+}
+
+__attribute__((always_inline)) static inline int
+longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
+  if (!(cl >= 0)) {
+    __builtin_unreachable(); // GCOVR_EXCL_LINE
+  }
+  if (cl <= 2 * int(sizeof(uint64_t))) {
+    return longestCommonPrefixShort(ap, bp, cl);
+  }
+  return longestCommonPrefixLong(ap, bp, cl);
+}

README.md

@@ -19,30 +19,30 @@ InstalledDir: /usr/lib/llvm-21/bin
 
 |               ns/op |                op/s |    err% |          ins/op |          cyc/op |    IPC |         bra/op |   miss% |     total | benchmark
 |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
-|              168.06 |        5,950,172.07 |    0.0% |        3,107.03 |          615.58 |  5.047 |         558.59 |    0.0% |      2.01 | `point reads`
-|              164.93 |        6,063,229.09 |    0.1% |        3,036.76 |          604.27 |  5.025 |         539.35 |    0.0% |      1.97 | `prefix reads`
-|              242.64 |        4,121,298.49 |    0.1% |        3,722.71 |          891.62 |  4.175 |         692.00 |    0.0% |      2.90 | `range reads`
-|              361.34 |        2,767,480.63 |    0.0% |        4,523.64 |        1,328.61 |  3.405 |         720.22 |    2.1% |      4.31 | `point writes`
-|              349.52 |        2,861,056.44 |    0.1% |        4,484.57 |        1,279.55 |  3.505 |         705.00 |    1.9% |      4.19 | `prefix writes`
-|              194.68 |        5,136,536.22 |    0.1% |        2,224.10 |          712.83 |  3.120 |         377.17 |    3.3% |      2.33 | `range writes`
-|              416.48 |        2,401,076.73 |    1.0% |        6,855.96 |        1,522.78 |  4.502 |       1,227.82 |    1.4% |      0.05 | `monotonic increasing point writes`
-|          131,614.80 |            7,597.93 |    1.0% |      812,044.75 |      484,589.00 |  1.676 |     151,246.50 |    0.1% |      0.01 | `worst case for radix tree`
-|               38.24 |       26,148,048.33 |    1.5% |          701.00 |          140.19 |  5.001 |         102.00 |    0.0% |      0.01 | `create and destroy`
+|              165.93 |        6,026,520.04 |    0.1% |        3,107.03 |          610.42 |  5.090 |         558.59 |    0.0% |      1.98 | `point reads`
+|              163.08 |        6,132,046.57 |    0.1% |        3,036.76 |          599.68 |  5.064 |         539.35 |    0.0% |      1.95 | `prefix reads`
+|              250.26 |        3,995,834.21 |    0.1% |        3,722.71 |          920.25 |  4.045 |         692.00 |    0.0% |      2.99 | `range reads`
+|              359.64 |        2,780,581.63 |    0.1% |        4,523.64 |        1,322.63 |  3.420 |         720.22 |    2.1% |      4.30 | `point writes`
+|              358.20 |        2,791,726.93 |    0.1% |        4,484.57 |        1,317.23 |  3.405 |         705.00 |    2.0% |      4.28 | `prefix writes`
+|              193.91 |        5,157,081.29 |    0.1% |        2,224.10 |          713.33 |  3.118 |         377.17 |    3.3% |      2.32 | `range writes`
+|              422.03 |        2,369,513.38 |    1.1% |        6,855.96 |        1,551.55 |  4.419 |       1,227.82 |    1.4% |      0.05 | `monotonic increasing point writes`
+|          142,467.00 |            7,019.17 |    3.8% |      812,047.33 |      508,535.40 |  1.597 |     151,246.50 |    1.0% |      0.01 | `worst case for radix tree`
+|               37.66 |       26,554,217.74 |    0.8% |          701.00 |          138.80 |  5.050 |         102.00 |    0.0% |      0.01 | `create and destroy`
 
 ## Radix tree (this implementation)
 
 
 |               ns/op |                op/s |    err% |          ins/op |          cyc/op |    IPC |         bra/op |   miss% |     total | benchmark
 |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
-|               12.15 |       82,299,987.93 |    0.3% |          233.49 |           44.55 |  5.241 |          31.23 |    0.7% |      0.15 | `point reads`
-|               14.31 |       69,893,290.91 |    0.2% |          283.68 |           52.40 |  5.414 |          40.16 |    0.4% |      0.17 | `prefix reads`
-|               33.44 |       29,901,162.28 |    0.2% |          714.70 |          122.88 |  5.816 |         100.97 |    0.2% |      0.40 | `range reads`
-|               19.73 |       50,684,375.53 |    0.1% |          369.51 |           72.61 |  5.089 |          47.92 |    0.5% |      0.24 | `point writes`
-|               38.10 |       26,248,730.55 |    0.0% |          662.02 |          140.17 |  4.723 |         100.26 |    0.3% |      0.45 | `prefix writes`
-|               39.76 |       25,152,283.94 |    0.0% |          725.90 |          146.25 |  4.964 |         111.09 |    0.1% |      0.48 | `range writes`
-|               80.73 |       12,387,533.79 |    4.3% |        1,424.45 |          296.99 |  4.796 |         271.47 |    0.1% |      0.01 | `monotonic increasing point writes`
-|          310,683.00 |            3,218.71 |    2.9% |    3,866,495.00 |    1,131,201.00 |  3.418 |     713,374.00 |    0.1% |      0.01 | `worst case for radix tree`
-|              101.32 |        9,869,495.84 |    0.9% |        1,775.00 |          372.75 |  4.762 |         288.00 |    0.0% |      0.01 | `create and destroy`
+|               12.25 |       81,664,669.29 |    0.4% |          233.49 |           44.94 |  5.196 |          31.23 |    0.7% |      0.15 | `point reads`
+|               13.35 |       74,903,950.49 |    0.1% |          271.75 |           49.10 |  5.534 |          40.21 |    0.4% |      0.16 | `prefix reads`
+|               31.37 |       31,874,583.62 |    0.1% |          700.39 |          115.43 |  6.067 |         100.83 |    0.2% |      0.37 | `range reads`
+|               19.55 |       51,153,057.80 |    0.1% |          366.51 |           71.90 |  5.098 |          47.92 |    0.5% |      0.23 | `point writes`
+|               36.92 |       27,088,344.17 |    0.1% |          645.11 |          135.72 |  4.753 |         100.31 |    0.4% |      0.44 | `prefix writes`
+|               36.84 |       27,143,553.08 |    0.1% |          691.04 |          135.42 |  5.103 |         114.01 |    0.1% |      0.45 | `range writes`
+|               74.62 |       13,400,467.72 |    0.9% |        1,418.37 |          274.20 |  5.173 |         273.39 |    0.1% |      0.01 | `monotonic increasing point writes`
+|          352,671.00 |            2,835.50 |   15.3% |    3,687,535.00 |    1,287,119.00 |  2.865 |     712,030.00 |    0.1% |      0.01 | :wavy_dash: `worst case for radix tree` (Unstable with ~1.0 iters. Increase `minEpochIterations` to e.g. 10)
+|               96.63 |       10,348,729.42 |    1.2% |        1,766.00 |          355.53 |  4.967 |         283.00 |    0.0% |      0.01 | `create and destroy`
 
 # "Real data" test
 
@@ -51,13 +51,13 @@ Point queries only. Gc ratio is the ratio of time spent doing garbage collection
 ## skip list
 
 ```
-Check: 4.65794 seconds, 350.058 MB/s, Add: 3.30748 seconds, 169.511 MB/s, Gc ratio: 38.2502%, Peak idle memory: 5.51852e+06
+Check: 4.69875 seconds, 347.017 MB/s, Add: 3.39475 seconds, 165.153 MB/s, Gc ratio: 38.7414%, Peak idle memory: 5.51852e+06
 ```
 
 ## radix tree
 
 ```
-Check: 0.984726 seconds, 1655.84 MB/s, Add: 1.22965 seconds, 455.946 MB/s, Gc ratio: 38.8292%, Peak idle memory: 2.0226e+06
+Check: 0.976745 seconds, 1669.37 MB/s, Add: 1.20464 seconds, 465.411 MB/s, Gc ratio: 39.4158%, Peak idle memory: 2.0226e+06
 ```
 
 ## hash table
@@ -65,6 +65,6 @@ Check: 0.984726 seconds, 1655.84 MB/s, Add: 1.22965 seconds, 455.946 MB/s, Gc ra
 (The hash table implementation doesn't work on range queries, and its purpose is to provide an idea of how fast point queries can be)
 
 ```
-Check: 0.864098 seconds, 1886.99 MB/s, Add: 0.636882 seconds, 880.31 MB/s, Gc ratio: 41.0631%, Peak idle memory: 0
+Check: 0.864589 seconds, 1885.92 MB/s, Add: 0.634604 seconds, 883.47 MB/s, Gc ratio: 41.184%, Peak idle memory: 0
 ```