Try eliminating some branches

ConflictSet.cpp

@@ -2190,13 +2190,16 @@ template <bool kAVX512> struct CheckRangeLeftSide {
 
   Node *n;
   std::span<const uint8_t> remaining;
-  int prefixLen;
+  const int prefixLen;
   InternalVersionT readVersion;
   ConflictSet::Impl *impl;
   int searchPathLen = 0;
   bool ok;
 
-  bool step() {
+  template <bool kSearchLengthMaybeShort> bool step() {
+    if constexpr (!kSearchLengthMaybeShort) {
+      assume(searchPathLen > prefixLen);
+    }
     if (maxVersion(n, impl) <= readVersion) {
       ok = true;
       return true;
@@ -2311,13 +2314,16 @@ template <bool kAVX512> struct CheckRangeRightSide {
   Node *n;
   std::span<const uint8_t> key;
   std::span<const uint8_t> remaining;
-  int prefixLen;
+  const int prefixLen;
   InternalVersionT readVersion;
   ConflictSet::Impl *impl;
   int searchPathLen = 0;
   bool ok;
 
-  bool step() {
+  template <bool kSearchLengthMaybeShort> bool step() {
+    if (!kSearchLengthMaybeShort) {
+      assume(searchPathLen > prefixLen);
+    }
 #if DEBUG_VERBOSE && !defined(NDEBUG)
     fprintf(stderr,
             "Search path: %s, searchPathLen: %d, prefixLen: %d, remaining: "
@@ -2474,8 +2480,17 @@ bool checkRangeReadImpl(Node *n, std::span<const uint8_t> begin,
   if (lcp == int(begin.size())) {
     CheckRangeRightSide<kAVX512> checkRangeRightSide{n, end, lcp, readVersion,
                                                      impl};
-    while (!checkRangeRightSide.step())
-      ;
+    // Advance until we know the search path is not too short. This lets us save
+    // a bunch of branches in the hot path.
+    while (!checkRangeRightSide
+                .template step</*kSearchLengthMaybeShort*/ true>()) {
+      if (checkRangeRightSide.searchPathLen > checkRangeRightSide.prefixLen) {
+        while (!checkRangeRightSide
+                    .template step</*kSearchLengthMaybeShort*/ false>())
+          ;
+        break;
+      }
+    }
     return checkRangeRightSide.ok;
   }
 
@@ -2489,24 +2504,46 @@ bool checkRangeReadImpl(Node *n, std::span<const uint8_t> begin,
   CheckRangeRightSide<kAVX512> checkRangeRightSide{n, end, lcp + 1, readVersion,
                                                    impl};
 
-  for (;;) {
-    bool leftDone = checkRangeLeftSide.step();
-    bool rightDone = checkRangeRightSide.step();
-    if (!leftDone && !rightDone) {
-      continue;
+  // Advance until we know the search paths are not too short. This lets us save
+  // a bunch of branches in the hot path.
+  bool leftDone = false;
+  while (!((leftDone = checkRangeLeftSide.template step<
+                       /*kSearchLengthMaybeShort*/
+                       true>()))) {
+    if (checkRangeLeftSide.searchPathLen > checkRangeLeftSide.prefixLen) {
+      break;
     }
+  }
+  bool rightDone = false;
+  while (!((rightDone = checkRangeRightSide.template step<
+                        /*kSearchLengthMaybeShort*/
+                        true>()))) {
+    if (checkRangeRightSide.searchPathLen > checkRangeRightSide.prefixLen) {
+      break;
+    }
+  }
+
+  for (;;) {
     if (leftDone && rightDone) {
       break;
-    } else if (leftDone) {
-      while (!checkRangeRightSide.step())
+    } else if (leftDone && !rightDone) {
+      while (!((rightDone =
+                    checkRangeRightSide
+                        .template step</*kSearchLengthMaybeShort*/ false>())))
+        ;
+      break;
+    } else if (!leftDone && rightDone) {
+      while (!(
+          (leftDone = checkRangeLeftSide
+                          .template step</*kSearchLengthMaybeShort*/ false>())))
         ;
       break;
     } else {
-      assert(rightDone);
-      while (!checkRangeLeftSide.step())
-        ;
+      leftDone =
+          checkRangeLeftSide.template step</*kSearchLengthMaybeShort*/ false>();
+      rightDone = checkRangeRightSide
+                      .template step</*kSearchLengthMaybeShort*/ false>();
     }
-    break;
   }
 
   return checkRangeLeftSide.ok & checkRangeRightSide.ok;