Run valgrind in arm64 release in CI

Move SIMD operations on potentially-indeterminate Node16::index bytes
into file-level assembly, where loading and operating on indeterminate
values is well-defined (unlike C++). Restructure scalar fallback loops
to iterate [0, numChildren) instead of [0, kMaxNodes). Fix TrivialSpan
construction from indeterminate pointers in check::Job::init and
insertPointWritesOrSorted to only construct when end.len > 0.

Add MSan toolchain to the debug CI build to catch these issues going
forward.

.gitea/workflows/ci.yml

@@ -40,7 +40,7 @@ jobs:
           - name: 64-bit-versions
             cmake_args: -DCMAKE_CXX_FLAGS=-DUSE_64_BIT=1
           - name: debug
-            cmake_args: -DCMAKE_BUILD_TYPE=Debug
+            cmake_args: -DCMAKE_BUILD_TYPE=Debug -DMSAN_TOOLCHAIN_PATH=/opt/msan
           - name: simd-fallback
             cmake_args: -DUSE_SIMD_FALLBACK=ON
           - name: gcc
@@ -76,6 +76,13 @@ jobs:
             sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
           done
 
+      - name: Download MSan toolchain
+        if: matrix.name == 'debug'
+        run: |
+          curl -Ls "https://minio.weaselab.dev/public/x86_64/msan-toolchain-21.1.8.tar.zst" -o /tmp/msan-toolchain.tar.zst
+          sudo mkdir -p /opt/msan
+          sudo tar --zstd -xf /tmp/msan-toolchain.tar.zst -C /opt/msan
+
       - name: Build
         run: |
           export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache"
@@ -116,8 +123,10 @@ jobs:
         include:
           - runner: ubuntu-latest-amd64
             arch: amd64
+            cmake_args: -DMSAN_TOOLCHAIN_PATH=/opt/msan
           - runner: ubuntu-latest-arm64
             arch: arm64
+            cmake_args: ""
     runs-on: ${{ matrix.runner }}
     steps:
       - uses: actions/checkout@v4
@@ -158,23 +167,25 @@ jobs:
           restore-keys: |
             ccache-release-${{ matrix.arch }}-
 
+      - name: Download MSan toolchain
+        if: matrix.arch == 'amd64'
+        run: |
+          curl -Ls "https://minio.weaselab.dev/public/x86_64/msan-toolchain-21.1.8.tar.zst" -o /tmp/msan-toolchain.tar.zst
+          sudo mkdir -p /opt/msan
+          sudo tar --zstd -xf /tmp/msan-toolchain.tar.zst -C /opt/msan
+
       - name: Build
         run: |
           export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache"
           rm -rf build
-          cmake -S . -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_FLAGS=-DNVALGRIND
+          cmake -S . -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_FLAGS=-DNVALGRIND ${{ matrix.cmake_args }}
           ninja -C build
           ccache -s
 
       - name: Test
         run: |
           cd build
-          # On arm64, valgrind needs the MAKE_MEM_DEFINED client requests for
-          # https://git.weaselab.dev/weaselab/conflict-set/issues/39, but this
-          # build has -DNVALGRIND, which compiles them out. Skip valgrind
-          # tests here; they run annotated in the test job.
-          ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 ${{ matrix.arch == 'arm64' && '-E valgrind' || '' }} -T Test -j "$(nproc)" --timeout 90 > /dev/null
-
+          ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 -T Test -j "$(nproc)" --timeout 90 > /dev/null
       - name: Package
         run: |
           cd build
@@ -218,87 +229,6 @@ jobs:
             --link "https://minio.weaselab.dev/jenkins/conflict-set/${{ gitea.run_number }}/release-${{ matrix.arch }}/Test.xml.zst" \
             | tee -a "$GITHUB_STEP_SUMMARY"
 
-  msan:
-    runs-on: ubuntu-latest-amd64
-    env:
-      MSAN_VERSION: "21.1.8"
-      MSAN_URL: "https://minio.weaselab.dev/public/x86_64/msan-toolchain-21.1.8.tar.zst"
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/cache@v4
-        with:
-          path: /var/cache/apt/archives
-          key: apt-amd64-${{ hashFiles('.gitea/workflows/ci.yml') }}
-
-      - name: Install dependencies
-        run: |
-          . /etc/os-release
-          wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
-          echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
-          sudo apt-get update -qq
-          sudo apt-get install -y \
-            build-essential ccache clang-21 cmake libc6-dbg \
-            llvm-21 lld-21 mold ninja-build python3 zstd
-          for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do
-            sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
-          done
-
-      - name: Download MSan toolchain
-        run: |
-          curl -Ls "${MSAN_URL}" -o /tmp/msan-toolchain.tar.zst
-          sudo mkdir -p /opt/msan
-          sudo tar --zstd -xf /tmp/msan-toolchain.tar.zst -C /opt/msan
-
-      - uses: actions/cache@v4
-        with:
-          path: .ccache
-          key: ccache-msan-${{ gitea.sha }}
-          restore-keys: |
-            ccache-msan-
-
-      - name: Build
-        run: |
-          export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache"
-          rm -rf build
-          cmake -S . -B build -G Ninja \
-            -DCMAKE_C_COMPILER=clang \
-            -DCMAKE_CXX_COMPILER=clang++ \
-            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-            -DCMAKE_BUILD_TYPE=Debug \
-            -DDISABLE_TSAN=ON \
-            -DUSE_MSAN=ON \
-            -DCMAKE_CXX_FLAGS="-fsanitize=memory -stdlib=libc++ -I/opt/msan/include/c++/v1 -L/opt/msan/lib -UNDEBUG" \
-            -DCMAKE_EXE_LINKER_FLAGS="-stdlib=libc++ -Wl,-rpath,/opt/msan/lib" \
-            -DCMAKE_SHARED_LINKER_FLAGS="-stdlib=libc++ -Wl,-rpath,/opt/msan/lib"
-          ninja -C build
-          ccache -s
-
-      - name: Test
-        run: |
-          cd build
-          ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 -T Test -j "$(nproc)" --timeout 300 > /dev/null
-
-      - name: Upload test results to MinIO
-        if: always()
-        env:
-          MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }}
-          MC_HOST_minio: https://${{ secrets.MINIO_ACCESS_KEY }}:${{ secrets.MINIO_SECRET_KEY }}@minio.weaselab.dev
-        run: |
-          if [ -z "$MINIO_ACCESS_KEY" ]; then
-            echo "MinIO credentials not configured; skipping upload"
-            exit 0
-          fi
-          zstd build/Testing/*/Test.xml
-          mc cp build/Testing/*/Test.xml.zst "minio/jenkins/conflict-set/${{ gitea.run_number }}/msan/"
-
-      - name: Test summary
-        if: always()
-        run: |
-          python3 ctest_summary.py build/Testing/*/Test.xml \
-            --link "https://minio.weaselab.dev/jenkins/conflict-set/${{ gitea.run_number }}/msan/Test.xml.zst" \
-            | tee -a "$GITHUB_STEP_SUMMARY"
-
   coverage:
     runs-on: ubuntu-latest-amd64
     steps:

CMakeLists.txt

@@ -5,7 +5,7 @@ project(
   DESCRIPTION
     "A data structure for optimistic concurrency control on ranges of bitwise-lexicographically-ordered keys."
   HOMEPAGE_URL "https://git.weaselab.dev/weaselab/conflict-set"
-  LANGUAGES C CXX)
+  LANGUAGES C CXX ASM)
 set(CMAKE_CXX_STANDARD 20)
 
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/version.txt ${PROJECT_VERSION})
@@ -49,18 +49,6 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
         ${LLVM_OBJCOPY}
         CACHE FILEPATH "path to objcopy binary" FORCE)
   endif()
-  if(USE_MSAN)
-    find_program(LLD_LINKER lld)
-    if(LLD_LINKER)
-      set(CMAKE_LINKER_TYPE
-          "LLD"
-          CACHE STRING "Use LLD linker" FORCE)
-      set(CMAKE_LINKER
-          ${LLD_LINKER}
-          CACHE FILEPATH "path to linker binary" FORCE)
-      add_link_options("-fuse-ld=lld")
-    endif()
-  endif()
 endif()
 
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
@@ -115,7 +103,12 @@ option(USE_SIMD_FALLBACK
 
 option(DISABLE_TSAN "Disable TSAN" OFF)
 
-option(USE_MSAN "Build with MemorySanitizer (disables ASan/UBSan)" OFF)
+set(MSAN_TOOLCHAIN_PATH
+    ""
+    CACHE
+      PATH
+      "Path to an MSan-instrumented libc++ toolchain. When set, an MSan fuzz_driver target is added. Must contain include/c++/v1 and lib. See build_msan_toolchain.sh to produce one."
+)
 
 # This is encouraged according to
 # https://valgrind.org/docs/manual/manual-core-adv.html#manual-core-adv.clientreq
@@ -137,7 +130,15 @@ endif()
 
 set(CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "")
 
-add_library(${PROJECT_NAME}-object OBJECT ConflictSet.cpp)
+# Architecture-specific SIMD assembly. These functions operate on
+# potentially-indeterminate memory, which is UB in C++ but well-defined in
+# assembly.
+set(SIMD_ASM_FILES)
+if(CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64 AND NOT USE_SIMD_FALLBACK)
+  set(SIMD_ASM_FILES ${CMAKE_CURRENT_SOURCE_DIR}/simd_x86_64.S)
+endif()
+
+add_library(${PROJECT_NAME}-object OBJECT ConflictSet.cpp ${SIMD_ASM_FILES})
 target_compile_options(${PROJECT_NAME}-object PRIVATE -fno-exceptions
                                                       -fvisibility=hidden)
 target_include_directories(${PROJECT_NAME}-object
@@ -240,16 +241,14 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
   endif()
 
   # ad hoc testing
-  add_executable(conflict_set_main ConflictSet.cpp)
+  add_executable(conflict_set_main ConflictSet.cpp ${SIMD_ASM_FILES})
   target_include_directories(conflict_set_main
                              PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
   target_compile_definitions(conflict_set_main PRIVATE ENABLE_MAIN)
   target_link_libraries(conflict_set_main PRIVATE nanobench)
 
-  if(NOT APPLE AND NOT USE_MSAN)
-    # libfuzzer target, to generate/manage corpus. MSan requires an instrumented
-    # libfuzzer runtime, which is not shipped with the compiler, so skip this
-    # target when building with MSan.
+  if(NOT APPLE)
+    # libfuzzer target, to generate/manage corpus
     set(FUZZ_FLAGS "-fsanitize=fuzzer-no-link,address,undefined")
     include(CheckCXXCompilerFlag)
     cmake_push_check_state()
@@ -258,7 +257,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
     cmake_pop_check_state()
 
     if(HAS_LIB_FUZZER)
-      add_executable(conflict_set_fuzz_test ConflictSet.cpp)
+      add_executable(conflict_set_fuzz_test ConflictSet.cpp ${SIMD_ASM_FILES})
       target_include_directories(conflict_set_fuzz_test
                                  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
       target_compile_definitions(conflict_set_fuzz_test PRIVATE ENABLE_FUZZ)
@@ -269,17 +268,13 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
     endif()
   endif()
 
-  # whitebox tests
-  add_executable(fuzz_driver ConflictSet.cpp FuzzTestDriver.cpp)
+  # whitebox tests asan+ubsan
+  add_executable(fuzz_driver ConflictSet.cpp FuzzTestDriver.cpp
+                             ${SIMD_ASM_FILES})
   target_compile_options(fuzz_driver PRIVATE ${TEST_FLAGS})
   if(NOT CMAKE_CROSSCOMPILING)
-    if(USE_MSAN)
-      target_compile_options(fuzz_driver PRIVATE -fsanitize=memory)
-      target_link_options(fuzz_driver PRIVATE -fsanitize=memory)
-    else()
-      target_compile_options(fuzz_driver PRIVATE -fsanitize=address,undefined)
-      target_link_options(fuzz_driver PRIVATE -fsanitize=address,undefined)
-    endif()
+    target_compile_options(fuzz_driver PRIVATE -fsanitize=address,undefined)
+    target_link_options(fuzz_driver PRIVATE -fsanitize=address,undefined)
   endif()
   target_compile_definitions(fuzz_driver PRIVATE ENABLE_FUZZ)
   target_include_directories(fuzz_driver
@@ -289,11 +284,39 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
     add_test(NAME conflict_set_fuzz_${hash} COMMAND fuzz_driver ${TEST})
   endforeach()
 
+  # whitebox tests msan
+  if(MSAN_TOOLCHAIN_PATH)
+    add_executable(fuzz_driver_msan ConflictSet.cpp FuzzTestDriver.cpp
+                                    ${SIMD_ASM_FILES})
+    target_compile_options(fuzz_driver_msan PRIVATE ${TEST_FLAGS})
+    if(NOT CMAKE_CROSSCOMPILING)
+      target_compile_options(
+        fuzz_driver_msan
+        PRIVATE -fsanitize=memory -fsanitize-memory-track-origins=2
+                $<$<COMPILE_LANGUAGE:CXX>:-stdlib=libc++>
+                -I${MSAN_TOOLCHAIN_PATH}/include/c++/v1)
+      target_link_options(
+        fuzz_driver_msan
+        PRIVATE
+        -fsanitize=memory
+        -fsanitize-memory-track-origins=2
+        -stdlib=libc++
+        -L${MSAN_TOOLCHAIN_PATH}/lib
+        LINKER:-rpath,${MSAN_TOOLCHAIN_PATH}/lib)
+    endif()
+    target_compile_definitions(fuzz_driver_msan PRIVATE ENABLE_FUZZ)
+    target_include_directories(fuzz_driver_msan
+                               PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
+    foreach(TEST ${CORPUS_TESTS})
+      get_filename_component(hash ${TEST} NAME)
+      add_test(NAME conflict_set_msan_${hash} COMMAND fuzz_driver_msan ${TEST})
+    endforeach()
+  endif()
+
   # tsan tests
-  if(NOT CMAKE_CROSSCOMPILING
-     AND NOT DISABLE_TSAN
-     AND NOT USE_MSAN)
-    add_executable(tsan_driver ConflictSet.cpp FuzzTestDriver.cpp)
+  if(NOT CMAKE_CROSSCOMPILING AND NOT DISABLE_TSAN)
+    add_executable(tsan_driver ConflictSet.cpp FuzzTestDriver.cpp
+                               ${SIMD_ASM_FILES})
     target_compile_options(tsan_driver PRIVATE ${TEST_FLAGS} -fsanitize=thread)
     target_link_options(tsan_driver PRIVATE -fsanitize=thread)
     target_compile_definitions(tsan_driver PRIVATE ENABLE_FUZZ THREAD_TEST)
@@ -514,11 +537,6 @@ target_include_directories(
   PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
          $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}>)
 
-if(USE_MSAN)
-  target_compile_options(${PROJECT_NAME} PUBLIC -fsanitize=memory)
-  target_link_options(${PROJECT_NAME} PUBLIC -fsanitize=memory)
-endif()
-
 set_target_properties(
   ${PROJECT_NAME} PROPERTIES VERSION ${PROJECT_VERSION}
                              SOVERSION ${PROJECT_VERSION_MAJOR})

ConflictSet.cpp

@@ -28,6 +28,7 @@ limitations under the License.
 #include "Internal.h"
 #include "LongestCommonPrefix.h"
 #include "Metrics.h"
+#include "simd.h"
 
 #include <algorithm>
 #include <bit>
@@ -910,34 +911,11 @@ int getNodeIndexExists(Node3 *self, uint8_t index) {
 
 int getNodeIndex(Node16 *self, uint8_t index) {
 
-#ifdef HAS_AVX
-  // Based on https://www.the-paper-trail.org/post/art-paper-notes/
-
-  // key_vec is 16 repeated copies of the searched-for byte, one for every
-  // possible position in child_keys that needs to be searched.
-  __m128i key_vec = _mm_set1_epi8(index);
-
-  // Compare all child_keys to 'index' in parallel. Don't worry if some of the
-  // keys aren't valid, we'll mask the results to only consider the valid ones
-  // below.
-  __m128i indices;
-  memcpy(&indices, self->index, Node16::kMaxNodes);
-  __m128i results = _mm_cmpeq_epi8(key_vec, indices);
-
-  // Build a mask to select only the first node->num_children values from the
-  // comparison (because the other values are meaningless)
-  uint32_t mask = (1 << self->numChildren) - 1;
-
-  // Change the results of the comparison into a bitfield, masking off any
-  // invalid comparisons.
-  uint32_t bitfield = _mm_movemask_epi8(results) & mask;
-
-  // No match if there are no '1's in the bitfield.
+#if defined(__x86_64__) && !defined(USE_SIMD_FALLBACK)
+  uint32_t bitfield =
+      find_eq_16(self->index, index) & ((1 << self->numChildren) - 1);
   if (bitfield == 0)
     return -1;
-
-  // Find the index of the first '1' in the bitfield by counting the leading
-  // zeros.
   return std::countr_zero(bitfield);
 #elif defined(HAS_ARM_NEON)
   // Based on
@@ -970,13 +948,9 @@ int getNodeIndex(Node16 *self, uint8_t index) {
 
 int getNodeIndexExists(Node16 *self, uint8_t index) {
 
-#ifdef HAS_AVX
-  __m128i key_vec = _mm_set1_epi8(index);
-  __m128i indices;
-  memcpy(&indices, self->index, Node16::kMaxNodes);
-  __m128i results = _mm_cmpeq_epi8(key_vec, indices);
-  uint32_t mask = (1 << self->numChildren) - 1;
-  uint32_t bitfield = _mm_movemask_epi8(results) & mask;
+#if defined(__x86_64__) && !defined(USE_SIMD_FALLBACK)
+  uint32_t bitfield =
+      find_eq_16(self->index, index) & ((1 << self->numChildren) - 1);
   assume(bitfield != 0);
   return std::countr_zero(bitfield);
 #elif defined(HAS_ARM_NEON)
@@ -1266,13 +1240,9 @@ TaggedNodePointer getChildGeq(Node16 *self, int child) {
     return nullptr;
   }
 
-#ifdef HAS_AVX
-  __m128i key_vec = _mm_set1_epi8(child);
-  __m128i indices;
-  memcpy(&indices, self->index, Node16::kMaxNodes);
-  __m128i results = _mm_cmpeq_epi8(key_vec, _mm_min_epu8(key_vec, indices));
-  int mask = (1 << self->numChildren) - 1;
-  uint32_t bitfield = _mm_movemask_epi8(results) & mask;
+#if defined(__x86_64__) && !defined(USE_SIMD_FALLBACK)
+  uint32_t bitfield =
+      find_ge_16(self->index, child) & ((1 << self->numChildren) - 1);
   return bitfield == 0 ? nullptr : self->children[std::countr_zero(bitfield)];
 #elif defined(HAS_ARM_NEON)
   uint8x16_t indices;
@@ -1332,8 +1302,8 @@ TaggedNodePointer getChildGeq(Node *self, int child) {
 
 TaggedNodePointer getFirstChild(Node0 *) { return nullptr; }
 TaggedNodePointer getFirstChild(Node3 *self) {
-  // Improves scan performance
-  self->children[1].prefetch();
+  // Don't prefetch since self->children[1] might be uninitialized and msan
+  // doesn't like that
   return self->children[0];
 }
 TaggedNodePointer getFirstChild(Node16 *self) {
@@ -2130,13 +2100,9 @@ bool scan16(const InternalVersionT *vs, const uint8_t *is, int begin, int end,
 
   return !(compared & mask);
 
-#elif defined(HAS_AVX)
+#elif defined(__x86_64__) && !defined(USE_SIMD_FALLBACK)
 
-  __m128i indices;
-  memcpy(&indices, is, 16);
-  indices = _mm_sub_epi8(indices, _mm_set1_epi8(begin));
-  uint32_t mask = ~_mm_movemask_epi8(_mm_cmpeq_epi8(
-      indices, _mm_max_epu8(indices, _mm_set1_epi8(end - begin))));
+  uint32_t mask = mask_in_range_16(is, begin, end);
 
   uint32_t compared = 0;
   if constexpr (kAVX512) {
@@ -2153,12 +2119,14 @@ bool scan16(const InternalVersionT *vs, const uint8_t *is, int begin, int end,
   auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
 
   uint32_t compared = 0;
-  for (int i = 0; i < 16; ++i) {
-    compared |= (vs[i] > readVersion) << i;
-  }
   uint32_t mask = 0;
   for (int i = 0; i < 16; ++i) {
-    mask |= inBounds(is[i]) << i;
+    if (vs[i] > readVersion) {
+      compared |= 1u << i;
+      if (inBounds(is[i])) {
+        mask |= 1u << i;
+      }
+    }
   }
   return !(compared & mask);
 
@@ -2250,17 +2218,9 @@ bool checkMaxBetweenExclusiveImpl(Node3 *n, int begin, int end,
   auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
 
   uint32_t mask = 0;
-  for (int i = 0; i < Node3::kMaxNodes; ++i) {
+  for (int i = 0; i < self->numChildren; ++i) {
     mask |= inBounds(self->index[i]) << i;
   }
-  mask &= (1 << self->numChildren) - 1;
-#ifdef __aarch64__
-  // The bits surviving the mask above don't derive from uninitialized slots,
-  // but clang 21+ on aarch64 lowers inBounds through flags+csel, which
-  // memcheck models imprecisely, tainting bits the mask provably clears.
-  // https://git.weaselab.dev/weaselab/conflict-set/issues/39
-  VALGRIND_MAKE_MEM_DEFINED(&mask, sizeof(mask));
-#endif
   if (!mask) {
     return true;
   }
@@ -2268,17 +2228,11 @@ bool checkMaxBetweenExclusiveImpl(Node3 *n, int begin, int end,
   const bool firstRangeOk =
       !child->entryPresent || child->entry.rangeVersion <= readVersion;
   uint32_t compared = 0;
-  for (int i = 0; i < Node3::kMaxNodes; ++i) {
+  for (int i = 0; i < self->numChildren; ++i) {
     compared |= (self->childMaxVersion[i] > readVersion) << i;
   }
 
-  uint32_t compared_masked = compared & mask;
-#ifdef __aarch64__
-  // Same imprecise csel modeling as above.
-  // https://git.weaselab.dev/weaselab/conflict-set/issues/39
-  VALGRIND_MAKE_MEM_DEFINED(&compared_masked, sizeof(compared_masked));
-#endif
-  return !compared_masked && firstRangeOk;
+  return !(compared & mask) && firstRangeOk;
 }
 
 template <bool kAVX512>
@@ -2344,15 +2298,10 @@ bool checkMaxBetweenExclusiveImpl(Node16 *n, int begin, int end,
 
   return !(compared & mask) && firstRangeOk;
 
-#elif defined(HAS_AVX)
+#elif defined(__x86_64__) && !defined(USE_SIMD_FALLBACK)
 
-  __m128i indices;
-  memcpy(&indices, self->index, 16);
-  indices = _mm_sub_epi8(indices, _mm_set1_epi8(begin));
-  uint32_t mask =
-      0xffff & ~_mm_movemask_epi8(_mm_cmpeq_epi8(
-                   indices, _mm_max_epu8(indices, _mm_set1_epi8(end - begin))));
-  mask &= (1 << self->numChildren) - 1;
+  uint32_t mask = mask_in_range_16(self->index, begin, end) &
+                  ((1 << self->numChildren) - 1);
   if (!mask) {
     return true;
   }
@@ -2375,10 +2324,9 @@ bool checkMaxBetweenExclusiveImpl(Node16 *n, int begin, int end,
   auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
 
   uint32_t mask = 0;
-  for (int i = 0; i < 16; ++i) {
+  for (int i = 0; i < self->numChildren; ++i) {
     mask |= inBounds(self->index[i]) << i;
   }
-  mask &= (1 << self->numChildren) - 1;
   if (!mask) {
     return true;
   }
@@ -2386,7 +2334,7 @@ bool checkMaxBetweenExclusiveImpl(Node16 *n, int begin, int end,
   const bool firstRangeOk =
       !child->entryPresent || child->entry.rangeVersion <= readVersion;
   uint32_t compared = 0;
-  for (int i = 0; i < 16; ++i) {
+  for (int i = 0; i < self->numChildren; ++i) {
     compared |= (self->childMaxVersion[i] > readVersion) << i;
   }
   return !(compared & mask) && firstRangeOk;
@@ -3864,17 +3812,17 @@ PRESERVE_NONE void right_side_iter(Job *job, Context *context) {
 void Job::init(const ConflictSet::ReadRange *read, ConflictSet::Result *result,
                Node *root, int64_t oldestVersionFullPrecision) {
   auto begin = TrivialSpan(read->begin.p, read->begin.len);
-  auto end = TrivialSpan(read->end.p, read->end.len);
   if (read->readVersion < oldestVersionFullPrecision) [[unlikely]] {
     *result = ConflictSet::TooOld;
     continuation = complete;
-  } else if (end.size() == 0) {
+  } else if (read->end.len == 0) {
     this->begin = begin;
     this->n = root;
     this->readVersion = InternalVersionT(read->readVersion);
     this->result = result;
     continuation = check::point_read_state_machine::begin;
   } else {
+    auto end = TrivialSpan(read->end.p, read->end.len);
     this->begin = begin;
     this->end = end;
     this->n = root;
@@ -5046,8 +4994,8 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
       for (int i = 0; i < count; ++i) {
         const auto &w = writes[i];
         auto begin = TrivialSpan(w.begin.p, w.begin.len);
-        auto end = TrivialSpan(w.end.p, w.end.len);
         if (w.end.len > 0) {
+          auto end = TrivialSpan(w.end.p, w.end.len);
           addWriteRange(rootParent->children[0], begin, end,
                         InternalVersionT(writeVersion), &writeContext);
         } else {

README.md

@@ -7,10 +7,10 @@ Hardware for all benchmarks is an AMD Ryzen 9 7900 with (2x32GB) 5600MT/s CL28-3
 ```
 $ clang++ --version
 
-Ubuntu clang version 20.0.0 (++20241120082228+86734c857724-1~exp1~20241120202359.554)
+Ubuntu clang version 21.1.8 (6ubuntu1)
 Target: x86_64-pc-linux-gnu
 Thread model: posix
-InstalledDir: /usr/lib/llvm-20/bin
+InstalledDir: /usr/lib/llvm-21/bin
 ```
 
 # Microbenchmark
@@ -19,30 +19,30 @@ InstalledDir: /usr/lib/llvm-20/bin
 
 |               ns/op |                op/s |    err% |          ins/op |          cyc/op |    IPC |         bra/op |   miss% |     total | benchmark
 |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
-|              161.29 |        6,200,056.17 |    0.1% |        3,014.03 |          831.04 |  3.627 |         504.59 |    0.0% |      1.93 | `point reads`
-|              158.32 |        6,316,160.64 |    0.1% |        2,954.16 |          815.80 |  3.621 |         490.17 |    0.0% |      1.89 | `prefix reads`
-|              237.39 |        4,212,409.50 |    0.2% |        3,592.41 |        1,233.96 |  2.911 |         629.31 |    0.0% |      2.84 | `range reads`
-|              442.11 |        2,261,878.94 |    0.0% |        4,450.57 |        2,314.25 |  1.923 |         707.92 |    2.1% |      5.28 | `point writes`
-|              439.89 |        2,273,308.53 |    0.1% |        4,410.22 |        2,302.29 |  1.916 |         694.74 |    2.1% |      5.25 | `prefix writes`
-|              290.96 |        3,436,936.78 |    0.0% |        2,315.38 |        1,528.68 |  1.515 |         396.69 |    3.3% |      3.49 | `range writes`
-|              476.93 |        2,096,762.02 |    0.6% |        6,999.33 |        2,484.94 |  2.817 |       1,251.73 |    1.3% |      0.06 | `monotonic increasing point writes`
-|          131,736.57 |            7,590.91 |    1.1% |      807,444.50 |      704,941.71 |  1.145 |     144,584.60 |    0.9% |      0.01 | `worst case for radix tree`
-|               45.50 |       21,978,369.95 |    1.1% |          902.00 |          232.36 |  3.882 |         132.00 |    0.0% |      0.01 | `create and destroy`
+|              164.29 |        6,086,873.38 |    0.0% |        3,107.03 |          604.19 |  5.142 |         558.59 |    0.0% |      1.96 | `point reads`
+|              161.05 |        6,209,395.38 |    0.1% |        3,036.76 |          592.21 |  5.128 |         539.35 |    0.0% |      1.93 | `prefix reads`
+|              239.55 |        4,174,539.38 |    0.1% |        3,722.71 |          880.68 |  4.227 |         692.00 |    0.0% |      2.86 | `range reads`
+|              354.75 |        2,818,919.14 |    0.7% |        4,523.64 |        1,304.75 |  3.467 |         720.22 |    2.0% |      4.23 | `point writes`
+|              345.32 |        2,895,878.47 |    0.1% |        4,484.57 |        1,270.31 |  3.530 |         705.00 |    1.8% |      4.12 | `prefix writes`
+|              193.48 |        5,168,547.42 |    0.1% |        2,224.10 |          711.72 |  3.125 |         377.17 |    3.3% |      2.32 | `range writes`
+|              404.89 |        2,469,777.50 |    2.4% |        6,855.96 |        1,489.70 |  4.602 |       1,227.82 |    1.3% |      0.05 | `monotonic increasing point writes`
+|          134,231.80 |            7,449.80 |    1.9% |      812,045.25 |      495,770.40 |  1.638 |     151,246.50 |    0.9% |      0.01 | `worst case for radix tree`
+|               37.80 |       26,454,311.17 |    0.4% |          701.00 |          139.14 |  5.038 |         102.00 |    0.0% |      0.01 | `create and destroy`
 
 ## Radix tree (this implementation)
 
 
 |               ns/op |                op/s |    err% |          ins/op |          cyc/op |    IPC |         bra/op |   miss% |     total | benchmark
 |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
-|               12.36 |       80,885,626.43 |    0.2% |          243.56 |           63.62 |  3.828 |          31.07 |    0.6% |      0.15 | `point reads`
-|               14.18 |       70,502,196.81 |    0.1% |          297.72 |           73.13 |  4.071 |          40.31 |    0.5% |      0.17 | `prefix reads`
-|               33.44 |       29,901,623.04 |    0.1% |          767.90 |          172.42 |  4.454 |         101.32 |    0.2% |      0.40 | `range reads`
-|               19.48 |       51,342,564.70 |    0.3% |          374.45 |          100.43 |  3.728 |          48.92 |    0.5% |      0.23 | `point writes`
-|               37.46 |       26,694,471.44 |    0.1% |          672.00 |          193.14 |  3.479 |         101.28 |    0.3% |      0.45 | `prefix writes`
-|               38.78 |       25,784,784.34 |    0.0% |          738.26 |          199.93 |  3.693 |         111.59 |    0.1% |      0.47 | `range writes`
-|               76.05 |       13,148,995.74 |    0.7% |        1,450.77 |          397.16 |  3.653 |         275.72 |    0.0% |      0.01 | `monotonic increasing point writes`
-|          286,920.33 |            3,485.29 |    0.4% |    4,117,948.00 |    1,521,352.00 |  2.707 |     714,833.00 |    0.1% |      0.01 | `worst case for radix tree`
-|               95.66 |       10,453,798.72 |    0.5% |        1,986.00 |          495.04 |  4.012 |         315.00 |    0.0% |      0.01 | `create and destroy`
+|               12.89 |       77,565,115.56 |    0.1% |          244.55 |           47.43 |  5.155 |          34.21 |    0.6% |      0.15 | `point reads`
+|               15.11 |       66,162,047.76 |    0.1% |          297.79 |           55.60 |  5.356 |          43.23 |    0.4% |      0.18 | `prefix reads`
+|               36.29 |       27,559,358.29 |    0.1% |          783.16 |          133.44 |  5.869 |         109.52 |    0.2% |      0.43 | `range reads`
+|               20.53 |       48,719,405.55 |    0.1% |          381.81 |           75.51 |  5.057 |          51.04 |    0.5% |      0.25 | `point writes`
+|               39.37 |       25,402,042.40 |    0.1% |          685.00 |          144.83 |  4.730 |         106.72 |    0.3% |      0.47 | `prefix writes`
+|               43.78 |       22,843,841.63 |    0.1% |          800.40 |          161.06 |  4.970 |         127.36 |    0.1% |      0.53 | `range writes`
+|               78.37 |       12,760,008.75 |    1.0% |        1,452.61 |          288.24 |  5.040 |         278.69 |    0.1% |      0.01 | `monotonic increasing point writes`
+|          322,885.50 |            3,097.07 |    1.5% |    4,362,382.00 |    1,183,852.00 |  3.685 |     765,301.00 |    0.1% |      0.01 | `worst case for radix tree`
+|               99.99 |       10,000,718.79 |    0.4% |        1,775.00 |          367.93 |  4.824 |         288.00 |    0.0% |      0.01 | `create and destroy`
 
 # "Real data" test
 
@@ -51,13 +51,13 @@ Point queries only. Gc ratio is the ratio of time spent doing garbage collection
 ## skip list
 
 ```
-Check: 4.53508 seconds, 371.81 MB/s, Add: 3.81222 seconds, 150.919 MB/s, Gc ratio: 33.66%, Peak idle memory: 5.61007e+06
+Check: 4.62967 seconds, 352.195 MB/s, Add: 3.34177 seconds, 167.771 MB/s, Gc ratio: 37.9399%, Peak idle memory: 5.51852e+06
 ```
 
 ## radix tree
 
 ```
-Check: 0.957735 seconds, 1760.6 MB/s, Add: 1.19942 seconds, 479.678 MB/s, Gc ratio: 38.6069%, Peak idle memory: 2.05667e+06
+Check: 1.00477 seconds, 1622.8 MB/s, Add: 1.21142 seconds, 462.808 MB/s, Gc ratio: 39.4716%, Peak idle memory: 2.0226e+06
 ```
 
 ## hash table
@@ -65,6 +65,6 @@ Check: 0.957735 seconds, 1760.6 MB/s, Add: 1.19942 seconds, 479.678 MB/s, Gc rat
 (The hash table implementation doesn't work on range queries, and its purpose is to provide an idea of how fast point queries can be)
 
 ```
-Check: 0.804598 seconds, 2095.69 MB/s, Add: 0.671221 seconds, 857.147 MB/s, Gc ratio: 35.0034%, Peak idle memory: 0
+Check: 0.854254 seconds, 1908.74 MB/s, Add: 0.632626 seconds, 886.232 MB/s, Gc ratio: 41.0827%, Peak idle memory: 0
 ```
 

build_msan_toolchain.sh

@@ -13,26 +13,26 @@ cd libcxx-msan
 
 git clone --depth=1 "https://github.com/llvm/llvm-project.git" -b "release/${LLVM_VERSION}.x"
 
-cmake -S llvm-project/runtimes -B build \
+cmake -S llvm-project/runtimes -B build_msan \
   -DCMAKE_BUILD_TYPE=Release \
-  -DCMAKE_C_COMPILER="clang-${LLVM_VERSION}" \
   -DCMAKE_CXX_COMPILER="clang++-${LLVM_VERSION}" \
-  -DLLVM_USE_SANITIZER=MemoryWithOrigins \
-  -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
-  -DLLVM_TARGETS_TO_BUILD=Native \
+  -DCMAKE_C_COMPILER="clang-${LLVM_VERSION}" \
   -DCMAKE_INSTALL_PREFIX="${MSAN_PREFIX}" \
-  -DLIBCXXABI_USE_LLVM_UNWINDER=ON \
-  -DLIBCXXABI_ENABLE_STATIC_UNWINDER=ON \
-  -DLIBCXX_USE_COMPILER_RT=ON \
-  -DLIBCXXABI_USE_COMPILER_RT=ON \
-  -DLIBUNWIND_USE_COMPILER_RT=ON \
-  -DLIBCXX_INCLUDE_TESTS=OFF \
-  -DLIBCXXABI_INCLUDE_TESTS=OFF \
-  -DLIBUNWIND_INCLUDE_TESTS=OFF \
-  -DLIBCXX_ENABLE_CLANG_TIDY=OFF
+  -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
+  -DLLVM_USE_SANITIZER=MemoryWithOrigins
 
-cmake --build build -j"${JOBS}"
-cmake --install build
+cmake --build build_msan -j"${JOBS}"
+cmake --install build_msan
+
+cmake -S llvm-project/runtimes -B build_nomsan \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_CXX_COMPILER="clang++-${LLVM_VERSION}" \
+  -DCMAKE_C_COMPILER="clang-${LLVM_VERSION}" \
+  -DCMAKE_INSTALL_PREFIX="${MSAN_PREFIX}" \
+  -DLLVM_ENABLE_RUNTIMES="libunwind"
+
+cmake --build build_nomsan -j"${JOBS}"
+cmake --install build_nomsan
 
 VERSION="$(clang-"${LLVM_VERSION}" --version | head -n1 | sed -E 's/.*clang version ([0-9.]+).*/\1/')"
 TARBALL="msan-toolchain-${VERSION}.tar.zst"

simd.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <cstdint>
+
+#if defined(__x86_64__) && !defined(USE_SIMD_FALLBACK)
+
+// SIMD operations on potentially-indeterminate Node16::index[16] bytes.
+// Implemented in file-level assembly (simd_x86_64.S) because loading and
+// operating on indeterminate values is UB in C++ but well-defined in
+// assembly. The caller must mask the returned bitfield to
+// [0, numChildren) before using it.
+//
+// Each function returns a 16-bit bitmask in the low 16 bits of a uint32_t
+// (upper 16 bits are zero). Bit i is set iff the condition holds at index i.
+
+extern "C" {
+// Returns bit i set iff idx[i] == key
+uint32_t find_eq_16(const uint8_t idx[16], uint8_t key);
+// Returns bit i set iff idx[i] >= child
+uint32_t find_ge_16(const uint8_t idx[16], uint8_t child);
+// Returns bit i set iff begin <= idx[i] < end
+uint32_t mask_in_range_16(const uint8_t idx[16], uint8_t begin, uint8_t end);
+}
+
+#endif

simd_x86_64.S

@@ -0,0 +1,74 @@
+// SIMD operations on potentially-indeterminate Node16::index[16] bytes.
+// Written in assembly so msan doesn't track the loads. The caller is
+// responsible for masking the returned bitfield to [0, numChildren) before
+// using it.
+//
+// All functions return a 16-bit bitmask in %eax (bit i set = condition true
+// at index i). The upper 16 bits of %eax are zero.
+//
+// System V AMD64 ABI:
+//   %rdi = const uint8_t *idx  (16 bytes)
+//   %esi = uint8_t key         (find_eq_16, find_ge_16)
+//   %sil = uint8_t begin       (mask_in_range_16)
+//   %dl  = uint8_t end         (mask_in_range_16)
+
+    .text
+
+// uint32_t find_eq_16(const uint8_t idx[16], uint8_t key)
+//   Returns bit i set if idx[i] == key
+    .globl find_eq_16
+    .type  find_eq_16, @function
+find_eq_16:
+    vmovd      %esi, %xmm1          // broadcast key
+    vpbroadcastb %xmm1, %xmm1
+    vmovdqu    (%rdi), %xmm0        // load 16 bytes (may contain indeterminate data)
+    vpcmpeqb   %xmm0, %xmm1, %xmm0  // 0xff for each match
+    vpmovmskb  %xmm0, %eax          // 16-bit bitmask
+    movzwl     %ax, %eax            // zero-extend to 32 bits
+    ret
+    .size find_eq_16, .-find_eq_16
+
+// uint32_t find_ge_16(const uint8_t idx[16], uint8_t child)
+//   Returns bit i set if idx[i] >= child
+//   x86 doesn't have a "compare unsigned >=" for bytes directly, so we use:
+//   min(key, idx[i]) == key  iff  idx[i] >= key
+    .globl find_ge_16
+    .type  find_ge_16, @function
+find_ge_16:
+    vmovd      %esi, %xmm1
+    vpbroadcastb %xmm1, %xmm1       // key broadcast
+    vmovdqu    (%rdi), %xmm0        // load 16 bytes
+    vpminub    %xmm0, %xmm1, %xmm2  // min(key, idx[i])
+    vpcmpeqb   %xmm2, %xmm1, %xmm0  // 0xff where min == key, i.e. idx[i] >= key
+    vpmovmskb  %xmm0, %eax
+    movzwl     %ax, %eax
+    ret
+    .size find_ge_16, .-find_ge_16
+
+// uint32_t mask_in_range_16(const uint8_t idx[16], uint8_t begin, uint8_t end)
+//   Returns bit i set if begin <= idx[i] < end
+//   Logic: (idx[i] - begin) < (end - begin)  [unsigned wrapping arithmetic]
+//   Equivalently: idx[i] - begin != max(idx[i] - begin, end - begin)
+//   i.e. idx[i] - begin is NOT equal to the saturated value.
+//   We compute: sub = idx - begin; result = (sub < (end-begin)) for each byte.
+//   Using: sub == max(sub, end-begin) means NOT in range.
+//   So: in_range = ~(movemask(cmpeq(sub, max(sub, range_size))))
+    .globl mask_in_range_16
+    .type  mask_in_range_16, @function
+mask_in_range_16:
+    vmovd      %esi, %xmm1          // begin
+    vpbroadcastb %xmm1, %xmm1
+    vmovd      %edx, %xmm2          // end
+    vpbroadcastb %xmm2, %xmm2
+    vmovdqu    (%rdi), %xmm0        // load 16 bytes
+    vpsubb     %xmm1, %xmm0, %xmm0  // idx - begin (wrapping)
+    vpsubb     %xmm1, %xmm2, %xmm2  // end - begin (range size)
+    vpmaxub    %xmm0, %xmm2, %xmm3  // max(idx-begin, range_size)
+    vpcmpeqb   %xmm3, %xmm0, %xmm0  // 0xff where NOT in range
+    vpmovmskb  %xmm0, %eax
+    not        %eax                 // invert: 1 = in range
+    movzwl     %ax, %eax
+    ret
+    .size mask_in_range_16, .-mask_in_range_16
+
+    .section .note.GNU-stack,"",@progbits