Update some of the invariant checks to account for leaves

.gitea/workflows/ci.yml

@@ -1,306 +0,0 @@
-name: CI
-
-on: [push, pull_request]
-
-env:
-  CC: clang
-  CXX: clang++
-
-jobs:
-  pre-commit:
-    runs-on: ubuntu-latest-amd64
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/cache@v4
-        with:
-          path: /var/cache/apt/archives
-          key: apt-amd64-${{ hashFiles('.gitea/workflows/ci.yml') }}
-
-      - name: Install dependencies
-        run: |
-          . /etc/os-release
-          wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
-          echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
-          sudo apt-get update -qq
-          sudo apt-get install -y \
-            clang-21 git nodejs pre-commit
-          for tool in clang clang++; do
-            sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
-          done
-
-      - uses: actions/cache@v4
-        with:
-          path: ~/.cache/pre-commit
-          key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
-
-      - name: Run pre-commit
-        run: |
-          git config --global --add safe.directory "$PWD"
-          pre-commit run --all-files --show-diff-on-failure
-
-  test:
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - name: 64-bit-versions
-            cmake_args: -DCMAKE_CXX_FLAGS=-DUSE_64_BIT=1
-          - name: debug
-            cmake_args: -DCMAKE_BUILD_TYPE=Debug
-          - name: simd-fallback
-            cmake_args: -DUSE_SIMD_FALLBACK=ON
-          - name: gcc
-            cmake_args: -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++
-    runs-on: ubuntu-latest-amd64
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/cache@v4
-        with:
-          path: /var/cache/apt/archives
-          key: apt-amd64-${{ hashFiles('.gitea/workflows/ci.yml') }}
-
-      - name: Install dependencies
-        run: |
-          . /etc/os-release
-          wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
-          echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
-          sudo apt-get update -qq
-          sudo apt-get install -y \
-            build-essential ccache clang-21 cmake gcc g++ \
-            libc6-dbg llvm-21 lld-21 mold ninja-build python3 valgrind zstd
-          sudo curl -Ls "https://dl.min.io/client/mc/release/linux-amd64/mc" \
-            -o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
-          for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do
-            sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
-          done
-
-      - uses: actions/cache@v4
-        with:
-          path: .ccache
-          key: ccache-${{ matrix.name }}-${{ gitea.sha }}
-          restore-keys: |
-            ccache-${{ matrix.name }}-
-
-      - name: Build
-        run: |
-          export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache"
-          rm -rf build
-          cmake -S . -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ${{ matrix.cmake_args }}
-          ninja -C build
-          ccache -s
-
-      - name: Test
-        run: |
-          cd build
-          ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 -T Test -j "$(nproc)" --timeout 90 > /dev/null
-
-      - name: Upload test results to MinIO
-        if: always()
-        env:
-          MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }}
-          MC_HOST_minio: https://${{ secrets.MINIO_ACCESS_KEY }}:${{ secrets.MINIO_SECRET_KEY }}@minio.weaselab.dev
-        run: |
-          if [ -z "$MINIO_ACCESS_KEY" ]; then
-            echo "MinIO credentials not configured; skipping upload"
-            exit 0
-          fi
-          zstd build/Testing/*/Test.xml
-          mc cp build/Testing/*/Test.xml.zst "minio/jenkins/conflict-set/${{ gitea.run_number }}/${{ matrix.name }}/"
-
-      - name: Test summary
-        if: always()
-        run: |
-          python3 ctest_summary.py build/Testing/*/Test.xml \
-            --link "https://minio.weaselab.dev/jenkins/conflict-set/${{ gitea.run_number }}/${{ matrix.name }}/Test.xml.zst" \
-            | tee -a "$GITHUB_STEP_SUMMARY"
-
-  release:
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - runner: ubuntu-latest-amd64
-            arch: amd64
-          - runner: ubuntu-latest-arm64
-            arch: arm64
-    runs-on: ${{ matrix.runner }}
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/cache@v4
-        with:
-          path: /var/cache/apt/archives
-          key: apt-${{ matrix.arch }}-${{ hashFiles('.gitea/workflows/ci.yml') }}
-
-      - name: Install dependencies
-        run: |
-          . /etc/os-release
-          wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
-          echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
-          sudo apt-get update -qq
-          sudo apt-get install -y \
-            biber build-essential ccache clang-21 cmake devscripts \
-            latexmk libc6-dbg llvm-21 lld-21 mold ninja-build rpm \
-            texlive-bibtex-extra texlive-fonts-recommended \
-            texlive-latex-extra texlive-pictures valgrind zstd
-          sudo curl -Ls "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/mc" \
-            -o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
-          for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do
-            sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
-          done
-
-      - uses: actions/cache@v4
-        with:
-          path: .ccache
-          key: ccache-release-${{ matrix.arch }}-${{ gitea.sha }}
-          restore-keys: |
-            ccache-release-${{ matrix.arch }}-
-
-      - name: Build
-        run: |
-          export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache"
-          rm -rf build
-          cmake -S . -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_FLAGS=-DNVALGRIND
-          ninja -C build
-          ccache -s
-
-      - name: Test
-        run: |
-          cd build
-          # On arm64, valgrind needs the MAKE_MEM_DEFINED client requests for
-          # https://git.weaselab.dev/weaselab/conflict-set/issues/39, but this
-          # build has -DNVALGRIND, which compiles them out. Skip valgrind
-          # tests here; they run annotated in the test job.
-          ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 ${{ matrix.arch == 'arm64' && '-E valgrind' || '' }} -T Test -j "$(nproc)" --timeout 90 > /dev/null
-
-      - name: Package
-        run: |
-          cd build
-          cpack -G DEB
-          cpack -G RPM
-
-      - name: Build paper
-        if: matrix.arch == 'amd64'
-        run: |
-          cd paper
-          make
-
-      - name: Upload artifacts to MinIO
-        if: always()
-        shell: bash
-        env:
-          MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }}
-          MC_HOST_minio: https://${{ secrets.MINIO_ACCESS_KEY }}:${{ secrets.MINIO_SECRET_KEY }}@minio.weaselab.dev
-        run: |
-          if [ -z "$MINIO_ACCESS_KEY" ]; then
-            echo "MinIO credentials not configured; skipping upload"
-            exit 0
-          fi
-          dest="minio/jenkins/conflict-set/${{ gitea.run_number }}/release-${{ matrix.arch }}/"
-          zstd build/Testing/*/Test.xml
-          mc cp build/Testing/*/Test.xml.zst "$dest"
-          if compgen -G "build/*.deb" > /dev/null; then
-            mc cp build/*.deb "$dest"
-          fi
-          if compgen -G "build/*.rpm" > /dev/null; then
-            mc cp build/*.rpm "$dest"
-          fi
-          if compgen -G "paper/*.pdf" > /dev/null; then
-            mc cp paper/*.pdf "$dest"
-          fi
-
-      - name: Test summary
-        if: always()
-        run: |
-          python3 ctest_summary.py build/Testing/*/Test.xml \
-            --link "https://minio.weaselab.dev/jenkins/conflict-set/${{ gitea.run_number }}/release-${{ matrix.arch }}/Test.xml.zst" \
-            | tee -a "$GITHUB_STEP_SUMMARY"
-
-  coverage:
-    runs-on: ubuntu-latest-amd64
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/cache@v4
-        with:
-          path: /var/cache/apt/archives
-          key: apt-amd64-${{ hashFiles('.gitea/workflows/ci.yml') }}
-
-      - name: Install dependencies
-        run: |
-          . /etc/os-release
-          wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
-          echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
-          sudo apt-get update -qq
-          sudo apt-get install -y \
-            build-essential ccache clang-21 cmake gcovr \
-            libc6-dbg llvm-21 lld-21 mold ninja-build python3 valgrind zstd
-          sudo curl -Ls "https://dl.min.io/client/mc/release/linux-amd64/mc" \
-            -o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
-          for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do
-            sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
-          done
-
-      - uses: actions/cache@v4
-        with:
-          path: .ccache
-          key: ccache-coverage-${{ gitea.sha }}
-          restore-keys: |
-            ccache-coverage-
-
-      - name: Build
-        run: |
-          export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache"
-          rm -rf build
-          cmake -S . -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-            -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage \
-            -DCMAKE_BUILD_TYPE=Debug -DDISABLE_TSAN=ON
-          ninja -C build
-          ccache -s
-
-      - name: Test
-        run: |
-          cd build
-          ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 -T Test -j "$(nproc)" --timeout 90 > /dev/null
-
-      - name: Coverage report
-        shell: bash
-        run: |
-          gcov_args=(-f ConflictSet.cpp -f LongestCommonPrefix.h -f Metrics.h
-                     --gcov-executable "llvm-cov gcov" --exclude-noncode-lines)
-          gcovr "${gcov_args[@]}" --cobertura > build/coverage.xml
-          gcovr "${gcov_args[@]}"
-          mkdir -p build/coverage_html
-          gcovr "${gcov_args[@]}" --html-details build/coverage_html/index.html
-          gcovr "${gcov_args[@]}" --fail-under-line 100 > /dev/null
-
-      - name: Upload results to MinIO
-        if: always()
-        env:
-          MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }}
-          MC_HOST_minio: https://${{ secrets.MINIO_ACCESS_KEY }}:${{ secrets.MINIO_SECRET_KEY }}@minio.weaselab.dev
-        run: |
-          if [ -z "$MINIO_ACCESS_KEY" ]; then
-            echo "MinIO credentials not configured; skipping upload"
-            exit 0
-          fi
-          dest="minio/jenkins/conflict-set/${{ gitea.run_number }}/coverage/"
-          zstd build/Testing/*/Test.xml
-          mc cp build/Testing/*/Test.xml.zst "$dest"
-          if [ -e build/coverage.xml ]; then
-            mc cp build/coverage.xml "$dest"
-          fi
-          if [ -d build/coverage_html ]; then
-            mc cp -r build/coverage_html "$dest"
-          fi
-
-      - name: Test summary
-        if: always()
-        run: |
-          python3 ctest_summary.py build/Testing/*/Test.xml \
-            --link "https://minio.weaselab.dev/jenkins/conflict-set/${{ gitea.run_number }}/coverage/Test.xml.zst" \
-            | tee -a "$GITHUB_STEP_SUMMARY"
-          echo "" | tee -a "$GITHUB_STEP_SUMMARY"
-          echo "📊 [Coverage report](https://minio.weaselab.dev/jenkins/conflict-set/${{ gitea.run_number }}/coverage/coverage_html/index.html)" | tee -a "$GITHUB_STEP_SUMMARY"

Bench.cpp

@@ -37,16 +37,15 @@ ConflictSet::ReadRange singleton(Arena &arena, TrivialSpan key) {
 }
 
 ConflictSet::ReadRange prefixRange(Arena &arena, TrivialSpan key) {
-  int index = key.size() - 1;
-  for (; index >= 0; index--)
-    if (key[index] != 255)
+  int index;
+  for (index = key.size() - 1; index >= 0; index--)
+    if ((key[index]) != 255)
       break;
 
   // Must not be called with a string that consists only of zero or more '\xff'
-  // bytes, or with an empty string (which has no finite upper bound).
+  // bytes.
   if (index < 0) {
     assert(false);
-    std::abort();
   }
 
   uint8_t *buf = new (arena) uint8_t[index + 1];

CMakeLists.txt

@@ -32,12 +32,8 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
 endif()
 
 add_compile_options(
-  -Werror=switch-enum
-  -Wswitch-enum
-  -Wunused-variable
-  -fPIC
-  -fdata-sections
-  -ffunction-sections
+  # -Werror=switch-enum
+  -Wswitch-enum -Wunused-variable -fPIC -fdata-sections -ffunction-sections
   -fno-jump-tables # https://github.com/llvm/llvm-project/issues/54247
 )
 
@@ -53,7 +49,6 @@ endif()
 
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
   add_compile_options("-Wno-maybe-uninitialized")
-  add_compile_options("-Wno-maybe-musttail-local-addr")
 endif()
 
 if(NOT APPLE)
@@ -61,16 +56,12 @@ if(NOT APPLE)
   add_compile_options(-g -fno-omit-frame-pointer)
 endif()
 
-set(relro_flags "LINKER:-z,relro,-z,now,-z,noexecstack")
-set(full_relro_flags "-pie;${relro_flags}")
+set(full_relro_flags "-pie;LINKER:-z,relro,-z,now,-z,noexecstack")
 cmake_push_check_state()
 list(APPEND CMAKE_REQUIRED_LINK_OPTIONS ${full_relro_flags})
 check_cxx_source_compiles("int main(){}" HAS_FULL_RELRO FAIL_REGEX "warning:")
 if(HAS_FULL_RELRO)
-  # -pie only applies to executables; passing it when linking a shared library
-  # makes the driver pull in Scrt1.o, which requires main.
-  add_link_options("$<$<STREQUAL:$<TARGET_PROPERTY:TYPE>,EXECUTABLE>:-pie>"
-                   ${relro_flags})
+  add_link_options(${full_relro_flags})
 endif()
 cmake_pop_check_state()
 
@@ -139,7 +130,6 @@ add_custom_command(
   COMMAND_EXPAND_LISTS)
 
 add_library(${PROJECT_NAME} SHARED ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o)
-add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-object)
 set_target_properties(
   ${PROJECT_NAME} PROPERTIES LIBRARY_OUTPUT_DIRECTORY
                              "${CMAKE_CURRENT_BINARY_DIR}/radix_tree")
@@ -156,7 +146,6 @@ if(HAS_VERSION_SCRIPT)
 endif()
 
 add_library(${PROJECT_NAME}-static STATIC ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o)
-add_dependencies(${PROJECT_NAME}-static ${PROJECT_NAME}-object)
 if(CMAKE_BUILD_TYPE STREQUAL Debug)
   set_target_properties(${PROJECT_NAME}-static PROPERTIES LINKER_LANGUAGE CXX)
 else()
@@ -385,30 +374,9 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
   if(NOT CMAKE_CROSSCOMPILING)
     find_program(HARDENING_CHECK hardening-check)
     if(HARDENING_CHECK)
-      # Not all versions of hardening-check support the same options, so query
-      # the help output before using architecture-specific skips.
-      execute_process(
-        COMMAND ${HARDENING_CHECK} --help
-        OUTPUT_VARIABLE hardening_check_help
-        ERROR_VARIABLE hardening_check_help
-        OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE)
-      set(hardening_check_arch_flags "")
-      # Control flow integrity (CET) is x86-only and branch protection (PAC/BTI)
-      # is arm64-only, so ignore whichever doesn't apply.
-      if(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64 OR CMAKE_SYSTEM_PROCESSOR
-                                                    STREQUAL arm64)
-        if(hardening_check_help MATCHES "nocfprotection")
-          list(APPEND hardening_check_arch_flags --nocfprotection)
-        endif()
-      else()
-        if(hardening_check_help MATCHES "nobranchprotection")
-          list(APPEND hardening_check_arch_flags --nobranchprotection)
-        endif()
-      endif()
-      add_test(
-        NAME hardening_check
-        COMMAND ${HARDENING_CHECK} $<TARGET_FILE:${PROJECT_NAME}> --nofortify
-                --nostackprotector ${hardening_check_arch_flags})
+      add_test(NAME hardening_check
+               COMMAND ${HARDENING_CHECK} $<TARGET_FILE:${PROJECT_NAME}>
+                       --nofortify --nostackprotector)
     endif()
   endif()
 

Dockerfile

@@ -8,36 +8,48 @@ RUN chmod -R 777 /tmp
 RUN apt-get update
 RUN apt-get upgrade -y
 RUN TZ=America/Los_Angeles DEBIAN_FRONTEND=noninteractive apt-get install -y \
-    biber \
+    binutils-aarch64-linux-gnu \
     build-essential \
     ccache \
-    clang \
     cmake \
     curl \
     devscripts \
+    g++-aarch64-linux-gnu \
     gcovr \
     git \
-    latexmk \
+    gnupg \
     libc6-dbg \
-    llvm \
+    lsb-release \
     mold \
     ninja-build \
-    nodejs \
     pre-commit \
     python3-requests \
+    qemu-user \
     rpm \
-    texlive-bibtex-extra \
-    texlive-fonts-recommended \
-    texlive-latex-extra \
-    texlive-pictures \
-    valgrind \
+    software-properties-common \
+    texlive-full \
     wget \
     zstd
 
-# MinIO client, for uploading build artifacts
-RUN curl -Ls "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/mc" \
-    -o /usr/local/bin/mc && chmod +x /usr/local/bin/mc
+# Install recent valgrind from source
+RUN curl -Ls https://sourceware.org/pub/valgrind/valgrind-3.22.0.tar.bz2 -o valgrind.tar.bz2 && \
+    echo "c811db5add2c5f729944caf47c4e7a65dcaabb9461e472b578765dd7bf6d2d4c  valgrind.tar.bz2" > valgrind-sha.txt && \
+    sha256sum --quiet -c valgrind-sha.txt && \
+    mkdir valgrind && \
+    tar --strip-components 1 --no-same-owner --no-same-permissions --directory valgrind -xjf valgrind.tar.bz2 && \
+    cd valgrind && \
+    ./configure --enable-only64bit --enable-lto && \
+    make -j`nproc` && \
+    make install && \
+    cd .. && \
+    rm -rf /tmp/*
 
+# Recent clang
+RUN wget https://apt.llvm.org/llvm.sh && chmod +x ./llvm.sh && ./llvm.sh 20
+
+RUN apt-get -y install clang llvm
+
+# Set after building valgrind, which doesn't build with clang for some reason
 ENV CC=clang
 ENV CXX=clang++
 

HashTable.cpp

@@ -96,9 +96,7 @@ void ConflictSet::setOldestVersion(int64_t oldestVersion) {
   return impl->setOldestVersion(oldestVersion);
 }
 
-// The hash_table implementation does not track memory usage, so return 0 to
-// satisfy the API contract that getBytes() returns a non-negative value.
-int64_t ConflictSet::getBytes() const { return 0; }
+int64_t ConflictSet::getBytes() const { return -1; }
 
 void ConflictSet::getMetricsV1(MetricsV1 **metrics, int *count) const {
   *metrics = nullptr;
@@ -121,13 +119,7 @@ ConflictSet::ConflictSet(ConflictSet &&other) noexcept
     : impl(std::exchange(other.impl, nullptr)) {}
 
 ConflictSet &ConflictSet::operator=(ConflictSet &&other) noexcept {
-  if (this != &other) {
-    if (impl) {
-      impl->~Impl();
-      safe_free(impl, sizeof(Impl));
-    }
-    impl = std::exchange(other.impl, nullptr);
-  }
+  impl = std::exchange(other.impl, nullptr);
   return *this;
 }
 
@@ -163,10 +155,7 @@ __attribute__((__visibility__("default"))) void ConflictSet_destroy(void *cs) {
 }
 __attribute__((__visibility__("default"))) int64_t
 ConflictSet_getBytes(void *cs) {
-  (void)cs;
-  // The hash_table implementation does not track memory usage, so return 0 to
-  // satisfy the API contract that ConflictSet_getBytes returns a non-negative
-  // value.
-  return 0;
+  using Impl = ConflictSet::Impl;
+  return -1;
 }
 }

Jenkinsfile

@@ -0,0 +1,151 @@
+def CleanBuildAndTest(String cmakeArgs) {
+    sh """
+    export CCACHE_DIR=/ccache
+    rm -rf build
+    mkdir build
+    cd build
+    cmake .. -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ${cmakeArgs} 
+    ninja
+    ccache -s
+    """
+    catchError {
+        sh '''
+        cd build
+        ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 -T Test -j `nproc` --timeout 90 > /dev/null
+        zstd Testing/*/Test.xml
+        '''
+    }
+    xunit tools: [CTest(pattern: 'build/Testing/*/Test.xml')], skipPublishingChecks: false
+    minio bucket: 'jenkins', credentialsId: 'jenkins-minio', excludes: '', host: 'minio.weaselab.dev', includes: 'build/Testing/*/Test.xml.zst', targetFolder: '${JOB_NAME}/${BUILD_NUMBER}/${STAGE_NAME}/'
+}
+
+pipeline {
+    agent any
+    stages {
+        stage('Pre-commit') {
+            agent {
+                dockerfile {
+                    args '-v /home/jenkins/ccache:/ccache'
+                    reuseNode true
+                }
+            }
+            steps {
+                script {
+                    env.HOME = env.WORKSPACE
+                }
+                sh 'pre-commit run --all-files --show-diff-on-failure'
+            }
+        }
+        stage('64 bit versions') {
+            agent {
+                dockerfile {
+                    args '-v /home/jenkins/ccache:/ccache'
+                    reuseNode true
+                }
+            }
+            steps {
+                CleanBuildAndTest("-DCMAKE_CXX_FLAGS=-DUSE_64_BIT=1")
+            }
+        }
+        stage('Debug') {
+            agent {
+                dockerfile {
+                    args '-v /home/jenkins/ccache:/ccache'
+                    reuseNode true
+                }
+            }
+            steps {
+                CleanBuildAndTest("-DCMAKE_BUILD_TYPE=Debug")
+            }
+        }
+        stage('SIMD fallback') {
+            agent {
+                dockerfile {
+                    args '-v /home/jenkins/ccache:/ccache'
+                    reuseNode true
+                }
+            }
+            steps {
+                CleanBuildAndTest("-DUSE_SIMD_FALLBACK=ON")
+            }
+        }
+        stage('Release [clang]') {
+            agent {
+                dockerfile {
+                    args '-v /home/jenkins/ccache:/ccache'
+                    reuseNode true
+                }
+            }
+            steps {
+                CleanBuildAndTest("-DCMAKE_CXX_FLAGS=-DNVALGRIND")
+                recordIssues(tools: [clang()])
+                sh '''
+                cd build
+                cpack -G DEB
+                cpack -G RPM
+                '''
+                sh '''
+                cd paper
+                make
+                '''
+                minio bucket: 'jenkins', credentialsId: 'jenkins-minio', excludes: '', host: 'minio.weaselab.dev', includes: 'build/*.deb,build/*.rpm,paper/*.pdf', targetFolder: '${JOB_NAME}/${BUILD_NUMBER}/${STAGE_NAME}/'
+            }
+        }
+        stage('gcc') {
+            agent {
+                dockerfile {
+                    args '-v /home/jenkins/ccache:/ccache'
+                    reuseNode true
+                }
+            }
+            steps {
+                CleanBuildAndTest("-DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++")
+                recordIssues(tools: [gcc()])
+            }
+        }
+        stage('Release [clang,aarch64]') {
+            agent {
+                dockerfile {
+                    args '-v /home/jenkins/ccache:/ccache'
+                    reuseNode true
+                }
+            }
+            steps {
+                CleanBuildAndTest("-DCMAKE_TOOLCHAIN_FILE=../aarch64-toolchain.cmake -DCMAKE_CXX_FLAGS=-DNVALGRIND")
+                sh '''
+                cd build
+                cpack -G DEB
+                cpack -G RPM
+                '''
+                minio bucket: 'jenkins', credentialsId: 'jenkins-minio', excludes: '', host: 'minio.weaselab.dev', includes: 'build/*.deb,build/*.rpm', targetFolder: '${JOB_NAME}/${BUILD_NUMBER}/${STAGE_NAME}'
+            }
+        }
+        stage('Coverage') {
+            agent {
+                dockerfile {
+                    args '-v /home/jenkins/ccache:/ccache'
+                    reuseNode true
+                }
+            }
+            steps {
+                script {
+                    gcov_args = "-f ConflictSet.cpp -f LongestCommonPrefix.h -f Metrics.h --gcov-executable 'llvm-cov gcov' --exclude-noncode-lines"
+                }
+                CleanBuildAndTest("-DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DCMAKE_BUILD_TYPE=Debug -DDISABLE_TSAN=ON")
+                sh """
+                gcovr ${gcov_args} --cobertura > build/coverage.xml
+                """
+                recordCoverage qualityGates: [[criticality: 'NOTE', metric: 'MODULE']], tools: [[parser: 'COBERTURA', pattern: 'build/coverage.xml']]
+                sh """
+                gcovr ${gcov_args}
+                gcovr ${gcov_args} --fail-under-line 100 > /dev/null
+                """
+            }
+        }
+    }
+    post {
+        always {
+            emailext mimeType: 'text/html', body: '${SCRIPT, template="groovy-html.template"}', subject: "${env.JOB_NAME} - Build# ${env.BUILD_NUMBER} - ${currentBuild.currentResult}", to: 'andrew@weaselab.dev'
+        }
+    }
+}

RealDataBench.cpp

@@ -1,6 +1,5 @@
 #include <ConflictSet.h>
 
-#include <algorithm>
 #include <cerrno>
 #include <chrono>
 #include <cstdio>
@@ -78,10 +77,10 @@ int main(int argc, const char **argv) {
       begin = end + 1;
       end = (uint8_t *)memchr(begin, '\n', size);
 
-      if (line.size() >= 2 && line[0] == 'P') {
-        write = line.subspan(2, line.size() - 2);
-      } else if (line.size() >= 2 && line[0] == 'L') {
-        reads.push_back(line.subspan(2, line.size() - 2));
+      if (line.size() > 0 && line[0] == 'P') {
+        write = line.subspan(2, line.size());
+      } else if (line.size() > 0 && line[0] == 'L') {
+        reads.push_back(line.subspan(2, line.size()));
       } else if (line.empty()) {
         {
           readRanges.resize(reads.size());
@@ -91,7 +90,7 @@ int main(int argc, const char **argv) {
             iter->begin.len = read.size();
             checkBytes += read.size();
             iter->end.len = 0;
-            iter->readVersion = std::max<int64_t>(0, version - 100);
+            iter->readVersion = version - 100;
             ++iter;
           }
         }
@@ -122,7 +121,7 @@ int main(int argc, const char **argv) {
         }
 
         timer = now();
-        cs.setOldestVersion(std::max<int64_t>(0, version - 10000));
+        cs.setOldestVersion(version - 10000);
         gcTime += now() - timer;
       }
     }

ServerBench.cpp

@@ -12,7 +12,6 @@
 #include <sys/ioctl.h>
 #include <sys/resource.h>
 #include <sys/socket.h>
-#include <sys/syscall.h>
 #include <sys/types.h>
 #include <sys/uio.h>
 #include <thread>

SkipList.cpp

@@ -38,9 +38,6 @@ std::span<const uint8_t> keyAfter(Arena &arena, std::span<const uint8_t> key) {
 
 std::span<const uint8_t> copyToArena(Arena &arena,
                                      std::span<const uint8_t> key) {
-  if (key.size() == 0) {
-    return std::span<const uint8_t>();
-  }
   auto result = std::span<uint8_t>(new (arena) uint8_t[key.size()], key.size());
   memcpy(result.data(), key.data(), key.size());
   return result;
@@ -100,7 +97,7 @@ force_inline bool getCharacter(const KeyInfo &ki, int character,
 
 bool operator<(const KeyInfo &lhs, const KeyInfo &rhs) {
   int i = std::min(lhs.key.size(), rhs.key.size());
-  int c = i > 0 ? memcmp(lhs.key.data(), rhs.key.data(), i) : 0;
+  int c = memcmp(lhs.key.data(), rhs.key.data(), i);
   if (c != 0)
     return c < 0;
 
@@ -290,8 +287,7 @@ private:
 
   static force_inline bool less(const uint8_t *a, int aLen, const uint8_t *b,
                                 int bLen) {
-    int n = std::min(aLen, bLen);
-    int c = n > 0 ? memcmp(a, b, n) : 0;
+    int c = memcmp(a, b, std::min(aLen, bLen));
     if (c < 0)
       return true;
     if (c > 0)
@@ -758,8 +754,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
 
   void addWrites(const ConflictSet::WriteRange *writes, int count,
                  int64_t writeVersion) {
-    auto points = std::vector<KeyInfo>();
-    points.reserve(count * 2);
+    auto points = std::vector<KeyInfo>(count * 2);
     Arena arena;
 
     for (int r = 0; r < count; r++) {
@@ -986,12 +981,7 @@ ConflictSet::ConflictSet(ConflictSet &&other) noexcept
     : impl(std::exchange(other.impl, nullptr)) {}
 
 ConflictSet &ConflictSet::operator=(ConflictSet &&other) noexcept {
-  if (this != &other) {
-    if (impl) {
-      internal_destroy(impl);
-    }
-    impl = std::exchange(other.impl, nullptr);
-  }
+  impl = std::exchange(other.impl, nullptr);
   return *this;
 }
 

conflict_set.py

@@ -27,37 +27,23 @@ class Result(enum.Enum):
     TOO_OLD = 2
 
 
-def _make_key(buf: bytes) -> tuple[_Key, bytearray]:
-    """Create a _Key and a backing bytearray that must be kept alive."""
-    backing = bytearray(buf)
-    array = (ctypes.c_ubyte * len(backing)).from_buffer(backing)
-    return _Key(array, len(array)), backing
-
-
 def write(begin: bytes, end: Optional[bytes] = None) -> WriteRange:
-    begin_key, begin_buf = _make_key(begin)
+    b = (ctypes.c_ubyte * len(begin)).from_buffer(bytearray(begin))
+
     if end is None:
-        end_key = _Key((ctypes.c_ubyte * 0)(), 0)
-        end_buf = None
+        e = (ctypes.c_ubyte * 0)()
     else:
-        end_key, end_buf = _make_key(end)
-    result = WriteRange(begin_key, end_key)
-    result._begin_buf = begin_buf
-    result._end_buf = end_buf
-    return result
+        e = (ctypes.c_ubyte * len(end)).from_buffer(bytearray(end))
+    return WriteRange(_Key(b, len(b)), _Key(e, len(e)))
 
 
 def read(version: int, begin: bytes, end: Optional[bytes] = None) -> ReadRange:
-    begin_key, begin_buf = _make_key(begin)
+    b = (ctypes.c_ubyte * len(begin)).from_buffer(bytearray(begin))
     if end is None:
-        end_key = _Key((ctypes.c_ubyte * 0)(), 0)
-        end_buf = None
+        e = (ctypes.c_ubyte * 0)()
     else:
-        end_key, end_buf = _make_key(end)
-    result = ReadRange(begin_key, end_key, version)
-    result._begin_buf = begin_buf
-    result._end_buf = end_buf
-    return result
+        e = (ctypes.c_ubyte * len(end)).from_buffer(bytearray(end))
+    return ReadRange(_Key(b, len(b)), _Key(e, len(e)), version)
 
 
 class ConflictSet:
@@ -102,7 +88,6 @@ class ConflictSet:
             ctypes.POINTER(ctypes.c_int),
             ctypes.c_int,
         )
-        self._lib.ConflictSet_check.restype = None
 
         self._lib.ConflictSet_addWrites.argtypes = (
             ctypes.c_void_p,
@@ -110,16 +95,13 @@ class ConflictSet:
             ctypes.c_int,
             ctypes.c_int64,
         )
-        self._lib.ConflictSet_addWrites.restype = None
 
         self._lib.ConflictSet_setOldestVersion.argtypes = (
             ctypes.c_void_p,
             ctypes.c_int64,
         )
-        self._lib.ConflictSet_setOldestVersion.restype = None
 
         self._lib.ConflictSet_destroy.argtypes = (ctypes.c_void_p,)
-        self._lib.ConflictSet_destroy.restype = None
 
         self._lib.ConflictSet_getBytes.argtypes = (ctypes.c_void_p,)
         self._lib.ConflictSet_getBytes.restype = ctypes.c_int64

ctest_summary.py

@@ -1,76 +0,0 @@
-#!/usr/bin/env python3
-"""Summarize a CTest Test.xml as markdown.
-
-Intended for $GITHUB_STEP_SUMMARY in CI, where only the first few failures
-are shown inline (pass --link to point at the full Test.xml). Also reusable
-locally to print every failure from a downloaded Test.xml with --all.
-"""
-
-import argparse
-import base64
-import gzip
-import xml.etree.ElementTree as ET
-
-# Failure output is truncated to this many trailing characters, which is
-# usually enough to include e.g. an ASan report's summary.
-OUTPUT_TAIL_CHARS = 3000
-
-
-def test_output(test):
-    value = test.find("./Results/Measurement/Value")
-    if value is None or value.text is None:
-        return ""
-    text = value.text
-    if value.get("encoding") == "base64":
-        raw = base64.b64decode(text)
-        if value.get("compression") == "gzip":
-            raw = gzip.decompress(raw)
-        text = raw.decode(errors="replace")
-    return text
-
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("test_xml", help="path to a ctest Testing/*/Test.xml")
-    parser.add_argument(
-        "--inline",
-        type=int,
-        default=5,
-        help="how many failures to show inline (default 5)",
-    )
-    parser.add_argument("--all", action="store_true", help="show every failure inline")
-    parser.add_argument(
-        "--link", help="URL of the full Test.xml, linked when failures are elided"
-    )
-    args = parser.parse_args()
-
-    testing = ET.parse(args.test_xml).getroot().find("Testing")
-    tests = testing.findall("Test")
-    failed = [t for t in tests if t.get("Status") == "failed"]
-    notrun = sum(1 for t in tests if t.get("Status") == "notrun")
-
-    if not failed:
-        print(f"✅ All {len(tests) - notrun} tests passed")
-    else:
-        print(f"❌ {len(failed)} of {len(tests)} tests failed\n")
-        shown = failed if args.all else failed[: args.inline]
-        for test in shown:
-            name = test.findtext("Name")
-            output = test_output(test)[-OUTPUT_TAIL_CHARS:].strip()
-            print(f"<details><summary><code>{name}</code></summary>\n")
-            print("````")
-            print(output)
-            print("````")
-            print("</details>\n")
-        remaining = len(failed) - len(shown)
-        if remaining > 0:
-            more = f"… and {remaining} more"
-            if args.link:
-                more += f" — full list in [Test.xml]({args.link})"
-            print(more)
-    if notrun:
-        print(f"\n⚠️ {notrun} tests not run")
-
-
-if __name__ == "__main__":
-    main()

include/ConflictSet.h

@@ -88,8 +88,7 @@ struct __attribute__((__visibility__("default"))) ConflictSet {
 
   ~ConflictSet();
 
-  /** Returns the total bytes in use by this ConflictSet. Implementations that
-   * do not track memory usage return 0. */
+  /** Returns the total bytes in use by this ConflictSet */
   int64_t getBytes() const;
 
   /** Experimental! */
@@ -133,13 +132,6 @@ struct __attribute__((__visibility__("default"))) ConflictSet {
 
 private:
   Impl *impl;
-#if __cplusplus <= 199711L
-  /* Declared private and left undefined to prevent copying in C++98/C++03.
-     The compiler would otherwise implicitly generate public copy operations,
-     which share the opaque Impl* and cause a double-free. */
-  ConflictSet(const ConflictSet &);
-  ConflictSet &operator=(const ConflictSet &);
-#endif
 };
 } /* namespace weaselab */
 
@@ -219,8 +211,7 @@ ConflictSet *ConflictSet_create(int64_t oldestVersion);
 
 void ConflictSet_destroy(ConflictSet *cs);
 
-/** Returns the total bytes in use by this ConflictSet. Implementations that
- * do not track memory usage return 0. */
+/** Returns the total bytes in use by this ConflictSet */
 int64_t ConflictSet_getBytes(const ConflictSet *cs);
 
 #endif

test_conflict_set.py

@@ -57,53 +57,6 @@ def test_conflict_set():
         assert cs.check(read(0, key), read(1, key)) == [Result.TOO_OLD, Result.COMMIT]
 
 
-def test_hash_table_getBytes():
-    # Regression test for issue #62: the hash_table implementation is
-    # point-query only and does not track memory usage, but getBytes() must
-    # still return a non-negative value rather than -1.
-    with ConflictSet(0, build_dir=build_dir, implementation="hash_table") as cs:
-        assert cs.getBytes() == 0
-        cs.addWrites(1, write(b"key"))
-        assert cs.getBytes() >= 0
-        assert cs.check(read(0, b"key")) == [Result.CONFLICT]
-
-
-def test_write_read_without_outer_reference():
-    # Regression test for issue #42: WriteRange/ReadRange must keep their
-    # backing key buffers alive, because the C library reads the pointer
-    # stored in _Key while addWrites/check run.
-    with DebugConflictSet() as cs:
-        # The bytes literal is not referenced after this expression.
-        cs.addWrites(1, write(b"key"))
-        assert cs.check(read(0, b"key")) == [Result.CONFLICT]
-
-        cs.addWrites(2, write(b"a", b"z"))
-        assert cs.check(read(1, b"a", b"z")) == [Result.CONFLICT]
-        assert cs.check(read(1, b"b")) == [Result.CONFLICT]
-        assert cs.check(read(1, b"0")) == [Result.COMMIT]
-
-
-def test_range_keeps_key_buffers_alive():
-    # Verify the fix for issue #42: returned range objects must retain a
-    # reference to the backing bytearray so the C pointer stays valid after
-    # the helper returns.
-    w = write(b"key")
-    assert w._begin_buf == bytearray(b"key")
-    assert w._end_buf is None
-
-    w2 = write(b"a", b"z")
-    assert w2._begin_buf == bytearray(b"a")
-    assert w2._end_buf == bytearray(b"z")
-
-    r = read(0, b"key")
-    assert r._begin_buf == bytearray(b"key")
-    assert r._end_buf is None
-
-    r2 = read(1, b"a", b"z")
-    assert r2._begin_buf == bytearray(b"a")
-    assert r2._end_buf == bytearray(b"z")
-
-
 def test_update_zero_should_commit():
     with DebugConflictSet() as cs1:
         with DebugConflictSet() as cs2:
@@ -115,7 +68,7 @@ def test_update_zero_should_commit():
             for i in range(256 - 17, 256):
                 cs2.addWrites(int(1), write(bytes([i])))
             # Scan until first point write
-            assert cs2.check(read(0, b"\x00", bytes([256 - 17]))) == [Result.COMMIT]
+            cs2.check(read(0, b"\x00", bytes([256 - 17])))
 
 
 def test_update_zero_should_conflict():
@@ -128,7 +81,7 @@ def test_update_zero_should_conflict():
             # "zero" is now 2**31 + 100
             cs1.addWrites(2**32 + 101, write(b"", b"\x02"), write(b"\x01"))
             # rangeVersion of \x01 is now 2**31 + 100 ("max" of (2**31 + 100, 2**32 + 101))
-            assert cs1.check(read(2**32 + 1, b"\x00")) == [Result.CONFLICT]
+            cs1.check(read(2**32 + 1, b"\x00"))
             # but 2**32 + 1 ">" 2**31 + 100 , and it incorrectly commits