diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..b255702 --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,306 @@ +name: CI + +on: [push, pull_request] + +env: + CC: clang + CXX: clang++ + +jobs: + pre-commit: + runs-on: ubuntu-latest-amd64 + steps: + - uses: actions/checkout@v4 + + - uses: actions/cache@v4 + with: + path: /var/cache/apt/archives + key: apt-amd64-${{ hashFiles('.gitea/workflows/ci.yml') }} + + - name: Install dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y git nodejs pre-commit + + - uses: actions/cache@v4 + with: + path: ~/.cache/pre-commit + key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} + + - name: Run pre-commit + run: | + git config --global --add safe.directory "$PWD" + pre-commit run --all-files --show-diff-on-failure + + test: + strategy: + fail-fast: false + matrix: + include: + - name: 64-bit-versions + cmake_args: -DCMAKE_CXX_FLAGS=-DUSE_64_BIT=1 + - name: debug + cmake_args: -DCMAKE_BUILD_TYPE=Debug + - name: simd-fallback + cmake_args: -DUSE_SIMD_FALLBACK=ON + - name: gcc + cmake_args: -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ + runs-on: ubuntu-latest-amd64 + steps: + - uses: actions/checkout@v4 + + - uses: actions/cache@v4 + with: + path: /var/cache/apt/archives + key: apt-amd64-${{ hashFiles('.gitea/workflows/ci.yml') }} + + - name: Install common dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y \ + build-essential ccache cmake gcc g++ libc6-dbg \ + ninja-build python3 valgrind zstd + sudo curl -Ls "https://minio.weaselab.dev/public/$(uname -m)/mc.RELEASE.2025-08-13T08-35-41Z" \ + -o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc + + - name: Install LLVM toolchain + if: matrix.name != 'gcc' + run: | + . /etc/os-release + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list + sudo apt-get update -qq + sudo apt-get install -y \ + clang-21 llvm-21 lld-21 mold + for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do + sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100 + done + + - name: Build + run: | + export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache" + rm -rf build + cmake -S . -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ${{ matrix.cmake_args }} + ninja -C build + ccache -s + + - name: Test + run: | + cd build + ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 -T Test -j "$(nproc)" --timeout 90 > /dev/null + + - name: Upload test results to MinIO + if: always() + env: + MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }} + MC_HOST_minio: https://${{ secrets.MINIO_ACCESS_KEY }}:${{ secrets.MINIO_SECRET_KEY }}@minio.weaselab.dev + run: | + if [ -z "$MINIO_ACCESS_KEY" ]; then + echo "MinIO credentials not configured; skipping upload" + exit 0 + fi + zstd build/Testing/*/Test.xml + mc cp build/Testing/*/Test.xml.zst "minio/jenkins/conflict-set/${{ gitea.run_number }}/${{ matrix.name }}/" + + - name: Test summary + if: always() + run: | + python3 ctest_summary.py build/Testing/*/Test.xml \ + --link "https://minio.weaselab.dev/jenkins/conflict-set/${{ gitea.run_number }}/${{ matrix.name }}/Test.xml.zst" \ + | tee -a "$GITHUB_STEP_SUMMARY" + + release: + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-latest-amd64 + arch: amd64 + - runner: ubuntu-latest-arm64 + arch: arm64 + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/cache@v4 + with: + path: /var/cache/apt/archives + key: apt-${{ matrix.arch }}-${{ hashFiles('.gitea/workflows/ci.yml') }} + + - name: Install dependencies + run: | + . /etc/os-release + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list + sudo apt-get update -qq + sudo apt-get install -y \ + build-essential ccache clang-21 cmake devscripts \ + libc6-dbg llvm-21 lld-21 mold ninja-build rpm \ + valgrind zstd + sudo curl -Ls "https://minio.weaselab.dev/public/$(uname -m)/mc.RELEASE.2025-08-13T08-35-41Z" \ + -o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc + for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do + sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100 + done + + - name: Install paper build dependencies + if: matrix.arch == 'amd64' + run: | + sudo apt-get install -y \ + biber latexmk \ + texlive-bibtex-extra texlive-fonts-recommended \ + texlive-latex-extra texlive-pictures + + - uses: actions/cache@v4 + with: + path: .ccache + key: ccache-release-${{ matrix.arch }}-${{ gitea.sha }} + restore-keys: | + ccache-release-${{ matrix.arch }}- + + - name: Build + run: | + export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache" + rm -rf build + cmake -S . -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_FLAGS=-DNVALGRIND + ninja -C build + ccache -s + + - name: Test + run: | + cd build + # On arm64, valgrind needs the MAKE_MEM_DEFINED client requests for + # https://git.weaselab.dev/weaselab/conflict-set/issues/39, but this + # build has -DNVALGRIND, which compiles them out. Skip valgrind + # tests here; they run annotated in the test job. + ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 ${{ matrix.arch == 'arm64' && '-E valgrind' || '' }} -T Test -j "$(nproc)" --timeout 90 > /dev/null + + - name: Package + run: | + cd build + cpack -G DEB + cpack -G RPM + + - name: Build paper + if: matrix.arch == 'amd64' + run: | + cd paper + make + + - name: Upload artifacts to MinIO + if: always() + shell: bash + env: + MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }} + MC_HOST_minio: https://${{ secrets.MINIO_ACCESS_KEY }}:${{ secrets.MINIO_SECRET_KEY }}@minio.weaselab.dev + run: | + if [ -z "$MINIO_ACCESS_KEY" ]; then + echo "MinIO credentials not configured; skipping upload" + exit 0 + fi + dest="minio/jenkins/conflict-set/${{ gitea.run_number }}/release-${{ matrix.arch }}/" + zstd build/Testing/*/Test.xml + mc cp build/Testing/*/Test.xml.zst "$dest" + if compgen -G "build/*.deb" > /dev/null; then + mc cp build/*.deb "$dest" + fi + if compgen -G "build/*.rpm" > /dev/null; then + mc cp build/*.rpm "$dest" + fi + if compgen -G "paper/*.pdf" > /dev/null; then + mc cp paper/*.pdf "$dest" + fi + + - name: Test summary + if: always() + run: | + python3 ctest_summary.py build/Testing/*/Test.xml \ + --link "https://minio.weaselab.dev/jenkins/conflict-set/${{ gitea.run_number }}/release-${{ matrix.arch }}/Test.xml.zst" \ + | tee -a "$GITHUB_STEP_SUMMARY" + + coverage: + runs-on: ubuntu-latest-amd64 + steps: + - uses: actions/checkout@v4 + + - uses: actions/cache@v4 + with: + path: /var/cache/apt/archives + key: apt-amd64-${{ hashFiles('.gitea/workflows/ci.yml') }} + + - name: Install dependencies + run: | + . /etc/os-release + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list + sudo apt-get update -qq + sudo apt-get install -y \ + build-essential ccache clang-21 cmake gcovr \ + libc6-dbg llvm-21 lld-21 mold ninja-build python3 valgrind zstd + sudo curl -Ls "https://minio.weaselab.dev/public/$(uname -m)/mc.RELEASE.2025-08-13T08-35-41Z" \ + -o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc + for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do + sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100 + done + + - uses: actions/cache@v4 + with: + path: .ccache + key: ccache-coverage-${{ gitea.sha }} + restore-keys: | + ccache-coverage- + + - name: Build + run: | + export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache" + rm -rf build + cmake -S . -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage \ + -DCMAKE_BUILD_TYPE=Debug -DDISABLE_TSAN=ON + ninja -C build + ccache -s + + - name: Test + run: | + cd build + ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 -T Test -j "$(nproc)" --timeout 90 > /dev/null + + - name: Coverage report + shell: bash + run: | + gcov_args=(-f ConflictSet.cpp -f LongestCommonPrefix.h -f Metrics.h + --gcov-executable "llvm-cov gcov" --exclude-noncode-lines) + gcovr "${gcov_args[@]}" --cobertura > build/coverage.xml + gcovr "${gcov_args[@]}" + mkdir -p build/coverage_html + gcovr "${gcov_args[@]}" --html-details build/coverage_html/index.html + gcovr "${gcov_args[@]}" --fail-under-line 100 > /dev/null + + - name: Upload results to MinIO + if: always() + env: + MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }} + MC_HOST_minio: https://${{ secrets.MINIO_ACCESS_KEY }}:${{ secrets.MINIO_SECRET_KEY }}@minio.weaselab.dev + run: | + if [ -z "$MINIO_ACCESS_KEY" ]; then + echo "MinIO credentials not configured; skipping upload" + exit 0 + fi + dest="minio/jenkins/conflict-set/${{ gitea.run_number }}/coverage/" + zstd build/Testing/*/Test.xml + mc cp build/Testing/*/Test.xml.zst "$dest" + if [ -e build/coverage.xml ]; then + mc cp build/coverage.xml "$dest" + fi + if [ -d build/coverage_html ]; then + mc cp -r build/coverage_html "$dest" + fi + + - name: Test summary + if: always() + run: | + python3 ctest_summary.py build/Testing/*/Test.xml \ + --link "https://minio.weaselab.dev/jenkins/conflict-set/${{ gitea.run_number }}/coverage/Test.xml.zst" \ + | tee -a "$GITHUB_STEP_SUMMARY" + echo "" | tee -a "$GITHUB_STEP_SUMMARY" + echo "šŸ“Š [Coverage report](https://minio.weaselab.dev/jenkins/conflict-set/${{ gitea.run_number }}/coverage/coverage_html/index.html)" | tee -a "$GITHUB_STEP_SUMMARY" diff --git a/ctest_summary.py b/ctest_summary.py new file mode 100644 index 0000000..5562f17 --- /dev/null +++ b/ctest_summary.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Summarize a CTest Test.xml as markdown. + +Intended for $GITHUB_STEP_SUMMARY in CI, where only the first few failures +are shown inline (pass --link to point at the full Test.xml). Also reusable +locally to print every failure from a downloaded Test.xml with --all. +""" + +import argparse +import base64 +import gzip +import xml.etree.ElementTree as ET + +# Failure output is truncated to this many trailing characters, which is +# usually enough to include e.g. an ASan report's summary. +OUTPUT_TAIL_CHARS = 3000 + + +def test_output(test): + value = test.find("./Results/Measurement/Value") + if value is None or value.text is None: + return "" + text = value.text + if value.get("encoding") == "base64": + raw = base64.b64decode(text) + if value.get("compression") == "gzip": + raw = gzip.decompress(raw) + text = raw.decode(errors="replace") + return text + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("test_xml", help="path to a ctest Testing/*/Test.xml") + parser.add_argument( + "--inline", + type=int, + default=5, + help="how many failures to show inline (default 5)", + ) + parser.add_argument("--all", action="store_true", help="show every failure inline") + parser.add_argument( + "--link", help="URL of the full Test.xml, linked when failures are elided" + ) + args = parser.parse_args() + + testing = ET.parse(args.test_xml).getroot().find("Testing") + tests = testing.findall("Test") + failed = [t for t in tests if t.get("Status") == "failed"] + notrun = sum(1 for t in tests if t.get("Status") == "notrun") + + if not failed: + print(f"āœ… All {len(tests) - notrun} tests passed") + else: + print(f"āŒ {len(failed)} of {len(tests)} tests failed\n") + shown = failed if args.all else failed[: args.inline] + for test in shown: + name = test.findtext("Name") + output = test_output(test)[-OUTPUT_TAIL_CHARS:].strip() + print(f"
{name}\n") + print("````") + print(output) + print("````") + print("
\n") + remaining = len(failed) - len(shown) + if remaining > 0: + more = f"… and {remaining} more" + if args.link: + more += f" — full list in [Test.xml]({args.link})" + print(more) + if notrun: + print(f"\nāš ļø {notrun} tests not run") + + +if __name__ == "__main__": + main()