forked from weaselab/conflict-set
Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
55e23bafba | ||
|
|
235938b5aa | ||
|
|
8b71852495 | ||
|
|
e5e6402b43 | ||
|
|
61f5612e1f | ||
|
|
406b27936c | ||
|
|
7972ed919b | ||
|
|
0619b6325c | ||
|
|
7b14c8f9d5 | ||
|
|
22632fc9f2 | ||
|
|
1fccb65bd8 | ||
|
|
ee5972f946 |
@@ -1,322 +0,0 @@
|
||||
name: CI
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
env:
|
||||
CC: clang
|
||||
CXX: clang++
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest-amd64
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: /var/cache/apt/archives
|
||||
key: apt-amd64-${{ hashFiles('.gitea/workflows/ci.yml') }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y git nodejs pre-commit
|
||||
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pre-commit
|
||||
key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
|
||||
|
||||
- name: Run pre-commit
|
||||
run: |
|
||||
git config --global --add safe.directory "$PWD"
|
||||
pre-commit run --all-files --show-diff-on-failure
|
||||
|
||||
test:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- name: 64-bit-versions
|
||||
cmake_args: -DCMAKE_CXX_FLAGS=-DUSE_64_BIT=1
|
||||
- name: debug
|
||||
cmake_args: -DCMAKE_BUILD_TYPE=Debug -DMSAN_TOOLCHAIN_PATH=/opt/msan
|
||||
- name: simd-fallback
|
||||
cmake_args: -DUSE_SIMD_FALLBACK=ON
|
||||
- name: gcc
|
||||
cmake_args: -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++
|
||||
runs-on: ubuntu-latest-amd64
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: /var/cache/apt/archives
|
||||
key: apt-amd64-${{ hashFiles('.gitea/workflows/ci.yml') }}
|
||||
|
||||
- name: Install common dependencies
|
||||
run: |
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y \
|
||||
build-essential ccache cmake gcc g++ libc6-dbg \
|
||||
ninja-build python3 valgrind zstd
|
||||
sudo curl -Ls "https://minio.weaselab.dev/public/$(uname -m)/mc.RELEASE.2025-08-13T08-35-41Z" \
|
||||
-o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
|
||||
|
||||
- name: Install LLVM toolchain
|
||||
if: matrix.name != 'gcc'
|
||||
run: |
|
||||
. /etc/os-release
|
||||
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
|
||||
echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y \
|
||||
clang-21 llvm-21 lld-21 mold
|
||||
for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do
|
||||
sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
|
||||
done
|
||||
|
||||
- name: Download MSan toolchain
|
||||
if: matrix.name == 'debug'
|
||||
run: |
|
||||
curl -Ls "https://minio.weaselab.dev/public/x86_64/msan-toolchain-21.1.8.tar.zst" -o /tmp/msan-toolchain.tar.zst
|
||||
sudo mkdir -p /opt/msan
|
||||
sudo tar --zstd -xf /tmp/msan-toolchain.tar.zst -C /opt/msan
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache"
|
||||
rm -rf build
|
||||
cmake -S . -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ${{ matrix.cmake_args }}
|
||||
ninja -C build
|
||||
ccache -s
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
cd build
|
||||
ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 -T Test -j "$(nproc)" --timeout 90 > /dev/null
|
||||
|
||||
- name: Upload test results to MinIO
|
||||
if: always()
|
||||
env:
|
||||
MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }}
|
||||
MC_HOST_minio: https://${{ secrets.MINIO_ACCESS_KEY }}:${{ secrets.MINIO_SECRET_KEY }}@minio.weaselab.dev
|
||||
run: |
|
||||
if [ -z "$MINIO_ACCESS_KEY" ]; then
|
||||
echo "MinIO credentials not configured; skipping upload"
|
||||
exit 0
|
||||
fi
|
||||
zstd build/Testing/*/Test.xml
|
||||
mc cp build/Testing/*/Test.xml.zst "minio/jenkins/conflict-set/${{ gitea.run_number }}/${{ matrix.name }}/"
|
||||
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
python3 ctest_summary.py build/Testing/*/Test.xml \
|
||||
--link "https://minio.weaselab.dev/jenkins/conflict-set/${{ gitea.run_number }}/${{ matrix.name }}/Test.xml.zst" \
|
||||
| tee -a "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
release:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- runner: ubuntu-latest-amd64
|
||||
arch: amd64
|
||||
cmake_args: -DMSAN_TOOLCHAIN_PATH=/opt/msan
|
||||
- runner: ubuntu-latest-arm64
|
||||
arch: arm64
|
||||
cmake_args: ""
|
||||
runs-on: ${{ matrix.runner }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: /var/cache/apt/archives
|
||||
key: apt-${{ matrix.arch }}-${{ hashFiles('.gitea/workflows/ci.yml') }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
. /etc/os-release
|
||||
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
|
||||
echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y \
|
||||
build-essential ccache clang-21 cmake devscripts \
|
||||
libc6-dbg llvm-21 lld-21 mold ninja-build rpm \
|
||||
valgrind zstd
|
||||
sudo curl -Ls "https://minio.weaselab.dev/public/$(uname -m)/mc.RELEASE.2025-08-13T08-35-41Z" \
|
||||
-o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
|
||||
for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do
|
||||
sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
|
||||
done
|
||||
|
||||
- name: Install paper build dependencies
|
||||
if: matrix.arch == 'amd64'
|
||||
run: |
|
||||
sudo apt-get install -y \
|
||||
biber latexmk \
|
||||
texlive-bibtex-extra texlive-fonts-recommended \
|
||||
texlive-latex-extra texlive-pictures
|
||||
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: .ccache
|
||||
key: ccache-release-${{ matrix.arch }}-${{ gitea.sha }}
|
||||
restore-keys: |
|
||||
ccache-release-${{ matrix.arch }}-
|
||||
|
||||
- name: Download MSan toolchain
|
||||
if: matrix.arch == 'amd64'
|
||||
run: |
|
||||
curl -Ls "https://minio.weaselab.dev/public/x86_64/msan-toolchain-21.1.8.tar.zst" -o /tmp/msan-toolchain.tar.zst
|
||||
sudo mkdir -p /opt/msan
|
||||
sudo tar --zstd -xf /tmp/msan-toolchain.tar.zst -C /opt/msan
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache"
|
||||
rm -rf build
|
||||
cmake -S . -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_FLAGS=-DNVALGRIND ${{ matrix.cmake_args }}
|
||||
ninja -C build
|
||||
ccache -s
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
cd build
|
||||
# On arm64, valgrind needs the MAKE_MEM_DEFINED client requests for
|
||||
# https://git.weaselab.dev/weaselab/conflict-set/issues/39, but this
|
||||
# build has -DNVALGRIND, which compiles them out. Skip valgrind
|
||||
# tests here; they run annotated in the test job.
|
||||
ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 ${{ matrix.arch == 'arm64' && '-E valgrind' || '' }} -T Test -j "$(nproc)" --timeout 90 > /dev/null
|
||||
|
||||
- name: Package
|
||||
run: |
|
||||
cd build
|
||||
cpack -G DEB
|
||||
cpack -G RPM
|
||||
|
||||
- name: Build paper
|
||||
if: matrix.arch == 'amd64'
|
||||
run: |
|
||||
cd paper
|
||||
make
|
||||
|
||||
- name: Upload artifacts to MinIO
|
||||
if: always()
|
||||
shell: bash
|
||||
env:
|
||||
MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }}
|
||||
MC_HOST_minio: https://${{ secrets.MINIO_ACCESS_KEY }}:${{ secrets.MINIO_SECRET_KEY }}@minio.weaselab.dev
|
||||
run: |
|
||||
if [ -z "$MINIO_ACCESS_KEY" ]; then
|
||||
echo "MinIO credentials not configured; skipping upload"
|
||||
exit 0
|
||||
fi
|
||||
dest="minio/jenkins/conflict-set/${{ gitea.run_number }}/release-${{ matrix.arch }}/"
|
||||
zstd build/Testing/*/Test.xml
|
||||
mc cp build/Testing/*/Test.xml.zst "$dest"
|
||||
if compgen -G "build/*.deb" > /dev/null; then
|
||||
mc cp build/*.deb "$dest"
|
||||
fi
|
||||
if compgen -G "build/*.rpm" > /dev/null; then
|
||||
mc cp build/*.rpm "$dest"
|
||||
fi
|
||||
if compgen -G "paper/*.pdf" > /dev/null; then
|
||||
mc cp paper/*.pdf "$dest"
|
||||
fi
|
||||
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
python3 ctest_summary.py build/Testing/*/Test.xml \
|
||||
--link "https://minio.weaselab.dev/jenkins/conflict-set/${{ gitea.run_number }}/release-${{ matrix.arch }}/Test.xml.zst" \
|
||||
| tee -a "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
coverage:
|
||||
runs-on: ubuntu-latest-amd64
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: /var/cache/apt/archives
|
||||
key: apt-amd64-${{ hashFiles('.gitea/workflows/ci.yml') }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
. /etc/os-release
|
||||
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
|
||||
echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y \
|
||||
build-essential ccache clang-21 cmake gcovr \
|
||||
libc6-dbg llvm-21 lld-21 mold ninja-build python3 valgrind zstd
|
||||
sudo curl -Ls "https://minio.weaselab.dev/public/$(uname -m)/mc.RELEASE.2025-08-13T08-35-41Z" \
|
||||
-o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
|
||||
for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do
|
||||
sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
|
||||
done
|
||||
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: .ccache
|
||||
key: ccache-coverage-${{ gitea.sha }}
|
||||
restore-keys: |
|
||||
ccache-coverage-
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache"
|
||||
rm -rf build
|
||||
cmake -S . -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
|
||||
-DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage \
|
||||
-DCMAKE_BUILD_TYPE=Debug -DDISABLE_TSAN=ON
|
||||
ninja -C build
|
||||
ccache -s
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
cd build
|
||||
ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 -T Test -j "$(nproc)" --timeout 90 > /dev/null
|
||||
|
||||
- name: Coverage report
|
||||
shell: bash
|
||||
run: |
|
||||
gcov_args=(-f ConflictSet.cpp -f LongestCommonPrefix.h -f Metrics.h
|
||||
--gcov-executable "llvm-cov gcov" --exclude-noncode-lines)
|
||||
gcovr "${gcov_args[@]}" --cobertura > build/coverage.xml
|
||||
gcovr "${gcov_args[@]}"
|
||||
mkdir -p build/coverage_html
|
||||
gcovr "${gcov_args[@]}" --html-details build/coverage_html/index.html
|
||||
gcovr "${gcov_args[@]}" --fail-under-line 100 > /dev/null
|
||||
|
||||
- name: Upload results to MinIO
|
||||
if: always()
|
||||
env:
|
||||
MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }}
|
||||
MC_HOST_minio: https://${{ secrets.MINIO_ACCESS_KEY }}:${{ secrets.MINIO_SECRET_KEY }}@minio.weaselab.dev
|
||||
run: |
|
||||
if [ -z "$MINIO_ACCESS_KEY" ]; then
|
||||
echo "MinIO credentials not configured; skipping upload"
|
||||
exit 0
|
||||
fi
|
||||
dest="minio/jenkins/conflict-set/${{ gitea.run_number }}/coverage/"
|
||||
zstd build/Testing/*/Test.xml
|
||||
mc cp build/Testing/*/Test.xml.zst "$dest"
|
||||
if [ -e build/coverage.xml ]; then
|
||||
mc cp build/coverage.xml "$dest"
|
||||
fi
|
||||
if [ -d build/coverage_html ]; then
|
||||
mc cp -r build/coverage_html "$dest"
|
||||
fi
|
||||
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
python3 ctest_summary.py build/Testing/*/Test.xml \
|
||||
--link "https://minio.weaselab.dev/jenkins/conflict-set/${{ gitea.run_number }}/coverage/Test.xml.zst" \
|
||||
| tee -a "$GITHUB_STEP_SUMMARY"
|
||||
echo "" | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
echo "📊 [Coverage report](https://minio.weaselab.dev/jenkins/conflict-set/${{ gitea.run_number }}/coverage/coverage_html/index.html)" | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
@@ -37,16 +37,15 @@ ConflictSet::ReadRange singleton(Arena &arena, TrivialSpan key) {
|
||||
}
|
||||
|
||||
ConflictSet::ReadRange prefixRange(Arena &arena, TrivialSpan key) {
|
||||
int index = key.size() - 1;
|
||||
for (; index >= 0; index--)
|
||||
if (key[index] != 255)
|
||||
int index;
|
||||
for (index = key.size() - 1; index >= 0; index--)
|
||||
if ((key[index]) != 255)
|
||||
break;
|
||||
|
||||
// Must not be called with a string that consists only of zero or more '\xff'
|
||||
// bytes, or with an empty string (which has no finite upper bound).
|
||||
// bytes.
|
||||
if (index < 0) {
|
||||
assert(false);
|
||||
std::abort();
|
||||
}
|
||||
|
||||
uint8_t *buf = new (arena) uint8_t[index + 1];
|
||||
|
||||
+15
-93
@@ -1,11 +1,11 @@
|
||||
cmake_minimum_required(VERSION 3.18)
|
||||
project(
|
||||
conflict-set
|
||||
VERSION 0.1.0
|
||||
VERSION 0.0.14
|
||||
DESCRIPTION
|
||||
"A data structure for optimistic concurrency control on ranges of bitwise-lexicographically-ordered keys."
|
||||
HOMEPAGE_URL "https://git.weaselab.dev/weaselab/conflict-set"
|
||||
LANGUAGES C CXX ASM)
|
||||
LANGUAGES C CXX)
|
||||
set(CMAKE_CXX_STANDARD 20)
|
||||
|
||||
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/version.txt ${PROJECT_VERSION})
|
||||
@@ -32,12 +32,8 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
|
||||
endif()
|
||||
|
||||
add_compile_options(
|
||||
-Werror=switch-enum
|
||||
-Wswitch-enum
|
||||
-Wunused-variable
|
||||
-fPIC
|
||||
-fdata-sections
|
||||
-ffunction-sections
|
||||
# -Werror=switch-enum
|
||||
-Wswitch-enum -Wunused-variable -fPIC -fdata-sections -ffunction-sections
|
||||
-fno-jump-tables # https://github.com/llvm/llvm-project/issues/54247
|
||||
)
|
||||
|
||||
@@ -53,7 +49,6 @@ endif()
|
||||
|
||||
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
add_compile_options("-Wno-maybe-uninitialized")
|
||||
add_compile_options("-Wno-maybe-musttail-local-addr")
|
||||
endif()
|
||||
|
||||
if(NOT APPLE)
|
||||
@@ -61,16 +56,12 @@ if(NOT APPLE)
|
||||
add_compile_options(-g -fno-omit-frame-pointer)
|
||||
endif()
|
||||
|
||||
set(relro_flags "LINKER:-z,relro,-z,now,-z,noexecstack")
|
||||
set(full_relro_flags "-pie;${relro_flags}")
|
||||
set(full_relro_flags "-pie;LINKER:-z,relro,-z,now,-z,noexecstack")
|
||||
cmake_push_check_state()
|
||||
list(APPEND CMAKE_REQUIRED_LINK_OPTIONS ${full_relro_flags})
|
||||
check_cxx_source_compiles("int main(){}" HAS_FULL_RELRO FAIL_REGEX "warning:")
|
||||
if(HAS_FULL_RELRO)
|
||||
# -pie only applies to executables; passing it when linking a shared library
|
||||
# makes the driver pull in Scrt1.o, which requires main.
|
||||
add_link_options("$<$<STREQUAL:$<TARGET_PROPERTY:TYPE>,EXECUTABLE>:-pie>"
|
||||
${relro_flags})
|
||||
add_link_options(${full_relro_flags})
|
||||
endif()
|
||||
cmake_pop_check_state()
|
||||
|
||||
@@ -103,13 +94,6 @@ option(USE_SIMD_FALLBACK
|
||||
|
||||
option(DISABLE_TSAN "Disable TSAN" OFF)
|
||||
|
||||
set(MSAN_TOOLCHAIN_PATH
|
||||
""
|
||||
CACHE
|
||||
PATH
|
||||
"Path to an MSan-instrumented libc++ toolchain. When set, an MSan fuzz_driver target is added. Must contain include/c++/v1 and lib. See build_msan_toolchain.sh to produce one."
|
||||
)
|
||||
|
||||
# This is encouraged according to
|
||||
# https://valgrind.org/docs/manual/manual-core-adv.html#manual-core-adv.clientreq
|
||||
include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/valgrind)
|
||||
@@ -130,15 +114,7 @@ endif()
|
||||
|
||||
set(CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "")
|
||||
|
||||
# Architecture-specific SIMD assembly. These functions operate on
|
||||
# potentially-indeterminate memory, which is UB in C++ but well-defined in
|
||||
# assembly.
|
||||
set(SIMD_ASM_FILES)
|
||||
if(CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64 AND NOT USE_SIMD_FALLBACK)
|
||||
set(SIMD_ASM_FILES ${CMAKE_CURRENT_SOURCE_DIR}/simd_x86_64.S)
|
||||
endif()
|
||||
|
||||
add_library(${PROJECT_NAME}-object OBJECT ConflictSet.cpp ${SIMD_ASM_FILES})
|
||||
add_library(${PROJECT_NAME}-object OBJECT ConflictSet.cpp)
|
||||
target_compile_options(${PROJECT_NAME}-object PRIVATE -fno-exceptions
|
||||
-fvisibility=hidden)
|
||||
target_include_directories(${PROJECT_NAME}-object
|
||||
@@ -154,7 +130,6 @@ add_custom_command(
|
||||
COMMAND_EXPAND_LISTS)
|
||||
|
||||
add_library(${PROJECT_NAME} SHARED ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o)
|
||||
add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-object)
|
||||
set_target_properties(
|
||||
${PROJECT_NAME} PROPERTIES LIBRARY_OUTPUT_DIRECTORY
|
||||
"${CMAKE_CURRENT_BINARY_DIR}/radix_tree")
|
||||
@@ -171,7 +146,6 @@ if(HAS_VERSION_SCRIPT)
|
||||
endif()
|
||||
|
||||
add_library(${PROJECT_NAME}-static STATIC ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o)
|
||||
add_dependencies(${PROJECT_NAME}-static ${PROJECT_NAME}-object)
|
||||
if(CMAKE_BUILD_TYPE STREQUAL Debug)
|
||||
set_target_properties(${PROJECT_NAME}-static PROPERTIES LINKER_LANGUAGE CXX)
|
||||
else()
|
||||
@@ -241,7 +215,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
|
||||
endif()
|
||||
|
||||
# ad hoc testing
|
||||
add_executable(conflict_set_main ConflictSet.cpp ${SIMD_ASM_FILES})
|
||||
add_executable(conflict_set_main ConflictSet.cpp)
|
||||
target_include_directories(conflict_set_main
|
||||
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
|
||||
target_compile_definitions(conflict_set_main PRIVATE ENABLE_MAIN)
|
||||
@@ -257,7 +231,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
|
||||
cmake_pop_check_state()
|
||||
|
||||
if(HAS_LIB_FUZZER)
|
||||
add_executable(conflict_set_fuzz_test ConflictSet.cpp ${SIMD_ASM_FILES})
|
||||
add_executable(conflict_set_fuzz_test ConflictSet.cpp)
|
||||
target_include_directories(conflict_set_fuzz_test
|
||||
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
|
||||
target_compile_definitions(conflict_set_fuzz_test PRIVATE ENABLE_FUZZ)
|
||||
@@ -268,9 +242,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# whitebox tests asan+ubsan
|
||||
add_executable(fuzz_driver ConflictSet.cpp FuzzTestDriver.cpp
|
||||
${SIMD_ASM_FILES})
|
||||
# whitebox tests
|
||||
add_executable(fuzz_driver ConflictSet.cpp FuzzTestDriver.cpp)
|
||||
target_compile_options(fuzz_driver PRIVATE ${TEST_FLAGS})
|
||||
if(NOT CMAKE_CROSSCOMPILING)
|
||||
target_compile_options(fuzz_driver PRIVATE -fsanitize=address,undefined)
|
||||
@@ -284,39 +257,9 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
|
||||
add_test(NAME conflict_set_fuzz_${hash} COMMAND fuzz_driver ${TEST})
|
||||
endforeach()
|
||||
|
||||
# whitebox tests msan
|
||||
if(MSAN_TOOLCHAIN_PATH)
|
||||
add_executable(fuzz_driver_msan ConflictSet.cpp FuzzTestDriver.cpp
|
||||
${SIMD_ASM_FILES})
|
||||
target_compile_options(fuzz_driver_msan PRIVATE ${TEST_FLAGS})
|
||||
if(NOT CMAKE_CROSSCOMPILING)
|
||||
target_compile_options(
|
||||
fuzz_driver_msan
|
||||
PRIVATE -fsanitize=memory -fsanitize-memory-track-origins=2
|
||||
$<$<COMPILE_LANGUAGE:CXX>:-stdlib=libc++>
|
||||
-I${MSAN_TOOLCHAIN_PATH}/include/c++/v1)
|
||||
target_link_options(
|
||||
fuzz_driver_msan
|
||||
PRIVATE
|
||||
-fsanitize=memory
|
||||
-fsanitize-memory-track-origins=2
|
||||
-stdlib=libc++
|
||||
-L${MSAN_TOOLCHAIN_PATH}/lib
|
||||
LINKER:-rpath,${MSAN_TOOLCHAIN_PATH}/lib)
|
||||
endif()
|
||||
target_compile_definitions(fuzz_driver_msan PRIVATE ENABLE_FUZZ)
|
||||
target_include_directories(fuzz_driver_msan
|
||||
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
|
||||
foreach(TEST ${CORPUS_TESTS})
|
||||
get_filename_component(hash ${TEST} NAME)
|
||||
add_test(NAME conflict_set_msan_${hash} COMMAND fuzz_driver_msan ${TEST})
|
||||
endforeach()
|
||||
endif()
|
||||
|
||||
# tsan tests
|
||||
if(NOT CMAKE_CROSSCOMPILING AND NOT DISABLE_TSAN)
|
||||
add_executable(tsan_driver ConflictSet.cpp FuzzTestDriver.cpp
|
||||
${SIMD_ASM_FILES})
|
||||
add_executable(tsan_driver ConflictSet.cpp FuzzTestDriver.cpp)
|
||||
target_compile_options(tsan_driver PRIVATE ${TEST_FLAGS} -fsanitize=thread)
|
||||
target_link_options(tsan_driver PRIVATE -fsanitize=thread)
|
||||
target_compile_definitions(tsan_driver PRIVATE ENABLE_FUZZ THREAD_TEST)
|
||||
@@ -431,30 +374,9 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
|
||||
if(NOT CMAKE_CROSSCOMPILING)
|
||||
find_program(HARDENING_CHECK hardening-check)
|
||||
if(HARDENING_CHECK)
|
||||
# Not all versions of hardening-check support the same options, so query
|
||||
# the help output before using architecture-specific skips.
|
||||
execute_process(
|
||||
COMMAND ${HARDENING_CHECK} --help
|
||||
OUTPUT_VARIABLE hardening_check_help
|
||||
ERROR_VARIABLE hardening_check_help
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE)
|
||||
set(hardening_check_arch_flags "")
|
||||
# Control flow integrity (CET) is x86-only and branch protection (PAC/BTI)
|
||||
# is arm64-only, so ignore whichever doesn't apply.
|
||||
if(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64 OR CMAKE_SYSTEM_PROCESSOR
|
||||
STREQUAL arm64)
|
||||
if(hardening_check_help MATCHES "nocfprotection")
|
||||
list(APPEND hardening_check_arch_flags --nocfprotection)
|
||||
endif()
|
||||
else()
|
||||
if(hardening_check_help MATCHES "nobranchprotection")
|
||||
list(APPEND hardening_check_arch_flags --nobranchprotection)
|
||||
endif()
|
||||
endif()
|
||||
add_test(
|
||||
NAME hardening_check
|
||||
COMMAND ${HARDENING_CHECK} $<TARGET_FILE:${PROJECT_NAME}> --nofortify
|
||||
--nostackprotector ${hardening_check_arch_flags})
|
||||
add_test(NAME hardening_check
|
||||
COMMAND ${HARDENING_CHECK} $<TARGET_FILE:${PROJECT_NAME}>
|
||||
--nofortify --nostackprotector)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
||||
+1383
-979
File diff suppressed because it is too large
Load Diff
+58
@@ -0,0 +1,58 @@
|
||||
FROM ubuntu:rolling
|
||||
|
||||
WORKDIR /tmp
|
||||
ENV HOME=/tmp
|
||||
RUN chmod -R 777 /tmp
|
||||
|
||||
# Install apt dependencies
|
||||
RUN apt-get update
|
||||
RUN apt-get upgrade -y
|
||||
RUN TZ=America/Los_Angeles DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
binutils-aarch64-linux-gnu \
|
||||
build-essential \
|
||||
ccache \
|
||||
cmake \
|
||||
curl \
|
||||
devscripts \
|
||||
g++-aarch64-linux-gnu \
|
||||
gcovr \
|
||||
git \
|
||||
gnupg \
|
||||
libc6-dbg \
|
||||
lsb-release \
|
||||
mold \
|
||||
ninja-build \
|
||||
pre-commit \
|
||||
python3-requests \
|
||||
qemu-user \
|
||||
rpm \
|
||||
software-properties-common \
|
||||
texlive-full \
|
||||
wget \
|
||||
zstd
|
||||
|
||||
# Install recent valgrind from source
|
||||
RUN curl -Ls https://sourceware.org/pub/valgrind/valgrind-3.22.0.tar.bz2 -o valgrind.tar.bz2 && \
|
||||
echo "c811db5add2c5f729944caf47c4e7a65dcaabb9461e472b578765dd7bf6d2d4c valgrind.tar.bz2" > valgrind-sha.txt && \
|
||||
sha256sum --quiet -c valgrind-sha.txt && \
|
||||
mkdir valgrind && \
|
||||
tar --strip-components 1 --no-same-owner --no-same-permissions --directory valgrind -xjf valgrind.tar.bz2 && \
|
||||
cd valgrind && \
|
||||
./configure --enable-only64bit --enable-lto && \
|
||||
make -j`nproc` && \
|
||||
make install && \
|
||||
cd .. && \
|
||||
rm -rf /tmp/*
|
||||
|
||||
# Recent clang
|
||||
RUN wget https://apt.llvm.org/llvm.sh && chmod +x ./llvm.sh && ./llvm.sh 20
|
||||
|
||||
RUN apt-get -y install clang llvm
|
||||
|
||||
# Set after building valgrind, which doesn't build with clang for some reason
|
||||
ENV CC=clang
|
||||
ENV CXX=clang++
|
||||
|
||||
# Try to have all the pre-commit hooks we'll need already initialized
|
||||
COPY .pre-commit-config.yaml /tmp/
|
||||
RUN git init && pre-commit install-hooks
|
||||
+4
-15
@@ -96,9 +96,7 @@ void ConflictSet::setOldestVersion(int64_t oldestVersion) {
|
||||
return impl->setOldestVersion(oldestVersion);
|
||||
}
|
||||
|
||||
// The hash_table implementation does not track memory usage, so return 0 to
|
||||
// satisfy the API contract that getBytes() returns a non-negative value.
|
||||
int64_t ConflictSet::getBytes() const { return 0; }
|
||||
int64_t ConflictSet::getBytes() const { return -1; }
|
||||
|
||||
void ConflictSet::getMetricsV1(MetricsV1 **metrics, int *count) const {
|
||||
*metrics = nullptr;
|
||||
@@ -121,13 +119,7 @@ ConflictSet::ConflictSet(ConflictSet &&other) noexcept
|
||||
: impl(std::exchange(other.impl, nullptr)) {}
|
||||
|
||||
ConflictSet &ConflictSet::operator=(ConflictSet &&other) noexcept {
|
||||
if (this != &other) {
|
||||
if (impl) {
|
||||
impl->~Impl();
|
||||
safe_free(impl, sizeof(Impl));
|
||||
}
|
||||
impl = std::exchange(other.impl, nullptr);
|
||||
}
|
||||
impl = std::exchange(other.impl, nullptr);
|
||||
return *this;
|
||||
}
|
||||
|
||||
@@ -163,10 +155,7 @@ __attribute__((__visibility__("default"))) void ConflictSet_destroy(void *cs) {
|
||||
}
|
||||
__attribute__((__visibility__("default"))) int64_t
|
||||
ConflictSet_getBytes(void *cs) {
|
||||
(void)cs;
|
||||
// The hash_table implementation does not track memory usage, so return 0 to
|
||||
// satisfy the API contract that ConflictSet_getBytes returns a non-negative
|
||||
// value.
|
||||
return 0;
|
||||
using Impl = ConflictSet::Impl;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+151
@@ -0,0 +1,151 @@
|
||||
def CleanBuildAndTest(String cmakeArgs) {
|
||||
sh """
|
||||
export CCACHE_DIR=/ccache
|
||||
rm -rf build
|
||||
mkdir build
|
||||
cd build
|
||||
cmake .. -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache ${cmakeArgs}
|
||||
ninja
|
||||
ccache -s
|
||||
"""
|
||||
catchError {
|
||||
sh '''
|
||||
cd build
|
||||
ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 -T Test -j `nproc` --timeout 90 > /dev/null
|
||||
zstd Testing/*/Test.xml
|
||||
'''
|
||||
}
|
||||
xunit tools: [CTest(pattern: 'build/Testing/*/Test.xml')], skipPublishingChecks: false
|
||||
minio bucket: 'jenkins', credentialsId: 'jenkins-minio', excludes: '', host: 'minio.weaselab.dev', includes: 'build/Testing/*/Test.xml.zst', targetFolder: '${JOB_NAME}/${BUILD_NUMBER}/${STAGE_NAME}/'
|
||||
}
|
||||
|
||||
pipeline {
|
||||
agent any
|
||||
stages {
|
||||
stage('Pre-commit') {
|
||||
agent {
|
||||
dockerfile {
|
||||
args '-v /home/jenkins/ccache:/ccache'
|
||||
reuseNode true
|
||||
}
|
||||
}
|
||||
steps {
|
||||
script {
|
||||
env.HOME = env.WORKSPACE
|
||||
}
|
||||
sh 'pre-commit run --all-files --show-diff-on-failure'
|
||||
}
|
||||
}
|
||||
stage('64 bit versions') {
|
||||
agent {
|
||||
dockerfile {
|
||||
args '-v /home/jenkins/ccache:/ccache'
|
||||
reuseNode true
|
||||
}
|
||||
}
|
||||
steps {
|
||||
CleanBuildAndTest("-DCMAKE_CXX_FLAGS=-DUSE_64_BIT=1")
|
||||
}
|
||||
}
|
||||
stage('Debug') {
|
||||
agent {
|
||||
dockerfile {
|
||||
args '-v /home/jenkins/ccache:/ccache'
|
||||
reuseNode true
|
||||
}
|
||||
}
|
||||
steps {
|
||||
CleanBuildAndTest("-DCMAKE_BUILD_TYPE=Debug")
|
||||
}
|
||||
}
|
||||
stage('SIMD fallback') {
|
||||
agent {
|
||||
dockerfile {
|
||||
args '-v /home/jenkins/ccache:/ccache'
|
||||
reuseNode true
|
||||
}
|
||||
}
|
||||
steps {
|
||||
CleanBuildAndTest("-DUSE_SIMD_FALLBACK=ON")
|
||||
}
|
||||
}
|
||||
stage('Release [clang]') {
|
||||
agent {
|
||||
dockerfile {
|
||||
args '-v /home/jenkins/ccache:/ccache'
|
||||
reuseNode true
|
||||
}
|
||||
}
|
||||
steps {
|
||||
CleanBuildAndTest("-DCMAKE_CXX_FLAGS=-DNVALGRIND")
|
||||
recordIssues(tools: [clang()])
|
||||
sh '''
|
||||
cd build
|
||||
cpack -G DEB
|
||||
cpack -G RPM
|
||||
'''
|
||||
sh '''
|
||||
cd paper
|
||||
make
|
||||
'''
|
||||
minio bucket: 'jenkins', credentialsId: 'jenkins-minio', excludes: '', host: 'minio.weaselab.dev', includes: 'build/*.deb,build/*.rpm,paper/*.pdf', targetFolder: '${JOB_NAME}/${BUILD_NUMBER}/${STAGE_NAME}/'
|
||||
}
|
||||
}
|
||||
stage('gcc') {
|
||||
agent {
|
||||
dockerfile {
|
||||
args '-v /home/jenkins/ccache:/ccache'
|
||||
reuseNode true
|
||||
}
|
||||
}
|
||||
steps {
|
||||
CleanBuildAndTest("-DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++")
|
||||
recordIssues(tools: [gcc()])
|
||||
}
|
||||
}
|
||||
stage('Release [clang,aarch64]') {
|
||||
agent {
|
||||
dockerfile {
|
||||
args '-v /home/jenkins/ccache:/ccache'
|
||||
reuseNode true
|
||||
}
|
||||
}
|
||||
steps {
|
||||
CleanBuildAndTest("-DCMAKE_TOOLCHAIN_FILE=../aarch64-toolchain.cmake -DCMAKE_CXX_FLAGS=-DNVALGRIND")
|
||||
sh '''
|
||||
cd build
|
||||
cpack -G DEB
|
||||
cpack -G RPM
|
||||
'''
|
||||
minio bucket: 'jenkins', credentialsId: 'jenkins-minio', excludes: '', host: 'minio.weaselab.dev', includes: 'build/*.deb,build/*.rpm', targetFolder: '${JOB_NAME}/${BUILD_NUMBER}/${STAGE_NAME}'
|
||||
}
|
||||
}
|
||||
stage('Coverage') {
|
||||
agent {
|
||||
dockerfile {
|
||||
args '-v /home/jenkins/ccache:/ccache'
|
||||
reuseNode true
|
||||
}
|
||||
}
|
||||
steps {
|
||||
script {
|
||||
gcov_args = "-f ConflictSet.cpp -f LongestCommonPrefix.h -f Metrics.h --gcov-executable 'llvm-cov gcov' --exclude-noncode-lines"
|
||||
}
|
||||
CleanBuildAndTest("-DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DCMAKE_BUILD_TYPE=Debug -DDISABLE_TSAN=ON")
|
||||
sh """
|
||||
gcovr ${gcov_args} --cobertura > build/coverage.xml
|
||||
"""
|
||||
recordCoverage qualityGates: [[criticality: 'NOTE', metric: 'MODULE']], tools: [[parser: 'COBERTURA', pattern: 'build/coverage.xml']]
|
||||
sh """
|
||||
gcovr ${gcov_args}
|
||||
gcovr ${gcov_args} --fail-under-line 100 > /dev/null
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
post {
|
||||
always {
|
||||
emailext mimeType: 'text/html', body: '${SCRIPT, template="groovy-html.template"}', subject: "${env.JOB_NAME} - Build# ${env.BUILD_NUMBER} - ${currentBuild.currentResult}", to: 'andrew@weaselab.dev'
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,10 +7,10 @@ Hardware for all benchmarks is an AMD Ryzen 9 7900 with (2x32GB) 5600MT/s CL28-3
|
||||
```
|
||||
$ clang++ --version
|
||||
|
||||
Ubuntu clang version 21.1.8 (6ubuntu1)
|
||||
Ubuntu clang version 20.0.0 (++20241120082228+86734c857724-1~exp1~20241120202359.554)
|
||||
Target: x86_64-pc-linux-gnu
|
||||
Thread model: posix
|
||||
InstalledDir: /usr/lib/llvm-21/bin
|
||||
InstalledDir: /usr/lib/llvm-20/bin
|
||||
```
|
||||
|
||||
# Microbenchmark
|
||||
@@ -19,30 +19,30 @@ InstalledDir: /usr/lib/llvm-21/bin
|
||||
|
||||
| ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark
|
||||
|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
|
||||
| 164.29 | 6,086,873.38 | 0.0% | 3,107.03 | 604.19 | 5.142 | 558.59 | 0.0% | 1.96 | `point reads`
|
||||
| 161.05 | 6,209,395.38 | 0.1% | 3,036.76 | 592.21 | 5.128 | 539.35 | 0.0% | 1.93 | `prefix reads`
|
||||
| 239.55 | 4,174,539.38 | 0.1% | 3,722.71 | 880.68 | 4.227 | 692.00 | 0.0% | 2.86 | `range reads`
|
||||
| 354.75 | 2,818,919.14 | 0.7% | 4,523.64 | 1,304.75 | 3.467 | 720.22 | 2.0% | 4.23 | `point writes`
|
||||
| 345.32 | 2,895,878.47 | 0.1% | 4,484.57 | 1,270.31 | 3.530 | 705.00 | 1.8% | 4.12 | `prefix writes`
|
||||
| 193.48 | 5,168,547.42 | 0.1% | 2,224.10 | 711.72 | 3.125 | 377.17 | 3.3% | 2.32 | `range writes`
|
||||
| 404.89 | 2,469,777.50 | 2.4% | 6,855.96 | 1,489.70 | 4.602 | 1,227.82 | 1.3% | 0.05 | `monotonic increasing point writes`
|
||||
| 134,231.80 | 7,449.80 | 1.9% | 812,045.25 | 495,770.40 | 1.638 | 151,246.50 | 0.9% | 0.01 | `worst case for radix tree`
|
||||
| 37.80 | 26,454,311.17 | 0.4% | 701.00 | 139.14 | 5.038 | 102.00 | 0.0% | 0.01 | `create and destroy`
|
||||
| 161.29 | 6,200,056.17 | 0.1% | 3,014.03 | 831.04 | 3.627 | 504.59 | 0.0% | 1.93 | `point reads`
|
||||
| 158.32 | 6,316,160.64 | 0.1% | 2,954.16 | 815.80 | 3.621 | 490.17 | 0.0% | 1.89 | `prefix reads`
|
||||
| 237.39 | 4,212,409.50 | 0.2% | 3,592.41 | 1,233.96 | 2.911 | 629.31 | 0.0% | 2.84 | `range reads`
|
||||
| 442.11 | 2,261,878.94 | 0.0% | 4,450.57 | 2,314.25 | 1.923 | 707.92 | 2.1% | 5.28 | `point writes`
|
||||
| 439.89 | 2,273,308.53 | 0.1% | 4,410.22 | 2,302.29 | 1.916 | 694.74 | 2.1% | 5.25 | `prefix writes`
|
||||
| 290.96 | 3,436,936.78 | 0.0% | 2,315.38 | 1,528.68 | 1.515 | 396.69 | 3.3% | 3.49 | `range writes`
|
||||
| 476.93 | 2,096,762.02 | 0.6% | 6,999.33 | 2,484.94 | 2.817 | 1,251.73 | 1.3% | 0.06 | `monotonic increasing point writes`
|
||||
| 131,736.57 | 7,590.91 | 1.1% | 807,444.50 | 704,941.71 | 1.145 | 144,584.60 | 0.9% | 0.01 | `worst case for radix tree`
|
||||
| 45.50 | 21,978,369.95 | 1.1% | 902.00 | 232.36 | 3.882 | 132.00 | 0.0% | 0.01 | `create and destroy`
|
||||
|
||||
## Radix tree (this implementation)
|
||||
|
||||
|
||||
| ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark
|
||||
|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
|
||||
| 12.89 | 77,565,115.56 | 0.1% | 244.55 | 47.43 | 5.155 | 34.21 | 0.6% | 0.15 | `point reads`
|
||||
| 15.11 | 66,162,047.76 | 0.1% | 297.79 | 55.60 | 5.356 | 43.23 | 0.4% | 0.18 | `prefix reads`
|
||||
| 36.29 | 27,559,358.29 | 0.1% | 783.16 | 133.44 | 5.869 | 109.52 | 0.2% | 0.43 | `range reads`
|
||||
| 20.53 | 48,719,405.55 | 0.1% | 381.81 | 75.51 | 5.057 | 51.04 | 0.5% | 0.25 | `point writes`
|
||||
| 39.37 | 25,402,042.40 | 0.1% | 685.00 | 144.83 | 4.730 | 106.72 | 0.3% | 0.47 | `prefix writes`
|
||||
| 43.78 | 22,843,841.63 | 0.1% | 800.40 | 161.06 | 4.970 | 127.36 | 0.1% | 0.53 | `range writes`
|
||||
| 78.37 | 12,760,008.75 | 1.0% | 1,452.61 | 288.24 | 5.040 | 278.69 | 0.1% | 0.01 | `monotonic increasing point writes`
|
||||
| 322,885.50 | 3,097.07 | 1.5% | 4,362,382.00 | 1,183,852.00 | 3.685 | 765,301.00 | 0.1% | 0.01 | `worst case for radix tree`
|
||||
| 99.99 | 10,000,718.79 | 0.4% | 1,775.00 | 367.93 | 4.824 | 288.00 | 0.0% | 0.01 | `create and destroy`
|
||||
| 12.36 | 80,885,626.43 | 0.2% | 243.56 | 63.62 | 3.828 | 31.07 | 0.6% | 0.15 | `point reads`
|
||||
| 14.18 | 70,502,196.81 | 0.1% | 297.72 | 73.13 | 4.071 | 40.31 | 0.5% | 0.17 | `prefix reads`
|
||||
| 33.44 | 29,901,623.04 | 0.1% | 767.90 | 172.42 | 4.454 | 101.32 | 0.2% | 0.40 | `range reads`
|
||||
| 19.48 | 51,342,564.70 | 0.3% | 374.45 | 100.43 | 3.728 | 48.92 | 0.5% | 0.23 | `point writes`
|
||||
| 37.46 | 26,694,471.44 | 0.1% | 672.00 | 193.14 | 3.479 | 101.28 | 0.3% | 0.45 | `prefix writes`
|
||||
| 38.78 | 25,784,784.34 | 0.0% | 738.26 | 199.93 | 3.693 | 111.59 | 0.1% | 0.47 | `range writes`
|
||||
| 76.05 | 13,148,995.74 | 0.7% | 1,450.77 | 397.16 | 3.653 | 275.72 | 0.0% | 0.01 | `monotonic increasing point writes`
|
||||
| 286,920.33 | 3,485.29 | 0.4% | 4,117,948.00 | 1,521,352.00 | 2.707 | 714,833.00 | 0.1% | 0.01 | `worst case for radix tree`
|
||||
| 95.66 | 10,453,798.72 | 0.5% | 1,986.00 | 495.04 | 4.012 | 315.00 | 0.0% | 0.01 | `create and destroy`
|
||||
|
||||
# "Real data" test
|
||||
|
||||
@@ -51,13 +51,13 @@ Point queries only. Gc ratio is the ratio of time spent doing garbage collection
|
||||
## skip list
|
||||
|
||||
```
|
||||
Check: 4.62967 seconds, 352.195 MB/s, Add: 3.34177 seconds, 167.771 MB/s, Gc ratio: 37.9399%, Peak idle memory: 5.51852e+06
|
||||
Check: 4.53508 seconds, 371.81 MB/s, Add: 3.81222 seconds, 150.919 MB/s, Gc ratio: 33.66%, Peak idle memory: 5.61007e+06
|
||||
```
|
||||
|
||||
## radix tree
|
||||
|
||||
```
|
||||
Check: 1.00477 seconds, 1622.8 MB/s, Add: 1.21142 seconds, 462.808 MB/s, Gc ratio: 39.4716%, Peak idle memory: 2.0226e+06
|
||||
Check: 0.957735 seconds, 1760.6 MB/s, Add: 1.19942 seconds, 479.678 MB/s, Gc ratio: 38.6069%, Peak idle memory: 2.05667e+06
|
||||
```
|
||||
|
||||
## hash table
|
||||
@@ -65,6 +65,6 @@ Check: 1.00477 seconds, 1622.8 MB/s, Add: 1.21142 seconds, 462.808 MB/s, Gc rati
|
||||
(The hash table implementation doesn't work on range queries, and its purpose is to provide an idea of how fast point queries can be)
|
||||
|
||||
```
|
||||
Check: 0.854254 seconds, 1908.74 MB/s, Add: 0.632626 seconds, 886.232 MB/s, Gc ratio: 41.0827%, Peak idle memory: 0
|
||||
Check: 0.804598 seconds, 2095.69 MB/s, Add: 0.671221 seconds, 857.147 MB/s, Gc ratio: 35.0034%, Peak idle memory: 0
|
||||
```
|
||||
|
||||
|
||||
+6
-7
@@ -1,6 +1,5 @@
|
||||
#include <ConflictSet.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cerrno>
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
@@ -78,10 +77,10 @@ int main(int argc, const char **argv) {
|
||||
begin = end + 1;
|
||||
end = (uint8_t *)memchr(begin, '\n', size);
|
||||
|
||||
if (line.size() >= 2 && line[0] == 'P') {
|
||||
write = line.subspan(2, line.size() - 2);
|
||||
} else if (line.size() >= 2 && line[0] == 'L') {
|
||||
reads.push_back(line.subspan(2, line.size() - 2));
|
||||
if (line.size() > 0 && line[0] == 'P') {
|
||||
write = line.subspan(2, line.size());
|
||||
} else if (line.size() > 0 && line[0] == 'L') {
|
||||
reads.push_back(line.subspan(2, line.size()));
|
||||
} else if (line.empty()) {
|
||||
{
|
||||
readRanges.resize(reads.size());
|
||||
@@ -91,7 +90,7 @@ int main(int argc, const char **argv) {
|
||||
iter->begin.len = read.size();
|
||||
checkBytes += read.size();
|
||||
iter->end.len = 0;
|
||||
iter->readVersion = std::max<int64_t>(0, version - 100);
|
||||
iter->readVersion = version - 100;
|
||||
++iter;
|
||||
}
|
||||
}
|
||||
@@ -122,7 +121,7 @@ int main(int argc, const char **argv) {
|
||||
}
|
||||
|
||||
timer = now();
|
||||
cs.setOldestVersion(std::max<int64_t>(0, version - 10000));
|
||||
cs.setOldestVersion(version - 10000);
|
||||
gcTime += now() - timer;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
#include <sys/ioctl.h>
|
||||
#include <sys/resource.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/syscall.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/uio.h>
|
||||
#include <thread>
|
||||
|
||||
+4
-14
@@ -38,9 +38,6 @@ std::span<const uint8_t> keyAfter(Arena &arena, std::span<const uint8_t> key) {
|
||||
|
||||
std::span<const uint8_t> copyToArena(Arena &arena,
|
||||
std::span<const uint8_t> key) {
|
||||
if (key.size() == 0) {
|
||||
return std::span<const uint8_t>();
|
||||
}
|
||||
auto result = std::span<uint8_t>(new (arena) uint8_t[key.size()], key.size());
|
||||
memcpy(result.data(), key.data(), key.size());
|
||||
return result;
|
||||
@@ -100,7 +97,7 @@ force_inline bool getCharacter(const KeyInfo &ki, int character,
|
||||
|
||||
bool operator<(const KeyInfo &lhs, const KeyInfo &rhs) {
|
||||
int i = std::min(lhs.key.size(), rhs.key.size());
|
||||
int c = i > 0 ? memcmp(lhs.key.data(), rhs.key.data(), i) : 0;
|
||||
int c = memcmp(lhs.key.data(), rhs.key.data(), i);
|
||||
if (c != 0)
|
||||
return c < 0;
|
||||
|
||||
@@ -290,8 +287,7 @@ private:
|
||||
|
||||
static force_inline bool less(const uint8_t *a, int aLen, const uint8_t *b,
|
||||
int bLen) {
|
||||
int n = std::min(aLen, bLen);
|
||||
int c = n > 0 ? memcmp(a, b, n) : 0;
|
||||
int c = memcmp(a, b, std::min(aLen, bLen));
|
||||
if (c < 0)
|
||||
return true;
|
||||
if (c > 0)
|
||||
@@ -758,8 +754,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
|
||||
|
||||
void addWrites(const ConflictSet::WriteRange *writes, int count,
|
||||
int64_t writeVersion) {
|
||||
auto points = std::vector<KeyInfo>();
|
||||
points.reserve(count * 2);
|
||||
auto points = std::vector<KeyInfo>(count * 2);
|
||||
Arena arena;
|
||||
|
||||
for (int r = 0; r < count; r++) {
|
||||
@@ -986,12 +981,7 @@ ConflictSet::ConflictSet(ConflictSet &&other) noexcept
|
||||
: impl(std::exchange(other.impl, nullptr)) {}
|
||||
|
||||
ConflictSet &ConflictSet::operator=(ConflictSet &&other) noexcept {
|
||||
if (this != &other) {
|
||||
if (impl) {
|
||||
internal_destroy(impl);
|
||||
}
|
||||
impl = std::exchange(other.impl, nullptr);
|
||||
}
|
||||
impl = std::exchange(other.impl, nullptr);
|
||||
return *this;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
LLVM_VERSION="${LLVM_VERSION:-21}"
|
||||
MSAN_PREFIX="${MSAN_PREFIX:-$PWD/msan}"
|
||||
JOBS="${JOBS:-$(nproc)}"
|
||||
|
||||
cd /tmp
|
||||
rm -rf libcxx-msan
|
||||
mkdir libcxx-msan
|
||||
cd libcxx-msan
|
||||
|
||||
git clone --depth=1 "https://github.com/llvm/llvm-project.git" -b "release/${LLVM_VERSION}.x"
|
||||
|
||||
cmake -S llvm-project/runtimes -B build_msan \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_CXX_COMPILER="clang++-${LLVM_VERSION}" \
|
||||
-DCMAKE_C_COMPILER="clang-${LLVM_VERSION}" \
|
||||
-DCMAKE_INSTALL_PREFIX="${MSAN_PREFIX}" \
|
||||
-DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
|
||||
-DLLVM_USE_SANITIZER=MemoryWithOrigins
|
||||
|
||||
cmake --build build_msan -j"${JOBS}"
|
||||
cmake --install build_msan
|
||||
|
||||
cmake -S llvm-project/runtimes -B build_nomsan \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_CXX_COMPILER="clang++-${LLVM_VERSION}" \
|
||||
-DCMAKE_C_COMPILER="clang-${LLVM_VERSION}" \
|
||||
-DCMAKE_INSTALL_PREFIX="${MSAN_PREFIX}" \
|
||||
-DLLVM_ENABLE_RUNTIMES="libunwind"
|
||||
|
||||
cmake --build build_nomsan -j"${JOBS}"
|
||||
cmake --install build_nomsan
|
||||
|
||||
VERSION="$(clang-"${LLVM_VERSION}" --version | head -n1 | sed -E 's/.*clang version ([0-9.]+).*/\1/')"
|
||||
TARBALL="msan-toolchain-${VERSION}.tar.zst"
|
||||
|
||||
tar --zstd -cf "${TARBALL}" -C "${MSAN_PREFIX}" .
|
||||
|
||||
TARBALL_PATH="$(pwd)/${TARBALL}"
|
||||
echo "Created: ${TARBALL_PATH}"
|
||||
+9
-27
@@ -27,37 +27,23 @@ class Result(enum.Enum):
|
||||
TOO_OLD = 2
|
||||
|
||||
|
||||
def _make_key(buf: bytes) -> tuple[_Key, bytearray]:
|
||||
"""Create a _Key and a backing bytearray that must be kept alive."""
|
||||
backing = bytearray(buf)
|
||||
array = (ctypes.c_ubyte * len(backing)).from_buffer(backing)
|
||||
return _Key(array, len(array)), backing
|
||||
|
||||
|
||||
def write(begin: bytes, end: Optional[bytes] = None) -> WriteRange:
|
||||
begin_key, begin_buf = _make_key(begin)
|
||||
b = (ctypes.c_ubyte * len(begin)).from_buffer(bytearray(begin))
|
||||
|
||||
if end is None:
|
||||
end_key = _Key((ctypes.c_ubyte * 0)(), 0)
|
||||
end_buf = None
|
||||
e = (ctypes.c_ubyte * 0)()
|
||||
else:
|
||||
end_key, end_buf = _make_key(end)
|
||||
result = WriteRange(begin_key, end_key)
|
||||
result._begin_buf = begin_buf
|
||||
result._end_buf = end_buf
|
||||
return result
|
||||
e = (ctypes.c_ubyte * len(end)).from_buffer(bytearray(end))
|
||||
return WriteRange(_Key(b, len(b)), _Key(e, len(e)))
|
||||
|
||||
|
||||
def read(version: int, begin: bytes, end: Optional[bytes] = None) -> ReadRange:
|
||||
begin_key, begin_buf = _make_key(begin)
|
||||
b = (ctypes.c_ubyte * len(begin)).from_buffer(bytearray(begin))
|
||||
if end is None:
|
||||
end_key = _Key((ctypes.c_ubyte * 0)(), 0)
|
||||
end_buf = None
|
||||
e = (ctypes.c_ubyte * 0)()
|
||||
else:
|
||||
end_key, end_buf = _make_key(end)
|
||||
result = ReadRange(begin_key, end_key, version)
|
||||
result._begin_buf = begin_buf
|
||||
result._end_buf = end_buf
|
||||
return result
|
||||
e = (ctypes.c_ubyte * len(end)).from_buffer(bytearray(end))
|
||||
return ReadRange(_Key(b, len(b)), _Key(e, len(e)), version)
|
||||
|
||||
|
||||
class ConflictSet:
|
||||
@@ -102,7 +88,6 @@ class ConflictSet:
|
||||
ctypes.POINTER(ctypes.c_int),
|
||||
ctypes.c_int,
|
||||
)
|
||||
self._lib.ConflictSet_check.restype = None
|
||||
|
||||
self._lib.ConflictSet_addWrites.argtypes = (
|
||||
ctypes.c_void_p,
|
||||
@@ -110,16 +95,13 @@ class ConflictSet:
|
||||
ctypes.c_int,
|
||||
ctypes.c_int64,
|
||||
)
|
||||
self._lib.ConflictSet_addWrites.restype = None
|
||||
|
||||
self._lib.ConflictSet_setOldestVersion.argtypes = (
|
||||
ctypes.c_void_p,
|
||||
ctypes.c_int64,
|
||||
)
|
||||
self._lib.ConflictSet_setOldestVersion.restype = None
|
||||
|
||||
self._lib.ConflictSet_destroy.argtypes = (ctypes.c_void_p,)
|
||||
self._lib.ConflictSet_destroy.restype = None
|
||||
|
||||
self._lib.ConflictSet_getBytes.argtypes = (ctypes.c_void_p,)
|
||||
self._lib.ConflictSet_getBytes.restype = ctypes.c_int64
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user