Compare commits

..
1 Commits
Author SHA1 Message Date
weaselbot 25307e9ed6 Include <sys/syscall.h> in ServerBench.cpp for SYS_perf_event_open
CI / build-image (arm64, ubuntu-latest-arm64) (pull_request) Failing after 20s
CI / build-image (amd64, ubuntu-latest-amd64) (pull_request) Failing after 37s
CI / pre-commit (pull_request) Has been skipped
CI / test (-DCMAKE_BUILD_TYPE=Debug, debug) (pull_request) Has been skipped
CI / test (-DCMAKE_CXX_FLAGS=-DUSE_64_BIT=1, 64-bit-versions) (pull_request) Has been skipped
CI / test (-DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++, gcc) (pull_request) Has been skipped
CI / test (-DUSE_SIMD_FALLBACK=ON, simd-fallback) (pull_request) Has been skipped
CI / release (amd64, ubuntu-latest-amd64) (pull_request) Has been skipped
CI / release (arm64, ubuntu-latest-arm64) (pull_request) Has been skipped
CI / coverage (pull_request) Has been skipped
`ServerBench.cpp` calls `syscall(SYS_perf_event_open, ...)` but did
not include `<sys/syscall.h>`, relying on `<unistd.h>` to transitively
provide the `SYS_*` constants. On toolchains where that does not happen,
the build fails with `SYS_perf_event_open` not declared.

Add the missing include so `server_bench` compiles on any platform that
provides `perf_event_open`.

Resolves weaselab/conflict-set#41.
2026-06-18 22:23:20 -04:00
107 changed files with 375 additions and 2844 deletions

No files matched your search

+76 -133
View File
@@ -2,108 +2,89 @@ name: CI
on: [push, pull_request]
env:
CC: clang
CXX: clang++
jobs:
pre-commit:
runs-on: ubuntu-latest-amd64
build-image:
strategy:
fail-fast: false
matrix:
include:
- runner: ubuntu-latest-amd64
arch: amd64
- runner: ubuntu-latest-arm64
arch: arm64
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
with:
path: /var/cache/apt/archives
key: apt-amd64-${{ hashFiles('.gitea/workflows/ci.yml') }}
- name: Install dependencies
- name: Log in to registry
env:
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
run: |
sudo apt-get update -qq
sudo apt-get install -y git nodejs pre-commit
echo "$REGISTRY_TOKEN" \
| docker login -u "$REGISTRY_USER" --password-stdin git.weaselab.dev
- uses: actions/cache@v4
with:
path: ~/.cache/pre-commit
key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
- name: Build and push image if changed
run: |
image=git.weaselab.dev/weaselab/conflict-set-ci
hash="$(sha256sum Dockerfile .pre-commit-config.yaml | sha256sum | cut -c 1-16)"
latest="$image:latest-${{ matrix.arch }}"
current="$(docker buildx imagetools inspect "$latest" \
--format '{{index .Image.Config.Labels "dev.weaselab.ci-hash"}}' 2> /dev/null || true)"
if [ "$current" = "$hash" ]; then
echo "$latest is up to date"
else
docker build --push --label "dev.weaselab.ci-hash=$hash" -t "$latest" .
fi
pre-commit:
needs: build-image
runs-on: ubuntu-latest-amd64
container:
image: git.weaselab.dev/weaselab/conflict-set-ci:latest-amd64
credentials:
username: ${{ secrets.REGISTRY_USER }}
password: ${{ secrets.REGISTRY_TOKEN }}
steps:
- uses: actions/checkout@v4
- name: Run pre-commit
env:
# use the hooks pre-installed in the image
HOME: /tmp
run: |
git config --global --add safe.directory "$PWD"
pre-commit run --all-files --show-diff-on-failure
test:
needs: build-image
strategy:
fail-fast: false
matrix:
include:
- name: 64-bit-versions
runner: ubuntu-latest-amd64
arch: amd64
llvm_version: "21"
msan_url: ""
cmake_args: -DCMAKE_CXX_FLAGS=-DUSE_64_BIT=1
- name: debug
runner: ubuntu-latest-amd64
arch: amd64
llvm_version: "21"
msan_url: https://minio.weaselab.dev/public/x86_64/msan-toolchain-21.1.8.tar.zst
cmake_args: -DCMAKE_BUILD_TYPE=Debug -DMSAN_TOOLCHAIN_PATH=/opt/msan
cmake_args: -DCMAKE_BUILD_TYPE=Debug
- name: simd-fallback
runner: ubuntu-latest-amd64
arch: amd64
llvm_version: "21"
msan_url: ""
cmake_args: -DUSE_SIMD_FALLBACK=ON
- name: gcc
runner: ubuntu-latest-amd64
arch: amd64
llvm_version: "21"
msan_url: ""
cmake_args: -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++
- name: debug-arm64
runner: ubuntu-latest-arm64
arch: arm64
llvm_version: "22"
msan_url: https://minio.weaselab.dev/public/aarch64/msan-toolchain-22.1.8.tar.zst
cmake_args: -DCMAKE_BUILD_TYPE=Debug -DMSAN_TOOLCHAIN_PATH=/opt/msan
runs-on: ${{ matrix.runner }}
runs-on: ubuntu-latest-amd64
container:
image: git.weaselab.dev/weaselab/conflict-set-ci:latest-amd64
credentials:
username: ${{ secrets.REGISTRY_USER }}
password: ${{ secrets.REGISTRY_TOKEN }}
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
with:
path: /var/cache/apt/archives
key: apt-${{ matrix.arch }}-${{ hashFiles('.gitea/workflows/ci.yml') }}
- name: Install common dependencies
run: |
sudo apt-get update -qq
sudo apt-get install -y \
build-essential ccache cmake gcc g++ libc6-dbg \
ninja-build python3 valgrind zstd
sudo curl -Ls "https://minio.weaselab.dev/public/$(uname -m)/mc.RELEASE.2025-08-13T08-35-41Z" \
-o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
- name: Install LLVM toolchain
if: matrix.name != 'gcc'
run: |
. /etc/os-release
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-${{ matrix.llvm_version }} main" | sudo tee /etc/apt/sources.list.d/llvm.list
sudo apt-get update -qq
sudo apt-get install -y \
clang-${{ matrix.llvm_version }} llvm-${{ matrix.llvm_version }} lld-${{ matrix.llvm_version }} mold
for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do
sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-${{ matrix.llvm_version }} 100
done
- name: Download MSan toolchain
if: matrix.msan_url != ''
run: |
curl -Ls "${{ matrix.msan_url }}" -o /tmp/msan-toolchain.tar.zst
sudo mkdir -p /opt/msan
sudo tar --zstd -xf /tmp/msan-toolchain.tar.zst -C /opt/msan
path: .ccache
key: ccache-${{ matrix.name }}-${{ gitea.sha }}
restore-keys: |
ccache-${{ matrix.name }}-
- name: Build
run: |
@@ -139,49 +120,24 @@ jobs:
| tee -a "$GITHUB_STEP_SUMMARY"
release:
needs: build-image
strategy:
fail-fast: false
matrix:
include:
- runner: ubuntu-latest-amd64
arch: amd64
cmake_args: -DMSAN_TOOLCHAIN_PATH=/opt/msan
- runner: ubuntu-latest-arm64
arch: arm64
cmake_args: ""
runs-on: ${{ matrix.runner }}
container:
image: git.weaselab.dev/weaselab/conflict-set-ci:latest-${{ matrix.arch }}
credentials:
username: ${{ secrets.REGISTRY_USER }}
password: ${{ secrets.REGISTRY_TOKEN }}
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
with:
path: /var/cache/apt/archives
key: apt-${{ matrix.arch }}-${{ hashFiles('.gitea/workflows/ci.yml') }}
- name: Install dependencies
run: |
. /etc/os-release
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
sudo apt-get update -qq
sudo apt-get install -y \
build-essential ccache clang-21 cmake devscripts \
libc6-dbg llvm-21 lld-21 mold ninja-build rpm \
valgrind zstd
sudo curl -Ls "https://minio.weaselab.dev/public/$(uname -m)/mc.RELEASE.2025-08-13T08-35-41Z" \
-o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do
sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
done
- name: Install paper build dependencies
if: matrix.arch == 'amd64'
run: |
sudo apt-get install -y \
biber latexmk \
texlive-bibtex-extra texlive-fonts-recommended \
texlive-latex-extra texlive-pictures
- uses: actions/cache@v4
with:
path: .ccache
@@ -189,25 +145,23 @@ jobs:
restore-keys: |
ccache-release-${{ matrix.arch }}-
- name: Download MSan toolchain
if: matrix.arch == 'amd64'
run: |
curl -Ls "https://minio.weaselab.dev/public/x86_64/msan-toolchain-21.1.8.tar.zst" -o /tmp/msan-toolchain.tar.zst
sudo mkdir -p /opt/msan
sudo tar --zstd -xf /tmp/msan-toolchain.tar.zst -C /opt/msan
- name: Build
run: |
export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache"
rm -rf build
cmake -S . -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_FLAGS=-DNVALGRIND ${{ matrix.cmake_args }}
cmake -S . -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_FLAGS=-DNVALGRIND
ninja -C build
ccache -s
- name: Test
run: |
cd build
ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 -T Test -j "$(nproc)" --timeout 90 > /dev/null
# On arm64, valgrind needs the MAKE_MEM_DEFINED client requests for
# https://git.weaselab.dev/weaselab/conflict-set/issues/39, but this
# build has -DNVALGRIND, which compiles them out. Skip valgrind
# tests here; they run annotated in the test job.
ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 ${{ matrix.arch == 'arm64' && '-E valgrind' || '' }} -T Test -j "$(nproc)" --timeout 90 > /dev/null
- name: Package
run: |
cd build
@@ -234,6 +188,9 @@ jobs:
dest="minio/jenkins/conflict-set/${{ gitea.run_number }}/release-${{ matrix.arch }}/"
zstd build/Testing/*/Test.xml
mc cp build/Testing/*/Test.xml.zst "$dest"
# This step runs even when a previous step failed, to upload test
# results. The packages may never have been built though, so skip
# them if they're missing.
if compgen -G "build/*.deb" > /dev/null; then
mc cp build/*.deb "$dest"
fi
@@ -252,30 +209,16 @@ jobs:
| tee -a "$GITHUB_STEP_SUMMARY"
coverage:
needs: build-image
runs-on: ubuntu-latest-amd64
container:
image: git.weaselab.dev/weaselab/conflict-set-ci:latest-amd64
credentials:
username: ${{ secrets.REGISTRY_USER }}
password: ${{ secrets.REGISTRY_TOKEN }}
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
with:
path: /var/cache/apt/archives
key: apt-amd64-${{ hashFiles('.gitea/workflows/ci.yml') }}
- name: Install dependencies
run: |
. /etc/os-release
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
sudo apt-get update -qq
sudo apt-get install -y \
build-essential ccache clang-21 cmake gcovr \
libc6-dbg llvm-21 lld-21 mold ninja-build python3 valgrind zstd
sudo curl -Ls "https://minio.weaselab.dev/public/$(uname -m)/mc.RELEASE.2025-08-13T08-35-41Z" \
-o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do
sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
done
- uses: actions/cache@v4
with:
path: .ccache
+4 -5
View File
@@ -37,16 +37,15 @@ ConflictSet::ReadRange singleton(Arena &arena, TrivialSpan key) {
}
ConflictSet::ReadRange prefixRange(Arena &arena, TrivialSpan key) {
int index = key.size() - 1;
for (; index >= 0; index--)
if (key[index] != 255)
int index;
for (index = key.size() - 1; index >= 0; index--)
if ((key[index]) != 255)
break;
// Must not be called with a string that consists only of zero or more '\xff'
// bytes, or with an empty string (which has no finite upper bound).
// bytes.
if (index < 0) {
assert(false);
std::abort();
}
uint8_t *buf = new (arena) uint8_t[index + 1];
+6 -76
View File
@@ -1,7 +1,7 @@
cmake_minimum_required(VERSION 3.18)
project(
conflict-set
VERSION 0.1.0
VERSION 0.0.14
DESCRIPTION
"A data structure for optimistic concurrency control on ranges of bitwise-lexicographically-ordered keys."
HOMEPAGE_URL "https://git.weaselab.dev/weaselab/conflict-set"
@@ -103,13 +103,6 @@ option(USE_SIMD_FALLBACK
option(DISABLE_TSAN "Disable TSAN" OFF)
set(MSAN_TOOLCHAIN_PATH
""
CACHE
PATH
"Path to an MSan-instrumented libc++ toolchain. When set, an MSan fuzz_driver target is added. Must contain include/c++/v1 and lib. See build_msan_toolchain.sh to produce one."
)
# This is encouraged according to
# https://valgrind.org/docs/manual/manual-core-adv.html#manual-core-adv.clientreq
include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/valgrind)
@@ -146,7 +139,6 @@ add_custom_command(
COMMAND_EXPAND_LISTS)
add_library(${PROJECT_NAME} SHARED ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o)
add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-object)
set_target_properties(
${PROJECT_NAME} PROPERTIES LIBRARY_OUTPUT_DIRECTORY
"${CMAKE_CURRENT_BINARY_DIR}/radix_tree")
@@ -163,7 +155,6 @@ if(HAS_VERSION_SCRIPT)
endif()
add_library(${PROJECT_NAME}-static STATIC ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o)
add_dependencies(${PROJECT_NAME}-static ${PROJECT_NAME}-object)
if(CMAKE_BUILD_TYPE STREQUAL Debug)
set_target_properties(${PROJECT_NAME}-static PROPERTIES LINKER_LANGUAGE CXX)
else()
@@ -182,8 +173,6 @@ endif()
include(CTest)
option(TEST_SKIP_LIST "Run the slow skip list cross-check tests" OFF)
# disable tests if this is being used through e.g. FetchContent
if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
@@ -226,9 +215,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
target_compile_options(driver_skip_list PRIVATE ${TEST_FLAGS})
target_link_libraries(driver_skip_list PRIVATE skip_list)
# Optional cross-check of SkipList.cpp against ReferenceImpl. Opt-in via
# -DTEST_SKIP_LIST=ON, since these are slow.
if(TEST_SKIP_LIST)
# enable to test skip list
if(0)
foreach(TEST ${CORPUS_TESTS})
get_filename_component(hash ${TEST} NAME)
add_test(NAME skip_list_${hash} COMMAND driver_skip_list ${TEST})
@@ -263,7 +251,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
endif()
endif()
# whitebox tests asan+ubsan
# whitebox tests
add_executable(fuzz_driver ConflictSet.cpp FuzzTestDriver.cpp)
target_compile_options(fuzz_driver PRIVATE ${TEST_FLAGS})
if(NOT CMAKE_CROSSCOMPILING)
@@ -278,34 +266,6 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
add_test(NAME conflict_set_fuzz_${hash} COMMAND fuzz_driver ${TEST})
endforeach()
# whitebox tests msan
if(MSAN_TOOLCHAIN_PATH)
add_executable(fuzz_driver_msan ConflictSet.cpp FuzzTestDriver.cpp)
target_compile_options(fuzz_driver_msan PRIVATE ${TEST_FLAGS})
if(NOT CMAKE_CROSSCOMPILING)
target_compile_options(
fuzz_driver_msan
PRIVATE -fsanitize=memory -fsanitize-memory-track-origins=2
$<$<COMPILE_LANGUAGE:CXX>:-stdlib=libc++>
-I${MSAN_TOOLCHAIN_PATH}/include/c++/v1)
target_link_options(
fuzz_driver_msan
PRIVATE
-fsanitize=memory
-fsanitize-memory-track-origins=2
-stdlib=libc++
-L${MSAN_TOOLCHAIN_PATH}/lib
LINKER:-rpath,${MSAN_TOOLCHAIN_PATH}/lib)
endif()
target_compile_definitions(fuzz_driver_msan PRIVATE ENABLE_FUZZ)
target_include_directories(fuzz_driver_msan
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
foreach(TEST ${CORPUS_TESTS})
get_filename_component(hash ${TEST} NAME)
add_test(NAME conflict_set_msan_${hash} COMMAND fuzz_driver_msan ${TEST})
endforeach()
endif()
# tsan tests
if(NOT CMAKE_CROSSCOMPILING AND NOT DISABLE_TSAN)
add_executable(tsan_driver ConflictSet.cpp FuzzTestDriver.cpp)
@@ -393,24 +353,6 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
PROPERTIES CXX_STANDARD_REQUIRED ON)
add_test(NAME conflict_set_cxx_api_test COMMAND conflict_set_cxx_api_test)
# SHOW_MEMORY instrumentation, which the rest of the test suite doesn't
# compile with
add_executable(conflict_set_show_memory_test ConflictSet.cpp
ShowMemoryTest.cpp)
target_compile_options(conflict_set_show_memory_test PRIVATE ${TEST_FLAGS})
target_compile_definitions(conflict_set_show_memory_test
PRIVATE SHOW_MEMORY=1)
target_include_directories(conflict_set_show_memory_test
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
if(NOT CMAKE_CROSSCOMPILING)
target_compile_options(conflict_set_show_memory_test
PRIVATE -fsanitize=address,undefined)
target_link_options(conflict_set_show_memory_test PRIVATE
-fsanitize=address,undefined)
endif()
add_test(NAME conflict_set_show_memory_test
COMMAND conflict_set_show_memory_test)
# symbol visibility tests
if(NOT CMAKE_BUILD_TYPE STREQUAL Debug)
if(APPLE)
@@ -441,25 +383,13 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
if(NOT CMAKE_CROSSCOMPILING)
find_program(HARDENING_CHECK hardening-check)
if(HARDENING_CHECK)
# Not all versions of hardening-check support the same options, so query
# the help output before using architecture-specific skips.
execute_process(
COMMAND ${HARDENING_CHECK} --help
OUTPUT_VARIABLE hardening_check_help
ERROR_VARIABLE hardening_check_help
OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE)
set(hardening_check_arch_flags "")
# Control flow integrity (CET) is x86-only and branch protection (PAC/BTI)
# is arm64-only, so ignore whichever doesn't apply.
if(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64 OR CMAKE_SYSTEM_PROCESSOR
STREQUAL arm64)
if(hardening_check_help MATCHES "nocfprotection")
list(APPEND hardening_check_arch_flags --nocfprotection)
endif()
set(hardening_check_arch_flags --nocfprotection)
else()
if(hardening_check_help MATCHES "nobranchprotection")
list(APPEND hardening_check_arch_flags --nobranchprotection)
endif()
set(hardening_check_arch_flags --nobranchprotection)
endif()
add_test(
NAME hardening_check
+119 -268
View File
@@ -80,19 +80,17 @@ using namespace weaselab;
#define assume assert
#endif
// The SHOW_MEMORY instrumentation. Declared unconditionally so that each of
// these has a single definition in every configuration (see the definitions at
// the end of this file), rather than a no-op stub standing in for the real
// thing.
#if SHOW_MEMORY
void addNode(struct Node *);
void removeNode(struct Node *);
void addKey(struct Node *);
void removeKey(struct Node *);
// Like `removeKey`, but with a caller-provided search path length. Used when
// the node's ancestors may already have been released, which makes walking
// `parent` pointers invalid.
void removeKey(struct Node *, int64_t searchPathLength);
int64_t getSearchPathLength(struct Node *);
#else
constexpr void addNode(struct Node *) {}
constexpr void removeNode(struct Node *) {}
constexpr void addKey(struct Node *) {}
constexpr void removeKey(struct Node *) {}
#endif
// ==================== BEGIN IMPLEMENTATION ====================
@@ -118,40 +116,13 @@ struct InternalVersionT {
constexpr InternalVersionT() = default;
constexpr explicit InternalVersionT(int64_t value) : value(value) {}
constexpr int64_t toInt64() const { return value; } // GCOVR_EXCL_LINE
constexpr bool operator<(const InternalVersionT &rhs) const {
constexpr auto operator<=>(const InternalVersionT &rhs) const {
#if USE_64_BIT
return value < rhs.value;
return value <=> rhs.value;
#else
// Maintains ordering after overflow, as long as the full-precision versions
// are within `kMaxCorrectVersionWindow` of eachother.
return int32_t(value - rhs.value) < 0;
#endif
}
constexpr bool operator<=(const InternalVersionT &rhs) const {
#if USE_64_BIT
return value <= rhs.value;
#else
// Maintains ordering after overflow, as long as the full-precision versions
// are within `kMaxCorrectVersionWindow` of eachother.
return int32_t(value - rhs.value) <= 0;
#endif
}
constexpr bool operator>=(const InternalVersionT &rhs) const {
#if USE_64_BIT
return value >= rhs.value;
#else
// Maintains ordering after overflow, as long as the full-precision versions
// are within `kMaxCorrectVersionWindow` of eachother.
return int32_t(value - rhs.value) >= 0;
#endif
}
constexpr bool operator>(const InternalVersionT &rhs) const {
#if USE_64_BIT
return value > rhs.value;
#else
// Maintains ordering after overflow, as long as the full-precision versions
// are within `kMaxCorrectVersionWindow` of eachother.
return int32_t(value - rhs.value) > 0;
return int32_t(value - rhs.value) <=> 0;
#endif
}
constexpr bool operator==(const InternalVersionT &) const = default;
@@ -336,9 +307,7 @@ struct Node {
uint8_t *partialKey();
Type getType() const { return type; }
int32_t getCapacity() const {
// `partialKeyCapacity` is not repurposed by `deferRelease`, so this is also
// valid for nodes with `releaseDeferred` set. It's needed to account for
// the capacity of a deferred node when it's finally released.
assert(!releaseDeferred);
return partialKeyCapacity;
}
@@ -519,7 +488,6 @@ inline void Node3::copyChildrenAndKeyFrom(const Node16 &other) {
inline void Node16::copyChildrenAndKeyFrom(const Node3 &other) {
copyCommon(*this, other);
memset(index, 0, sizeof(index));
memcpy(index, other.index, Node3::kMaxNodes);
memcpy(children, other.children,
Node3::kMaxNodes * sizeof(children[0])); // NOLINT
@@ -531,12 +499,6 @@ inline void Node16::copyChildrenAndKeyFrom(const Node3 &other) {
assert(children[i]->parent == &other);
children[i]->parent = this;
}
// The index bytes beyond numChildren are initialized so that the SIMD loads
// of the full 16 bytes are well-defined (reading indeterminate bytes is UB
// in C++). valgrind then treats them as undefined so that it can still flag
// any read that relies on them.
VALGRIND_MAKE_MEM_UNDEFINED(index + numChildren,
sizeof(index) - numChildren * sizeof(index[0]));
}
inline void Node16::copyChildrenAndKeyFrom(const Node16 &other) {
@@ -553,7 +515,6 @@ inline void Node16::copyChildrenAndKeyFrom(const Node16 &other) {
inline void Node16::copyChildrenAndKeyFrom(const Node48 &other) {
copyCommon(*this, other);
memset(index, 0, sizeof(index));
int i = 0;
other.bitSet.forEachSet([&](int c) {
// Suppress a false positive -Waggressive-loop-optimizations warning
@@ -567,12 +528,6 @@ inline void Node16::copyChildrenAndKeyFrom(const Node48 &other) {
++i;
});
memcpy(partialKey(), &other + 1, partialKeyLen);
// The index bytes beyond numChildren are initialized so that the SIMD loads
// of the full 16 bytes are well-defined (reading indeterminate bytes is UB
// in C++). valgrind then treats them as undefined so that it can still flag
// any read that relies on them.
VALGRIND_MAKE_MEM_UNDEFINED(index + numChildren,
sizeof(index) - numChildren * sizeof(index[0]));
}
inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) {
@@ -580,7 +535,6 @@ inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) {
assert(numChildren == Node16::kMaxNodes);
memset(index, -1, sizeof(index));
memset(children, 0, sizeof(children));
memset(reverseIndex, 0, sizeof(reverseIndex));
const auto z = InternalVersionT::zero;
for (auto &v : childMaxVersion) {
v = z;
@@ -600,9 +554,6 @@ inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) {
std::max(maxOfMax[i >> Node48::kMaxOfMaxShift], childMaxVersion[i]);
++i;
}
// The reverseIndex bytes beyond numChildren are initialized so that the
// SIMD loads of a full 16-byte page in scan16 are well-defined (reading
// indeterminate bytes is UB in C++).
}
inline void Node48::copyChildrenAndKeyFrom(const Node48 &other) {
@@ -629,7 +580,6 @@ inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) {
copyCommon(*this, other);
memset(index, -1, sizeof(index));
memset(children, 0, sizeof(children));
memset(reverseIndex, 0, sizeof(reverseIndex));
const auto z = InternalVersionT::zero;
for (auto &v : childMaxVersion) {
v = z;
@@ -651,9 +601,6 @@ inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) {
++i;
});
memcpy(partialKey(), &other + 1, partialKeyLen);
// The reverseIndex bytes beyond numChildren are initialized so that the
// SIMD loads of a full 16-byte page in scan16 are well-defined (reading
// indeterminate bytes is UB in C++).
}
inline void Node256::copyChildrenAndKeyFrom(const Node48 &other) {
@@ -1385,8 +1332,8 @@ TaggedNodePointer getChildGeq(Node *self, int child) {
TaggedNodePointer getFirstChild(Node0 *) { return nullptr; }
TaggedNodePointer getFirstChild(Node3 *self) {
// Don't prefetch since self->children[1] might be uninitialized and msan
// doesn't like that
// Improves scan performance
self->children[1].prefetch();
return self->children[0];
}
TaggedNodePointer getFirstChild(Node16 *self) {
@@ -2045,6 +1992,7 @@ Node *erase(Node *self, WriteContext *writeContext, bool logical) {
parent48->index[parentIndex] = toRemoveChildrenIndex;
parent48->reverseIndex[toRemoveChildrenIndex] = parentIndex;
}
parent48->childMaxVersion[lastChildrenIndex] = writeContext->zero;
VALGRIND_MAKE_MEM_UNDEFINED(parent48->children + lastChildrenIndex,
sizeof(parent48->children[0]));
@@ -2139,53 +2087,6 @@ compare16_avx512(const InternalVersionT *vs, InternalVersionT rv) {
}
#endif
#ifdef HAS_ARM_NEON
// Returns a uint8x16_t whose byte i is 0xff if vs[i] > readVersion and 0
// otherwise. When USE_64_BIT is set the comparison uses the full 64-bit value
// (via vcgtq_s64); otherwise it falls back to the 32-bit subtract-and-compare
// used by the original NEON fast path. The caller narrows this 16-byte vector
// into the nibble-packed (4 bits per version) mask expected by scan16 and
// checkMaxBetweenExclusiveImpl.
inline uint8x16_t conflictMask16(const InternalVersionT *vs,
InternalVersionT readVersion) {
#if USE_64_BIT
int64_t rv;
memcpy(&rv, &readVersion, sizeof(rv));
const auto rvVec = vdupq_n_s64(rv);
int32x2_t r32[8];
const auto *vsp = reinterpret_cast<const int64_t *>(vs);
for (int j = 0; j < 8; ++j) {
r32[j] = vmovn_s64(
vreinterpretq_s64_u64(vcgtq_s64(vld1q_s64(vsp + 2 * j), rvVec)));
}
uint32x4_t w4[4];
for (int k = 0; k < 4; ++k) {
w4[k] = vreinterpretq_u32_s32(vcombine_s32(r32[2 * k], r32[2 * k + 1]));
}
return vcombine_u8(
vmovn_u16(vcombine_u16(vmovn_u32(w4[0]), vmovn_u32(w4[1]))),
vmovn_u16(vcombine_u16(vmovn_u32(w4[2]), vmovn_u32(w4[3]))));
#else
uint32x4_t w4[4];
memcpy(w4, vs, sizeof(w4));
uint32_t rv;
memcpy(&rv, &readVersion, sizeof(rv));
const auto rvVec = vdupq_n_u32(rv);
int32x4_t z;
memset(&z, 0, sizeof(z));
uint16x4_t conflicting[4];
for (int i = 0; i < 4; ++i) {
conflicting[i] =
vmovn_u32(vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
}
return vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
vmovn_u16(vcombine_u16(conflicting[2], conflicting[3])));
#endif
}
#endif
// Returns true if v[i] <= readVersion for all i such that begin <= is[i] < end
// Preconditions: begin <= end, end - begin < 256
template <bool kAVX512>
@@ -2206,10 +2107,26 @@ bool scan16(const InternalVersionT *vs, const uint8_t *is, int begin, int end,
uint64_t mask = vget_lane_u64(
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(results), 4)), 0);
uint32x4_t w4[4];
memcpy(w4, vs, sizeof(w4));
uint32_t rv;
memcpy(&rv, &readVersion, sizeof(rv));
const auto rvVec = vdupq_n_u32(rv);
int32x4_t z;
memset(&z, 0, sizeof(z));
uint16x4_t conflicting[4];
for (int i = 0; i < 4; ++i) {
conflicting[i] =
vmovn_u32(vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
}
auto combined =
vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
vmovn_u16(vcombine_u16(conflicting[2], conflicting[3])));
uint64_t compared = vget_lane_u64(
vreinterpret_u64_u8(vshrn_n_u16(
vreinterpretq_u16_u8(conflictMask16(vs, readVersion)), 4)),
0);
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(combined), 4)), 0);
return !(compared & mask);
@@ -2236,14 +2153,12 @@ bool scan16(const InternalVersionT *vs, const uint8_t *is, int begin, int end,
auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
uint32_t compared = 0;
for (int i = 0; i < 16; ++i) {
compared |= (vs[i] > readVersion) << i;
}
uint32_t mask = 0;
for (int i = 0; i < 16; ++i) {
if (vs[i] > readVersion) {
compared |= 1u << i;
if (inBounds(is[i])) {
mask |= 1u << i;
}
}
mask |= inBounds(is[i]) << i;
}
return !(compared & mask);
@@ -2259,10 +2174,26 @@ bool scan16(const InternalVersionT *vs, int begin, int end,
assert(begin <= end);
#if defined(HAS_ARM_NEON)
uint32x4_t w4[4];
memcpy(w4, vs, sizeof(w4));
uint32_t rv;
memcpy(&rv, &readVersion, sizeof(rv));
const auto rvVec = vdupq_n_u32(rv);
int32x4_t z;
memset(&z, 0, sizeof(z));
uint16x4_t conflicting[4];
for (int i = 0; i < 4; ++i) {
conflicting[i] =
vmovn_u32(vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
}
auto combined =
vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
vmovn_u16(vcombine_u16(conflicting[2], conflicting[3])));
uint64_t conflict = vget_lane_u64(
vreinterpret_u64_u8(vshrn_n_u16(
vreinterpretq_u16_u8(conflictMask16(vs, readVersion)), 4)),
0);
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(combined), 4)), 0);
conflict &= end == 16 ? -1 : (uint64_t(1) << (end << 2)) - 1;
conflict >>= begin << 2;
@@ -2311,8 +2242,6 @@ bool checkMaxBetweenExclusiveImpl(Node3 *n, int begin, int end,
assert(!(begin == -1 && end == 256));
auto *self = static_cast<Node3 *>(n);
assume(1 <= self->numChildren);
assume(self->numChildren <= 3);
++begin;
@@ -2321,9 +2250,17 @@ bool checkMaxBetweenExclusiveImpl(Node3 *n, int begin, int end,
auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
uint32_t mask = 0;
for (int i = 0; i < self->numChildren; ++i) {
for (int i = 0; i < Node3::kMaxNodes; ++i) {
mask |= inBounds(self->index[i]) << i;
}
mask &= (1 << self->numChildren) - 1;
#ifdef __aarch64__
// The bits surviving the mask above don't derive from uninitialized slots,
// but clang 21+ on aarch64 lowers inBounds through flags+csel, which
// memcheck models imprecisely, tainting bits the mask provably clears.
// https://git.weaselab.dev/weaselab/conflict-set/issues/39
VALGRIND_MAKE_MEM_DEFINED(&mask, sizeof(mask));
#endif
if (!mask) {
return true;
}
@@ -2331,11 +2268,17 @@ bool checkMaxBetweenExclusiveImpl(Node3 *n, int begin, int end,
const bool firstRangeOk =
!child->entryPresent || child->entry.rangeVersion <= readVersion;
uint32_t compared = 0;
for (int i = 0; i < self->numChildren; ++i) {
for (int i = 0; i < Node3::kMaxNodes; ++i) {
compared |= (self->childMaxVersion[i] > readVersion) << i;
}
return !(compared & mask) && firstRangeOk;
uint32_t compared_masked = compared & mask;
#ifdef __aarch64__
// Same imprecise csel modeling as above.
// https://git.weaselab.dev/weaselab/conflict-set/issues/39
VALGRIND_MAKE_MEM_DEFINED(&compared_masked, sizeof(compared_masked));
#endif
return !compared_masked && firstRangeOk;
}
template <bool kAVX512>
@@ -2378,11 +2321,26 @@ bool checkMaxBetweenExclusiveImpl(Node16 *n, int begin, int end,
const bool firstRangeOk =
!child->entryPresent || child->entry.rangeVersion <= readVersion;
uint32x4_t w4[4];
memcpy(w4, self->childMaxVersion, sizeof(w4));
uint32_t rv;
memcpy(&rv, &readVersion, sizeof(rv));
const auto rvVec = vdupq_n_u32(rv);
int32x4_t z;
memset(&z, 0, sizeof(z));
uint16x4_t conflicting[4];
for (int i = 0; i < 4; ++i) {
conflicting[i] =
vmovn_u32(vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
}
auto combined =
vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
vmovn_u16(vcombine_u16(conflicting[2], conflicting[3])));
uint64_t compared = vget_lane_u64(
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(conflictMask16(
self->childMaxVersion, readVersion)),
4)),
0);
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(combined), 4)), 0);
return !(compared & mask) && firstRangeOk;
@@ -2417,9 +2375,10 @@ bool checkMaxBetweenExclusiveImpl(Node16 *n, int begin, int end,
auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
uint32_t mask = 0;
for (int i = 0; i < self->numChildren; ++i) {
for (int i = 0; i < 16; ++i) {
mask |= inBounds(self->index[i]) << i;
}
mask &= (1 << self->numChildren) - 1;
if (!mask) {
return true;
}
@@ -2427,7 +2386,7 @@ bool checkMaxBetweenExclusiveImpl(Node16 *n, int begin, int end,
const bool firstRangeOk =
!child->entryPresent || child->entry.rangeVersion <= readVersion;
uint32_t compared = 0;
for (int i = 0; i < self->numChildren; ++i) {
for (int i = 0; i < 16; ++i) {
compared |= (self->childMaxVersion[i] > readVersion) << i;
}
return !(compared & mask) && firstRangeOk;
@@ -2786,39 +2745,10 @@ checkMaxBetweenExclusiveImpl<true>(Node256 *n, int begin, int end,
return self;
}
#if SHOW_MEMORY
// Length of the key leading to `child`, given the length of the key leading to
// its parent. Consistent with `getSearchPathLength`.
int64_t childSearchPathLength(int64_t parentSearchPathLength, Node *child) {
return parentSearchPathLength + 1 + child->partialKeyLen;
}
#endif
void eraseTree(Node *root, WriteContext *writeContext) {
Arena arena;
auto toFree = vector<Node *>(arena);
toFree.push_back(root);
// Length of the key leading to each node in `toFree`, on a parallel stack.
// A node is released before its children are visited, so the `parent`
// pointers that `getSearchPathLength` walks are dangling by the time a queued
// node is popped. Carry the lengths along instead. Only the SHOW_MEMORY
// accounting needs them.
#if SHOW_MEMORY
auto searchPathLengths = vector<int64_t>(arena);
searchPathLengths.push_back(getSearchPathLength(root));
#endif
// Queue `child` for release, where `parentSearchPathLength` is the length of
// the key leading to its parent.
auto queueChild = [&](Node *child, int64_t parentSearchPathLength) {
toFree.push_back(child);
#if SHOW_MEMORY
searchPathLengths.push_back(
childSearchPathLength(parentSearchPathLength, child));
#else
(void)parentSearchPathLength;
#endif
};
while (toFree.size() > 0) {
auto *n = toFree.back();
@@ -2826,13 +2756,7 @@ void eraseTree(Node *root, WriteContext *writeContext) {
writeContext->accum.entries_erased += n->entryPresent;
++writeContext->accum.nodes_released;
#if SHOW_MEMORY
const int64_t searchPathLength = searchPathLengths.back();
searchPathLengths.pop_back();
#else
const int64_t searchPathLength = 0;
#endif
removeKey(n, searchPathLength);
removeKey(n);
switch (n->getType()) {
case Type_Node0: {
@@ -2842,39 +2766,28 @@ void eraseTree(Node *root, WriteContext *writeContext) {
case Type_Node3: {
auto *n3 = static_cast<Node3 *>(n);
for (int i = 0; i < n3->numChildren; ++i) {
queueChild(n3->children[i], searchPathLength);
toFree.push_back(n3->children[i]);
}
writeContext->release(n3);
} break;
case Type_Node16: {
auto *n16 = static_cast<Node16 *>(n);
for (int i = 0; i < n16->numChildren; ++i) {
queueChild(n16->children[i], searchPathLength);
toFree.push_back(n16->children[i]);
}
writeContext->release(n16);
} break;
case Type_Node48: {
auto *n48 = static_cast<Node48 *>(n);
for (int i = 0; i < n48->numChildren; ++i) {
queueChild(n48->children[i], searchPathLength);
toFree.push_back(n48->children[i]);
}
writeContext->release(n48);
} break;
case Type_Node256: {
auto *n256 = static_cast<Node256 *>(n);
auto *out = toFree.unsafePrepareAppend(n256->numChildren).data();
#if SHOW_MEMORY
auto *outSearchPathLengths =
searchPathLengths.unsafePrepareAppend(n256->numChildren).data();
#endif
n256->bitSet.forEachSet([&](int i) {
Node *child = n256->children[i];
*out++ = child;
#if SHOW_MEMORY
*outSearchPathLengths++ =
childSearchPathLength(searchPathLength, child);
#endif
});
n256->bitSet.forEachSet([&](int i) { *out++ = n256->children[i]; });
assert(out == toFree.end());
writeContext->release(n256);
} break;
@@ -2971,11 +2884,7 @@ horizontalMax16(InternalVersionT *vs, InternalVersionT) {
return max;
}
/** @pre `node` is not the root, `node->entryPresent`, and `writeContext !=
* nullptr`.
* @post `maxVersion(node)` is the max of `writeContext->zero` and the max
* version of all logical keys starting with the search path of `node`.
*/
// Precondition: `node->entryPresent`, and node is not the root
void fixupMaxVersion(Node *node, WriteContext *writeContext) {
assert(node->parent);
InternalVersionT max;
@@ -2986,13 +2895,9 @@ void fixupMaxVersion(Node *node, WriteContext *writeContext) {
break;
case Type_Node3: {
auto *self3 = static_cast<Node3 *>(node);
assume(1 <= self3->numChildren);
assume(self3->numChildren <= 3);
InternalVersionT m = self3->childMaxVersion[0];
for (int i = 1; i < self3->numChildren; ++i) {
m = std::max(m, self3->childMaxVersion[i]);
}
max = std::max(max, m);
max = std::max(max,
horizontalMaxUpTo16(self3->childMaxVersion,
writeContext->zero, self3->numChildren));
} break;
case Type_Node16: {
auto *self16 = static_cast<Node16 *>(node);
@@ -3008,7 +2913,8 @@ void fixupMaxVersion(Node *node, WriteContext *writeContext) {
} break;
case Type_Node256: {
auto *self256 = static_cast<Node256 *>(node);
max = std::max(max, horizontalMax16(self256->maxOfMax, writeContext->zero));
max = std::max(
max, horizontalMax16(self256->childMaxVersion, writeContext->zero));
} break;
default: // GCOVR_EXCL_LINE
__builtin_unreachable(); // GCOVR_EXCL_LINE
@@ -3958,17 +3864,17 @@ PRESERVE_NONE void right_side_iter(Job *job, Context *context) {
void Job::init(const ConflictSet::ReadRange *read, ConflictSet::Result *result,
Node *root, int64_t oldestVersionFullPrecision) {
auto begin = TrivialSpan(read->begin.p, read->begin.len);
auto end = TrivialSpan(read->end.p, read->end.len);
if (read->readVersion < oldestVersionFullPrecision) [[unlikely]] {
*result = ConflictSet::TooOld;
continuation = complete;
} else if (read->end.len == 0) {
} else if (end.size() == 0) {
this->begin = begin;
this->n = root;
this->readVersion = InternalVersionT(read->readVersion);
this->result = result;
continuation = check::point_read_state_machine::begin;
} else {
auto end = TrivialSpan(read->end.p, read->end.len);
this->begin = begin;
this->end = end;
this->n = root;
@@ -4955,7 +4861,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
auto *results2 = new (arena) Result[count];
check::Context context2;
useSequential(reads, results2, count, context2);
assert(memcmp(result, results2, count * sizeof(Result)) == 0);
assert(memcmp(result, results2, count) == 0);
assert(context.readContext == context2.readContext);
#endif
}
@@ -5056,18 +4962,16 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
}
if (e != nullptr) {
while (e->releaseDeferred) {
e = e->forwardTo;
e = b->forwardTo;
}
}
fprintf(stderr, "search path: %s, begin: %s\n",
getSearchPathPrintable(b).c_str(),
printable(context.results[i].remaining).c_str());
if (e != nullptr) {
fprintf(stderr, "search path: %s, end: %s\n",
getSearchPathPrintable(e).c_str(),
printable(context.results[i].endRemaining).c_str());
}
}
#endif
while (context.results[i].insertionPoint->releaseDeferred) {
@@ -5142,8 +5046,8 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
for (int i = 0; i < count; ++i) {
const auto &w = writes[i];
auto begin = TrivialSpan(w.begin.p, w.begin.len);
if (w.end.len > 0) {
auto end = TrivialSpan(w.end.p, w.end.len);
if (w.end.len > 0) {
addWriteRange(rootParent->children[0], begin, end,
InternalVersionT(writeVersion), &writeContext);
} else {
@@ -5168,7 +5072,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
if (oldestExtantVersion < writeVersion - kMaxCorrectVersionWindow)
[[unlikely]] {
if (writeVersion - kNominalVersionWindow > newestVersionFullPrecision) {
if (writeVersion > newestVersionFullPrecision + kNominalVersionWindow) {
eraseTree(rootParent->children[0], &writeContext);
init(writeVersion - kNominalVersionWindow);
}
@@ -5546,9 +5450,8 @@ void internal_addWrites(ConflictSet::Impl *impl,
impl->totalBytes += mallocBytesDelta;
impl->memory_bytes.set(impl->totalBytes);
#if SHOW_MEMORY
// Testing-only cross-check of the accounting.
if (impl->totalBytes != mallocBytes) {
abort(); // GCOVR_EXCL_LINE
abort();
}
#endif
}
@@ -5559,9 +5462,8 @@ void internal_setOldestVersion(ConflictSet::Impl *impl, int64_t oldestVersion) {
impl->totalBytes += mallocBytesDelta;
impl->memory_bytes.set(impl->totalBytes);
#if SHOW_MEMORY
// Testing-only cross-check of the accounting.
if (impl->totalBytes != mallocBytes) {
abort(); // GCOVR_EXCL_LINE
abort();
}
#endif
}
@@ -5687,12 +5589,7 @@ ConflictSet::ConflictSet(ConflictSet &&other) noexcept
: impl(std::exchange(other.impl, nullptr)) {}
ConflictSet &ConflictSet::operator=(ConflictSet &&other) noexcept {
if (this != &other) {
if (impl) {
internal_destroy(impl);
}
impl = std::exchange(other.impl, nullptr);
}
return *this;
}
@@ -5777,13 +5674,13 @@ std::string getPartialKeyPrintable(Node *n) {
}
std::string strinc(std::string_view str, bool &ok) {
int index = static_cast<int>(str.size()) - 1;
for (; index >= 0; index--)
if (static_cast<uint8_t>(str[index]) != 255)
int index;
for (index = str.size() - 1; index >= 0; index--)
if ((uint8_t &)(str[index]) != 255)
break;
// Must not be called with a string that consists only of zero or more
// '\xff' bytes, and the empty string has no successor.
// '\xff' bytes.
if (index < 0) {
ok = false;
return {};
@@ -5791,8 +5688,7 @@ std::string strinc(std::string_view str, bool &ok) {
ok = true;
auto r = std::string(str.substr(0, index + 1));
auto &last = r[r.size() - 1];
last = static_cast<char>(static_cast<uint8_t>(last) + 1);
((uint8_t &)r[r.size() - 1])++;
return r;
}
@@ -6027,14 +5923,6 @@ checkMaxVersion(Node *root, Node *node, InternalVersionT oldestVersion,
} // namespace
// The SHOW_MEMORY instrumentation. Each of these functions is defined exactly
// once, with the accounting compiled out when SHOW_MEMORY is disabled. Defining
// them once (rather than defining no-op stubs when SHOW_MEMORY is disabled)
// keeps every configuration of this file reporting each function on the same
// line, which merging coverage from objects compiled with different options
// requires: gcovr, merging the SHOW_MEMORY test with the rest of the build,
// otherwise aborts with "Got function ... on multiple lines". The optimizer
// removes the resulting empty calls.
#if SHOW_MEMORY
int64_t nodeBytes = 0;
@@ -6063,10 +5951,7 @@ int64_t getNodeSize(struct Node *n) {
}
}
#endif
int64_t getSearchPathLength(Node *n) {
#if SHOW_MEMORY
assert(n != nullptr);
int64_t result = 0;
for (;;) {
@@ -6078,14 +5963,9 @@ int64_t getSearchPathLength(Node *n) {
n = n->parent;
}
return result;
#else
(void)n;
return 0;
#endif
}
void addNode(Node *n) {
#if SHOW_MEMORY
nodeBytes += getNodeSize(n);
partialCapacityBytes += n->getCapacity();
if (nodeBytes > peakNodeBytes) {
@@ -6094,22 +5974,14 @@ void addNode(Node *n) {
if (partialCapacityBytes > peakPartialCapacityBytes) {
peakPartialCapacityBytes = partialCapacityBytes;
}
#else
(void)n;
#endif
}
void removeNode(Node *n) {
#if SHOW_MEMORY
nodeBytes -= getNodeSize(n);
partialCapacityBytes -= n->getCapacity();
#else
(void)n;
#endif
}
void addKey(Node *n) {
#if SHOW_MEMORY
if (!n->entryPresent) {
++totalKeys;
keyBytes += getSearchPathLength(n);
@@ -6120,36 +5992,15 @@ void addKey(Node *n) {
peakKeyBytes = keyBytes;
}
}
#else
(void)n;
#endif
}
void removeKey(Node *n) {
#if SHOW_MEMORY
if (n->entryPresent) {
--totalKeys;
keyBytes -= getSearchPathLength(n);
}
#else
(void)n;
#endif
}
void removeKey(Node *n, int64_t searchPathLength) {
#if SHOW_MEMORY
if (n->entryPresent) {
--totalKeys;
keyBytes -= searchPathLength;
}
#else
(void)n;
(void)searchPathLength;
#endif
}
#if SHOW_MEMORY
struct __attribute__((visibility("default"))) PeakPrinter {
~PeakPrinter() {
printf("--- radix_tree ---\n");
+46
View File
@@ -0,0 +1,46 @@
FROM ubuntu:rolling
WORKDIR /tmp
ENV HOME=/tmp
RUN chmod -R 777 /tmp
# Install apt dependencies
RUN apt-get update
RUN apt-get upgrade -y
RUN TZ=America/Los_Angeles DEBIAN_FRONTEND=noninteractive apt-get install -y \
biber \
build-essential \
ccache \
clang \
cmake \
curl \
devscripts \
gcovr \
git \
latexmk \
libc6-dbg \
llvm \
mold \
ninja-build \
nodejs \
pre-commit \
python3-requests \
rpm \
texlive-bibtex-extra \
texlive-fonts-recommended \
texlive-latex-extra \
texlive-pictures \
valgrind \
wget \
zstd
# MinIO client, for uploading build artifacts
RUN curl -Ls "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/mc" \
-o /usr/local/bin/mc && chmod +x /usr/local/bin/mc
ENV CC=clang
ENV CXX=clang++
# Try to have all the pre-commit hooks we'll need already initialized
COPY .pre-commit-config.yaml /tmp/
RUN git init && pre-commit install-hooks
+3 -20
View File
@@ -59,12 +59,6 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
while (keyUpdates > 0) {
if (iter == map.end()) {
iter = map.begin();
// The map is empty, so there are no more entries to garbage-collect.
// Break rather than spinning forever (keyUpdates is charged at 2x
// the entry count, so it may still be > 0 after emptying the map).
if (iter == map.end()) {
break;
}
}
for (; iter != map.end(); --keyUpdates) {
if (iter->second <= oldestVersion) {
@@ -102,9 +96,7 @@ void ConflictSet::setOldestVersion(int64_t oldestVersion) {
return impl->setOldestVersion(oldestVersion);
}
// The hash_table implementation does not track memory usage, so return 0 to
// satisfy the API contract that getBytes() returns a non-negative value.
int64_t ConflictSet::getBytes() const { return 0; }
int64_t ConflictSet::getBytes() const { return -1; }
void ConflictSet::getMetricsV1(MetricsV1 **metrics, int *count) const {
*metrics = nullptr;
@@ -127,13 +119,7 @@ ConflictSet::ConflictSet(ConflictSet &&other) noexcept
: impl(std::exchange(other.impl, nullptr)) {}
ConflictSet &ConflictSet::operator=(ConflictSet &&other) noexcept {
if (this != &other) {
if (impl) {
impl->~Impl();
safe_free(impl, sizeof(Impl));
}
impl = std::exchange(other.impl, nullptr);
}
return *this;
}
@@ -169,10 +155,7 @@ __attribute__((__visibility__("default"))) void ConflictSet_destroy(void *cs) {
}
__attribute__((__visibility__("default"))) int64_t
ConflictSet_getBytes(void *cs) {
(void)cs;
// The hash_table implementation does not track memory usage, so return 0 to
// satisfy the API contract that ConflictSet_getBytes returns a non-negative
// value.
return 0;
using Impl = ConflictSet::Impl;
return -1;
}
}
-7
View File
@@ -23,9 +23,7 @@ using namespace weaselab;
#include <callgrind.h>
#define DEBUG_VERBOSE 0
#ifndef SHOW_MEMORY
#define SHOW_MEMORY 0
#endif
// std::span is not trivially constructible. We want a span that leaves its
// members uninitialized for performance reasons.
@@ -98,11 +96,6 @@ static_assert(std::is_trivial_v<TrivialSpan>);
#if SHOW_MEMORY
inline int64_t mallocBytes = 0;
inline int64_t peakMallocBytes = 0;
// Defined in ConflictSet.cpp
extern int64_t nodeBytes;
extern int64_t partialCapacityBytes;
extern int64_t totalKeys;
extern int64_t keyBytes;
#endif
inline thread_local int64_t mallocBytesDelta = 0;
+2 -70
View File
@@ -96,7 +96,7 @@ inline int firstNeqStride(const uint8_t *ap, const uint8_t *bp) {
// GCOVR_EXCL_START
#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
__attribute__((target("avx512f,avx512bw"))) inline int
longestCommonPrefixLong(const uint8_t *ap, const uint8_t *bp, int cl) {
longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
int i = 0;
int end = cl & ~63;
while (i < end) {
@@ -123,7 +123,7 @@ __attribute__((target("default")))
// GCOVR_EXCL_STOP
inline int
longestCommonPrefixLong(const uint8_t *ap, const uint8_t *bp, int cl) {
longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
if (!(cl >= 0)) {
__builtin_unreachable(); // GCOVR_EXCL_LINE
}
@@ -183,71 +183,3 @@ longestCommonPrefixLong(const uint8_t *ap, const uint8_t *bp, int cl) {
return i;
}
// Use in-bounds native-word loads for short common lengths. The second load
// overlaps the first; because the first load matched, any mismatch found in
// the overlap is impossible and the returned absolute index is still correct.
// Precondition: cl <= 2 * sizeof(uint64_t)
__attribute__((always_inline)) static inline int
longestCommonPrefixShort(const uint8_t *ap, const uint8_t *bp, int cl) {
assert(cl <= 2 * int(sizeof(uint64_t)));
if (cl >= 8) {
uint64_t a0;
uint64_t b0;
memcpy(&a0, ap, sizeof(a0));
memcpy(&b0, bp, sizeof(b0));
if (uint64_t mismatched = a0 ^ b0) {
return std::countr_zero(mismatched) / 8;
}
if (cl == 8) {
return 8;
}
uint64_t a1;
uint64_t b1;
memcpy(&a1, ap + cl - 8, sizeof(a1));
memcpy(&b1, bp + cl - 8, sizeof(b1));
if (uint64_t mismatched = a1 ^ b1) {
return cl - 8 + std::countr_zero(mismatched) / 8;
}
return cl;
}
if (cl >= 4) {
uint32_t a0;
uint32_t b0;
memcpy(&a0, ap, sizeof(a0));
memcpy(&b0, bp, sizeof(b0));
if (uint32_t mismatched = a0 ^ b0) {
return std::countr_zero(mismatched) / 8;
}
if (cl == 4) {
return 4;
}
uint32_t a1;
uint32_t b1;
memcpy(&a1, ap + cl - 4, sizeof(a1));
memcpy(&b1, bp + cl - 4, sizeof(b1));
if (uint32_t mismatched = a1 ^ b1) {
return cl - 4 + std::countr_zero(mismatched) / 8;
}
return cl;
}
int i = 0;
while (i < cl && ap[i] == bp[i]) {
++i;
}
return i;
}
__attribute__((always_inline)) static inline int
longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
if (!(cl >= 0)) {
__builtin_unreachable(); // GCOVR_EXCL_LINE
}
if (cl <= 2 * int(sizeof(uint64_t))) {
return longestCommonPrefixShort(ap, bp, cl);
}
return longestCommonPrefixLong(ap, bp, cl);
}
+23 -23
View File
@@ -7,10 +7,10 @@ Hardware for all benchmarks is an AMD Ryzen 9 7900 with (2x32GB) 5600MT/s CL28-3
```
$ clang++ --version
Ubuntu clang version 21.1.8 (6ubuntu1)
Ubuntu clang version 20.0.0 (++20241120082228+86734c857724-1~exp1~20241120202359.554)
Target: x86_64-pc-linux-gnu
Thread model: posix
InstalledDir: /usr/lib/llvm-21/bin
InstalledDir: /usr/lib/llvm-20/bin
```
# Microbenchmark
@@ -19,30 +19,30 @@ InstalledDir: /usr/lib/llvm-21/bin
| ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark
|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
| 165.93 | 6,026,520.04 | 0.1% | 3,107.03 | 610.42 | 5.090 | 558.59 | 0.0% | 1.98 | `point reads`
| 163.08 | 6,132,046.57 | 0.1% | 3,036.76 | 599.68 | 5.064 | 539.35 | 0.0% | 1.95 | `prefix reads`
| 250.26 | 3,995,834.21 | 0.1% | 3,722.71 | 920.25 | 4.045 | 692.00 | 0.0% | 2.99 | `range reads`
| 359.64 | 2,780,581.63 | 0.1% | 4,523.64 | 1,322.63 | 3.420 | 720.22 | 2.1% | 4.30 | `point writes`
| 358.20 | 2,791,726.93 | 0.1% | 4,484.57 | 1,317.23 | 3.405 | 705.00 | 2.0% | 4.28 | `prefix writes`
| 193.91 | 5,157,081.29 | 0.1% | 2,224.10 | 713.33 | 3.118 | 377.17 | 3.3% | 2.32 | `range writes`
| 422.03 | 2,369,513.38 | 1.1% | 6,855.96 | 1,551.55 | 4.419 | 1,227.82 | 1.4% | 0.05 | `monotonic increasing point writes`
| 142,467.00 | 7,019.17 | 3.8% | 812,047.33 | 508,535.40 | 1.597 | 151,246.50 | 1.0% | 0.01 | `worst case for radix tree`
| 37.66 | 26,554,217.74 | 0.8% | 701.00 | 138.80 | 5.050 | 102.00 | 0.0% | 0.01 | `create and destroy`
| 161.29 | 6,200,056.17 | 0.1% | 3,014.03 | 831.04 | 3.627 | 504.59 | 0.0% | 1.93 | `point reads`
| 158.32 | 6,316,160.64 | 0.1% | 2,954.16 | 815.80 | 3.621 | 490.17 | 0.0% | 1.89 | `prefix reads`
| 237.39 | 4,212,409.50 | 0.2% | 3,592.41 | 1,233.96 | 2.911 | 629.31 | 0.0% | 2.84 | `range reads`
| 442.11 | 2,261,878.94 | 0.0% | 4,450.57 | 2,314.25 | 1.923 | 707.92 | 2.1% | 5.28 | `point writes`
| 439.89 | 2,273,308.53 | 0.1% | 4,410.22 | 2,302.29 | 1.916 | 694.74 | 2.1% | 5.25 | `prefix writes`
| 290.96 | 3,436,936.78 | 0.0% | 2,315.38 | 1,528.68 | 1.515 | 396.69 | 3.3% | 3.49 | `range writes`
| 476.93 | 2,096,762.02 | 0.6% | 6,999.33 | 2,484.94 | 2.817 | 1,251.73 | 1.3% | 0.06 | `monotonic increasing point writes`
| 131,736.57 | 7,590.91 | 1.1% | 807,444.50 | 704,941.71 | 1.145 | 144,584.60 | 0.9% | 0.01 | `worst case for radix tree`
| 45.50 | 21,978,369.95 | 1.1% | 902.00 | 232.36 | 3.882 | 132.00 | 0.0% | 0.01 | `create and destroy`
## Radix tree (this implementation)
| ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark
|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
| 12.25 | 81,664,669.29 | 0.4% | 233.49 | 44.94 | 5.196 | 31.23 | 0.7% | 0.15 | `point reads`
| 13.35 | 74,903,950.49 | 0.1% | 271.75 | 49.10 | 5.534 | 40.21 | 0.4% | 0.16 | `prefix reads`
| 31.37 | 31,874,583.62 | 0.1% | 700.39 | 115.43 | 6.067 | 100.83 | 0.2% | 0.37 | `range reads`
| 19.55 | 51,153,057.80 | 0.1% | 366.51 | 71.90 | 5.098 | 47.92 | 0.5% | 0.23 | `point writes`
| 36.92 | 27,088,344.17 | 0.1% | 645.11 | 135.72 | 4.753 | 100.31 | 0.4% | 0.44 | `prefix writes`
| 36.84 | 27,143,553.08 | 0.1% | 691.04 | 135.42 | 5.103 | 114.01 | 0.1% | 0.45 | `range writes`
| 74.62 | 13,400,467.72 | 0.9% | 1,418.37 | 274.20 | 5.173 | 273.39 | 0.1% | 0.01 | `monotonic increasing point writes`
| 352,671.00 | 2,835.50 | 15.3% | 3,687,535.00 | 1,287,119.00 | 2.865 | 712,030.00 | 0.1% | 0.01 | :wavy_dash: `worst case for radix tree` (Unstable with ~1.0 iters. Increase `minEpochIterations` to e.g. 10)
| 96.63 | 10,348,729.42 | 1.2% | 1,766.00 | 355.53 | 4.967 | 283.00 | 0.0% | 0.01 | `create and destroy`
| 12.36 | 80,885,626.43 | 0.2% | 243.56 | 63.62 | 3.828 | 31.07 | 0.6% | 0.15 | `point reads`
| 14.18 | 70,502,196.81 | 0.1% | 297.72 | 73.13 | 4.071 | 40.31 | 0.5% | 0.17 | `prefix reads`
| 33.44 | 29,901,623.04 | 0.1% | 767.90 | 172.42 | 4.454 | 101.32 | 0.2% | 0.40 | `range reads`
| 19.48 | 51,342,564.70 | 0.3% | 374.45 | 100.43 | 3.728 | 48.92 | 0.5% | 0.23 | `point writes`
| 37.46 | 26,694,471.44 | 0.1% | 672.00 | 193.14 | 3.479 | 101.28 | 0.3% | 0.45 | `prefix writes`
| 38.78 | 25,784,784.34 | 0.0% | 738.26 | 199.93 | 3.693 | 111.59 | 0.1% | 0.47 | `range writes`
| 76.05 | 13,148,995.74 | 0.7% | 1,450.77 | 397.16 | 3.653 | 275.72 | 0.0% | 0.01 | `monotonic increasing point writes`
| 286,920.33 | 3,485.29 | 0.4% | 4,117,948.00 | 1,521,352.00 | 2.707 | 714,833.00 | 0.1% | 0.01 | `worst case for radix tree`
| 95.66 | 10,453,798.72 | 0.5% | 1,986.00 | 495.04 | 4.012 | 315.00 | 0.0% | 0.01 | `create and destroy`
# "Real data" test
@@ -51,13 +51,13 @@ Point queries only. Gc ratio is the ratio of time spent doing garbage collection
## skip list
```
Check: 4.69875 seconds, 347.017 MB/s, Add: 3.39475 seconds, 165.153 MB/s, Gc ratio: 38.7414%, Peak idle memory: 5.51852e+06
Check: 4.53508 seconds, 371.81 MB/s, Add: 3.81222 seconds, 150.919 MB/s, Gc ratio: 33.66%, Peak idle memory: 5.61007e+06
```
## radix tree
```
Check: 0.976745 seconds, 1669.37 MB/s, Add: 1.20464 seconds, 465.411 MB/s, Gc ratio: 39.4158%, Peak idle memory: 2.0226e+06
Check: 0.957735 seconds, 1760.6 MB/s, Add: 1.19942 seconds, 479.678 MB/s, Gc ratio: 38.6069%, Peak idle memory: 2.05667e+06
```
## hash table
@@ -65,6 +65,6 @@ Check: 0.976745 seconds, 1669.37 MB/s, Add: 1.20464 seconds, 465.411 MB/s, Gc ra
(The hash table implementation doesn't work on range queries, and its purpose is to provide an idea of how fast point queries can be)
```
Check: 0.864589 seconds, 1885.92 MB/s, Add: 0.634604 seconds, 883.47 MB/s, Gc ratio: 41.184%, Peak idle memory: 0
Check: 0.804598 seconds, 2095.69 MB/s, Add: 0.671221 seconds, 857.147 MB/s, Gc ratio: 35.0034%, Peak idle memory: 0
```
+6 -7
View File
@@ -1,6 +1,5 @@
#include <ConflictSet.h>
#include <algorithm>
#include <cerrno>
#include <chrono>
#include <cstdio>
@@ -78,10 +77,10 @@ int main(int argc, const char **argv) {
begin = end + 1;
end = (uint8_t *)memchr(begin, '\n', size);
if (line.size() >= 2 && line[0] == 'P') {
write = line.subspan(2, line.size() - 2);
} else if (line.size() >= 2 && line[0] == 'L') {
reads.push_back(line.subspan(2, line.size() - 2));
if (line.size() > 0 && line[0] == 'P') {
write = line.subspan(2, line.size());
} else if (line.size() > 0 && line[0] == 'L') {
reads.push_back(line.subspan(2, line.size()));
} else if (line.empty()) {
{
readRanges.resize(reads.size());
@@ -91,7 +90,7 @@ int main(int argc, const char **argv) {
iter->begin.len = read.size();
checkBytes += read.size();
iter->end.len = 0;
iter->readVersion = std::max<int64_t>(0, version - 100);
iter->readVersion = version - 100;
++iter;
}
}
@@ -122,7 +121,7 @@ int main(int argc, const char **argv) {
}
timer = now();
cs.setOldestVersion(std::max<int64_t>(0, version - 10000));
cs.setOldestVersion(version - 10000);
gcTime += now() - timer;
}
}
-115
View File
@@ -1,115 +0,0 @@
// Exercises the SHOW_MEMORY instrumentation, which the other tests don't
// compile with (this target passes -DSHOW_MEMORY=1). The instrumentation has
// its own consistency checks, which abort on failure, and asserts are enabled
// by TEST_FLAGS. Only one ConflictSet may exist at a time, since the
// accounting is process-wide.
#include "ConflictSet.h"
#include "Internal.h"
#include <cassert>
#include <cstdint>
#include <string>
using namespace weaselab;
namespace {
void writePoint(ConflictSet &cs, const std::string &key, int64_t writeVersion) {
ConflictSet::WriteRange w{{(const uint8_t *)key.data(), int(key.size())},
{nullptr, 0}};
cs.addWrites(&w, 1, writeVersion);
}
ConflictSet::Result checkPoint(const ConflictSet &cs, const std::string &key,
int64_t readVersion) {
ConflictSet::ReadRange r{{(const uint8_t *)key.data(), int(key.size())},
{nullptr, 0},
readVersion};
ConflictSet::Result result;
cs.check(&r, &result, 1);
return result;
}
// Everything a ConflictSet allocated must have been accounted for and released
// by the time it's destroyed.
void checkFullyReleased() {
assert(nodeBytes == 0);
assert(partialCapacityBytes == 0);
assert(totalKeys == 0);
assert(keyBytes == 0);
assert(mallocBytes == 0);
}
} // namespace
int main() {
// Splitting a node replaces it, which defers the release of the old node.
// Releasing a deferred node must account for its capacity, and the key it
// held must be accounted for without walking `parent` pointers of nodes that
// have already been released.
{
ConflictSet cs{0};
writePoint(cs, "a", 1);
writePoint(cs, "ab", 2);
assert(checkPoint(cs, "a", 0) == ConflictSet::Conflict);
assert(checkPoint(cs, "ab", 0) == ConflictSet::Conflict);
assert(checkPoint(cs, "b", 0) == ConflictSet::Commit);
assert(nodeBytes > 0);
assert(keyBytes > 0);
}
checkFullyReleased();
// Exceeding the version window erases the tree mid-workload.
{
ConflictSet cs{0};
writePoint(cs, "abc", 1);
const int64_t version = int64_t(1) << 40;
writePoint(cs, "abc", version);
assert(checkPoint(cs, "abc", 0) == ConflictSet::TooOld);
assert(checkPoint(cs, "abc", version) == ConflictSet::Commit);
}
checkFullyReleased();
// Erase a tree with many nodes, both mid-workload and at destruction.
{
ConflictSet cs{0};
for (int i = 0; i < 1000; ++i) {
writePoint(cs, "prefix/" + std::to_string(i), i + 1);
}
const int64_t version = int64_t(1) << 40;
writePoint(cs, "prefix/1", version);
assert(checkPoint(cs, "prefix/1", version) == ConflictSet::Commit);
for (int i = 0; i < 1000; ++i) {
writePoint(cs, "other/" + std::to_string(i), version + i + 1);
}
assert(checkPoint(cs, "other/999", version) == ConflictSet::Conflict);
}
checkFullyReleased();
// Grow a node through every node type, so that the accounting for each node
// type's size is exercised, then erase the entries one at a time so that
// nodes are released individually rather than as part of a whole tree.
{
ConflictSet cs{0};
std::string key(1, '\0');
for (int i = 0; i < 256; ++i) {
key[0] = char(i);
writePoint(cs, key, i + 1);
}
for (int i = 0; i < 256; ++i) {
key[0] = char(i);
assert(checkPoint(cs, key, i) == ConflictSet::Conflict);
}
// Garbage collection erases entries that can only fail with TooOld anyway.
// Doing it one version at a time walks the tree, erasing, downsizing and
// releasing nodes as it goes.
for (int i = 0; i < 256; ++i) {
cs.setOldestVersion(i + 1);
}
key[0] = '\0';
assert(checkPoint(cs, key, 0) == ConflictSet::TooOld);
}
checkFullyReleased();
}
+4 -21
View File
@@ -38,9 +38,6 @@ std::span<const uint8_t> keyAfter(Arena &arena, std::span<const uint8_t> key) {
std::span<const uint8_t> copyToArena(Arena &arena,
std::span<const uint8_t> key) {
if (key.size() == 0) {
return std::span<const uint8_t>();
}
auto result = std::span<uint8_t>(new (arena) uint8_t[key.size()], key.size());
memcpy(result.data(), key.data(), key.size());
return result;
@@ -100,7 +97,7 @@ force_inline bool getCharacter(const KeyInfo &ki, int character,
bool operator<(const KeyInfo &lhs, const KeyInfo &rhs) {
int i = std::min(lhs.key.size(), rhs.key.size());
int c = i > 0 ? memcmp(lhs.key.data(), rhs.key.data(), i) : 0;
int c = memcmp(lhs.key.data(), rhs.key.data(), i);
if (c != 0)
return c < 0;
@@ -290,8 +287,7 @@ private:
static force_inline bool less(const uint8_t *a, int aLen, const uint8_t *b,
int bLen) {
int n = std::min(aLen, bLen);
int c = n > 0 ? memcmp(a, b, n) : 0;
int c = memcmp(a, b, std::min(aLen, bLen));
if (c < 0)
return true;
if (c > 0)
@@ -758,8 +754,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
void addWrites(const ConflictSet::WriteRange *writes, int count,
int64_t writeVersion) {
auto points = std::vector<KeyInfo>();
points.reserve(count * 2);
auto points = std::vector<KeyInfo>(count * 2);
Arena arena;
for (int r = 0; r < count; r++) {
@@ -819,17 +814,10 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
}
void setOldestVersion(int64_t oldestVersion) {
// oldestVersion becomes the max of its previous value and the argument, so
// an older version is a no-op. We still fall through in the equal case so
// that garbage collection can make progress without advancing the oldest
// version.
if (oldestVersion < this->oldestVersion) {
return;
}
// This isn't 100% accurate. It overcounts if you hit the end
gc_iterations_total.add(keyUpdates);
assert(oldestVersion >= this->oldestVersion);
this->oldestVersion = oldestVersion;
SkipList::Finger finger;
int temp;
@@ -993,12 +981,7 @@ ConflictSet::ConflictSet(ConflictSet &&other) noexcept
: impl(std::exchange(other.impl, nullptr)) {}
ConflictSet &ConflictSet::operator=(ConflictSet &&other) noexcept {
if (this != &other) {
if (impl) {
internal_destroy(impl);
}
impl = std::exchange(other.impl, nullptr);
}
return *this;
}
+31
View File
@@ -35,5 +35,36 @@ int main(int argc, char **argv) {
break;
}
}
{
ConflictSet::MetricsV1 *metrics;
int metricsCount;
driver1.cs.getMetricsV1(&metrics, &metricsCount);
printf("#################### METRICS for ConflictSet 1 for %s "
"####################\n",
argv[i]);
for (int i = 0; i < metricsCount; ++i) {
printf("# HELP %s %s\n", metrics[i].name, metrics[i].help);
printf("# TYPE %s %s\n", metrics[i].name,
metrics[i].type == metrics[i].Counter ? "counter" : "gauge");
printf("%s %g\n", metrics[i].name, metrics[i].getValue());
}
puts("");
}
{
ConflictSet::MetricsV1 *metrics;
int metricsCount;
driver2.cs.getMetricsV1(&metrics, &metricsCount);
printf("#################### METRICS for ConflictSet 2 for %s "
"####################\n",
argv[i]);
for (int i = 0; i < metricsCount; ++i) {
printf("# HELP %s %s\n", metrics[i].name, metrics[i].help);
printf("# TYPE %s %s\n", metrics[i].name,
metrics[i].type == metrics[i].Counter ? "counter" : "gauge");
printf("%s %g\n", metrics[i].name, metrics[i].getValue());
}
puts("");
}
}
}
-43
View File
@@ -1,43 +0,0 @@
#!/bin/bash
set -euxo pipefail
LLVM_VERSION="${LLVM_VERSION:-21}"
MSAN_PREFIX="${MSAN_PREFIX:-$PWD/msan}"
JOBS="${JOBS:-$(nproc)}"
cd /tmp
rm -rf libcxx-msan
mkdir libcxx-msan
cd libcxx-msan
git clone --depth=1 "https://github.com/llvm/llvm-project.git" -b "release/${LLVM_VERSION}.x"
cmake -S llvm-project/runtimes -B build_msan \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_CXX_COMPILER="clang++-${LLVM_VERSION}" \
-DCMAKE_C_COMPILER="clang-${LLVM_VERSION}" \
-DCMAKE_INSTALL_PREFIX="${MSAN_PREFIX}" \
-DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
-DLLVM_USE_SANITIZER=MemoryWithOrigins
cmake --build build_msan -j"${JOBS}"
cmake --install build_msan
cmake -S llvm-project/runtimes -B build_nomsan \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_CXX_COMPILER="clang++-${LLVM_VERSION}" \
-DCMAKE_C_COMPILER="clang-${LLVM_VERSION}" \
-DCMAKE_INSTALL_PREFIX="${MSAN_PREFIX}" \
-DLLVM_ENABLE_RUNTIMES="libunwind"
cmake --build build_nomsan -j"${JOBS}"
cmake --install build_nomsan
VERSION="$(clang-"${LLVM_VERSION}" --version | head -n1 | sed -E 's/.*clang version ([0-9.]+).*/\1/')"
TARBALL="msan-toolchain-${VERSION}.tar.zst"
tar --zstd -cf "${TARBALL}" -C "${MSAN_PREFIX}" .
TARBALL_PATH="$(pwd)/${TARBALL}"
echo "Created: ${TARBALL_PATH}"
+9 -27
View File
@@ -27,37 +27,23 @@ class Result(enum.Enum):
TOO_OLD = 2
def _make_key(buf: bytes) -> tuple[_Key, bytearray]:
"""Create a _Key and a backing bytearray that must be kept alive."""
backing = bytearray(buf)
array = (ctypes.c_ubyte * len(backing)).from_buffer(backing)
return _Key(array, len(array)), backing
def write(begin: bytes, end: Optional[bytes] = None) -> WriteRange:
begin_key, begin_buf = _make_key(begin)
b = (ctypes.c_ubyte * len(begin)).from_buffer(bytearray(begin))
if end is None:
end_key = _Key((ctypes.c_ubyte * 0)(), 0)
end_buf = None
e = (ctypes.c_ubyte * 0)()
else:
end_key, end_buf = _make_key(end)
result = WriteRange(begin_key, end_key)
result._begin_buf = begin_buf
result._end_buf = end_buf
return result
e = (ctypes.c_ubyte * len(end)).from_buffer(bytearray(end))
return WriteRange(_Key(b, len(b)), _Key(e, len(e)))
def read(version: int, begin: bytes, end: Optional[bytes] = None) -> ReadRange:
begin_key, begin_buf = _make_key(begin)
b = (ctypes.c_ubyte * len(begin)).from_buffer(bytearray(begin))
if end is None:
end_key = _Key((ctypes.c_ubyte * 0)(), 0)
end_buf = None
e = (ctypes.c_ubyte * 0)()
else:
end_key, end_buf = _make_key(end)
result = ReadRange(begin_key, end_key, version)
result._begin_buf = begin_buf
result._end_buf = end_buf
return result
e = (ctypes.c_ubyte * len(end)).from_buffer(bytearray(end))
return ReadRange(_Key(b, len(b)), _Key(e, len(e)), version)
class ConflictSet:
@@ -102,7 +88,6 @@ class ConflictSet:
ctypes.POINTER(ctypes.c_int),
ctypes.c_int,
)
self._lib.ConflictSet_check.restype = None
self._lib.ConflictSet_addWrites.argtypes = (
ctypes.c_void_p,
@@ -110,16 +95,13 @@ class ConflictSet:
ctypes.c_int,
ctypes.c_int64,
)
self._lib.ConflictSet_addWrites.restype = None
self._lib.ConflictSet_setOldestVersion.argtypes = (
ctypes.c_void_p,
ctypes.c_int64,
)
self._lib.ConflictSet_setOldestVersion.restype = None
self._lib.ConflictSet_destroy.argtypes = (ctypes.c_void_p,)
self._lib.ConflictSet_destroy.restype = None
self._lib.ConflictSet_getBytes.argtypes = (ctypes.c_void_p,)
self._lib.ConflictSet_getBytes.restype = ctypes.c_int64
-23
View File
@@ -2,7 +2,6 @@
#include <cassert>
#include <cstdio>
#include <limits>
using namespace weaselab;
@@ -24,28 +23,6 @@ int main(void) {
int64_t bytes = cs.getBytes();
assert(bytes > 0);
// Regression: a write just below INT64_MAX must not overflow the nominal
// version window comparison and erase entries that reads still observe.
// Versions are only required to be >= 0, so this is valid input.
{
const int64_t imax = std::numeric_limits<int64_t>::max();
const int64_t v0 = imax - int64_t(1000000000);
ConflictSet nearMax(0);
ConflictSet::WriteRange w2;
w2.begin.p = (const uint8_t *)"key";
w2.begin.len = 3;
w2.end.len = 0;
nearMax.addWrites(&w2, 1, v0);
nearMax.addWrites(&w2, 0, imax);
ConflictSet::ReadRange r2;
r2.begin.p = (const uint8_t *)"key";
r2.begin.len = 3;
r2.end.len = 0;
r2.readVersion = v0 - 1;
nearMax.check(&r2, &result, 1);
assert(result == ConflictSet::Conflict);
}
ConflictSet::MetricsV1 *metrics;
int metricsCount;
cs.getMetricsV1(&metrics, &metricsCount);
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Loaded 100 of 107 files, more files were not shown because too many files have changed in this diff. Show more