1 Commits
Author SHA1 Message Date
weaselbot 494e836aae Address review: handle empty keys explicitly in operator<
CI / pre-commit (pull_request) Successful in 2m9s
CI / release (arm64, ubuntu-latest-arm64) (pull_request) Successful in 3m27s
CI / test (-DCMAKE_BUILD_TYPE=Debug, debug) (pull_request) Successful in 3m38s
CI / test (-DCMAKE_CXX_FLAGS=-DUSE_64_BIT=1, 64-bit-versions) (pull_request) Successful in 3m29s
CI / test (-DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++, gcc) (pull_request) Successful in 3m42s
CI / test (-DUSE_SIMD_FALLBACK=ON, simd-fallback) (pull_request) Successful in 3m33s
CI / release (amd64, ubuntu-latest-amd64) (pull_request) Successful in 4m55s
CI / coverage (pull_request) Successful in 3m40s
Per review feedback on the i > 0 memcmp guard: handle empty keys up
front instead, so memcmp only runs when both keys are non-empty (and
therefore have valid data pointers). An empty key is a prefix of every
key, so it sorts before any non-empty key; when both are empty, fall
back to the extra ordering. This makes the empty-vs-non-empty case
explicit rather than relying on the length check after a defaulted c = 0.
2026-07-13 16:28:47 -04:00
104 changed files with 309 additions and 2629 deletions

No files matched your search

+37 -70
View File
@@ -19,8 +19,15 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: | run: |
. /etc/os-release
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
sudo apt-get update -qq sudo apt-get update -qq
sudo apt-get install -y git nodejs pre-commit sudo apt-get install -y \
clang-21 git nodejs pre-commit
for tool in clang clang++; do
sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
done
- uses: actions/cache@v4 - uses: actions/cache@v4
with: with:
@@ -38,72 +45,43 @@ jobs:
matrix: matrix:
include: include:
- name: 64-bit-versions - name: 64-bit-versions
runner: ubuntu-latest-amd64
arch: amd64
llvm_version: "21"
msan_url: ""
cmake_args: -DCMAKE_CXX_FLAGS=-DUSE_64_BIT=1 cmake_args: -DCMAKE_CXX_FLAGS=-DUSE_64_BIT=1
- name: debug - name: debug
runner: ubuntu-latest-amd64 cmake_args: -DCMAKE_BUILD_TYPE=Debug
arch: amd64
llvm_version: "21"
msan_url: https://minio.weaselab.dev/public/x86_64/msan-toolchain-21.1.8.tar.zst
cmake_args: -DCMAKE_BUILD_TYPE=Debug -DMSAN_TOOLCHAIN_PATH=/opt/msan
- name: simd-fallback - name: simd-fallback
runner: ubuntu-latest-amd64
arch: amd64
llvm_version: "21"
msan_url: ""
cmake_args: -DUSE_SIMD_FALLBACK=ON cmake_args: -DUSE_SIMD_FALLBACK=ON
- name: gcc - name: gcc
runner: ubuntu-latest-amd64
arch: amd64
llvm_version: "21"
msan_url: ""
cmake_args: -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ cmake_args: -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++
- name: debug-arm64 runs-on: ubuntu-latest-amd64
runner: ubuntu-latest-arm64
arch: arm64
llvm_version: "22"
msan_url: https://minio.weaselab.dev/public/aarch64/msan-toolchain-22.1.8.tar.zst
cmake_args: -DCMAKE_BUILD_TYPE=Debug -DMSAN_TOOLCHAIN_PATH=/opt/msan
runs-on: ${{ matrix.runner }}
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- uses: actions/cache@v4 - uses: actions/cache@v4
with: with:
path: /var/cache/apt/archives path: /var/cache/apt/archives
key: apt-${{ matrix.arch }}-${{ hashFiles('.gitea/workflows/ci.yml') }} key: apt-amd64-${{ hashFiles('.gitea/workflows/ci.yml') }}
- name: Install common dependencies - name: Install dependencies
run: |
sudo apt-get update -qq
sudo apt-get install -y \
build-essential ccache cmake gcc g++ libc6-dbg \
ninja-build python3 valgrind zstd
sudo curl -Ls "https://minio.weaselab.dev/public/$(uname -m)/mc.RELEASE.2025-08-13T08-35-41Z" \
-o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
- name: Install LLVM toolchain
if: matrix.name != 'gcc'
run: | run: |
. /etc/os-release . /etc/os-release
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-${{ matrix.llvm_version }} main" | sudo tee /etc/apt/sources.list.d/llvm.list echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
sudo apt-get update -qq sudo apt-get update -qq
sudo apt-get install -y \ sudo apt-get install -y \
clang-${{ matrix.llvm_version }} llvm-${{ matrix.llvm_version }} lld-${{ matrix.llvm_version }} mold build-essential ccache clang-21 cmake gcc g++ \
libc6-dbg llvm-21 lld-21 mold ninja-build python3 valgrind zstd
sudo curl -Ls "https://dl.min.io/client/mc/release/linux-amd64/mc" \
-o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do
sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-${{ matrix.llvm_version }} 100 sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
done done
- name: Download MSan toolchain - uses: actions/cache@v4
if: matrix.msan_url != '' with:
run: | path: .ccache
curl -Ls "${{ matrix.msan_url }}" -o /tmp/msan-toolchain.tar.zst key: ccache-${{ matrix.name }}-${{ gitea.sha }}
sudo mkdir -p /opt/msan restore-keys: |
sudo tar --zstd -xf /tmp/msan-toolchain.tar.zst -C /opt/msan ccache-${{ matrix.name }}-
- name: Build - name: Build
run: | run: |
@@ -145,10 +123,8 @@ jobs:
include: include:
- runner: ubuntu-latest-amd64 - runner: ubuntu-latest-amd64
arch: amd64 arch: amd64
cmake_args: -DMSAN_TOOLCHAIN_PATH=/opt/msan
- runner: ubuntu-latest-arm64 - runner: ubuntu-latest-arm64
arch: arm64 arch: arm64
cmake_args: ""
runs-on: ${{ matrix.runner }} runs-on: ${{ matrix.runner }}
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
@@ -165,23 +141,16 @@ jobs:
echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
sudo apt-get update -qq sudo apt-get update -qq
sudo apt-get install -y \ sudo apt-get install -y \
build-essential ccache clang-21 cmake devscripts \ biber build-essential ccache clang-21 cmake devscripts \
libc6-dbg llvm-21 lld-21 mold ninja-build rpm \ latexmk libc6-dbg llvm-21 lld-21 mold ninja-build rpm \
valgrind zstd texlive-bibtex-extra texlive-fonts-recommended \
sudo curl -Ls "https://minio.weaselab.dev/public/$(uname -m)/mc.RELEASE.2025-08-13T08-35-41Z" \ texlive-latex-extra texlive-pictures valgrind zstd
sudo curl -Ls "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/mc" \
-o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc -o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do
sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100 sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
done done
- name: Install paper build dependencies
if: matrix.arch == 'amd64'
run: |
sudo apt-get install -y \
biber latexmk \
texlive-bibtex-extra texlive-fonts-recommended \
texlive-latex-extra texlive-pictures
- uses: actions/cache@v4 - uses: actions/cache@v4
with: with:
path: .ccache path: .ccache
@@ -189,25 +158,23 @@ jobs:
restore-keys: | restore-keys: |
ccache-release-${{ matrix.arch }}- ccache-release-${{ matrix.arch }}-
- name: Download MSan toolchain
if: matrix.arch == 'amd64'
run: |
curl -Ls "https://minio.weaselab.dev/public/x86_64/msan-toolchain-21.1.8.tar.zst" -o /tmp/msan-toolchain.tar.zst
sudo mkdir -p /opt/msan
sudo tar --zstd -xf /tmp/msan-toolchain.tar.zst -C /opt/msan
- name: Build - name: Build
run: | run: |
export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache" export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache"
rm -rf build rm -rf build
cmake -S . -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_FLAGS=-DNVALGRIND ${{ matrix.cmake_args }} cmake -S . -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_FLAGS=-DNVALGRIND
ninja -C build ninja -C build
ccache -s ccache -s
- name: Test - name: Test
run: | run: |
cd build cd build
ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 -T Test -j "$(nproc)" --timeout 90 > /dev/null # On arm64, valgrind needs the MAKE_MEM_DEFINED client requests for
# https://git.weaselab.dev/weaselab/conflict-set/issues/39, but this
# build has -DNVALGRIND, which compiles them out. Skip valgrind
# tests here; they run annotated in the test job.
ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 ${{ matrix.arch == 'arm64' && '-E valgrind' || '' }} -T Test -j "$(nproc)" --timeout 90 > /dev/null
- name: Package - name: Package
run: | run: |
cd build cd build
@@ -270,7 +237,7 @@ jobs:
sudo apt-get install -y \ sudo apt-get install -y \
build-essential ccache clang-21 cmake gcovr \ build-essential ccache clang-21 cmake gcovr \
libc6-dbg llvm-21 lld-21 mold ninja-build python3 valgrind zstd libc6-dbg llvm-21 lld-21 mold ninja-build python3 valgrind zstd
sudo curl -Ls "https://minio.weaselab.dev/public/$(uname -m)/mc.RELEASE.2025-08-13T08-35-41Z" \ sudo curl -Ls "https://dl.min.io/client/mc/release/linux-amd64/mc" \
-o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc -o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do
sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100 sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
+4 -60
View File
@@ -1,7 +1,7 @@
cmake_minimum_required(VERSION 3.18) cmake_minimum_required(VERSION 3.18)
project( project(
conflict-set conflict-set
VERSION 0.1.0 VERSION 0.0.14
DESCRIPTION DESCRIPTION
"A data structure for optimistic concurrency control on ranges of bitwise-lexicographically-ordered keys." "A data structure for optimistic concurrency control on ranges of bitwise-lexicographically-ordered keys."
HOMEPAGE_URL "https://git.weaselab.dev/weaselab/conflict-set" HOMEPAGE_URL "https://git.weaselab.dev/weaselab/conflict-set"
@@ -103,13 +103,6 @@ option(USE_SIMD_FALLBACK
option(DISABLE_TSAN "Disable TSAN" OFF) option(DISABLE_TSAN "Disable TSAN" OFF)
set(MSAN_TOOLCHAIN_PATH
""
CACHE
PATH
"Path to an MSan-instrumented libc++ toolchain. When set, an MSan fuzz_driver target is added. Must contain include/c++/v1 and lib. See build_msan_toolchain.sh to produce one."
)
# This is encouraged according to # This is encouraged according to
# https://valgrind.org/docs/manual/manual-core-adv.html#manual-core-adv.clientreq # https://valgrind.org/docs/manual/manual-core-adv.html#manual-core-adv.clientreq
include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/valgrind) include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/valgrind)
@@ -182,8 +175,6 @@ endif()
include(CTest) include(CTest)
option(TEST_SKIP_LIST "Run the slow skip list cross-check tests" OFF)
# disable tests if this is being used through e.g. FetchContent # disable tests if this is being used through e.g. FetchContent
if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING) if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
@@ -226,9 +217,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
target_compile_options(driver_skip_list PRIVATE ${TEST_FLAGS}) target_compile_options(driver_skip_list PRIVATE ${TEST_FLAGS})
target_link_libraries(driver_skip_list PRIVATE skip_list) target_link_libraries(driver_skip_list PRIVATE skip_list)
# Optional cross-check of SkipList.cpp against ReferenceImpl. Opt-in via # enable to test skip list
# -DTEST_SKIP_LIST=ON, since these are slow. if(0)
if(TEST_SKIP_LIST)
foreach(TEST ${CORPUS_TESTS}) foreach(TEST ${CORPUS_TESTS})
get_filename_component(hash ${TEST} NAME) get_filename_component(hash ${TEST} NAME)
add_test(NAME skip_list_${hash} COMMAND driver_skip_list ${TEST}) add_test(NAME skip_list_${hash} COMMAND driver_skip_list ${TEST})
@@ -263,7 +253,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
endif() endif()
endif() endif()
# whitebox tests asan+ubsan # whitebox tests
add_executable(fuzz_driver ConflictSet.cpp FuzzTestDriver.cpp) add_executable(fuzz_driver ConflictSet.cpp FuzzTestDriver.cpp)
target_compile_options(fuzz_driver PRIVATE ${TEST_FLAGS}) target_compile_options(fuzz_driver PRIVATE ${TEST_FLAGS})
if(NOT CMAKE_CROSSCOMPILING) if(NOT CMAKE_CROSSCOMPILING)
@@ -278,34 +268,6 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
add_test(NAME conflict_set_fuzz_${hash} COMMAND fuzz_driver ${TEST}) add_test(NAME conflict_set_fuzz_${hash} COMMAND fuzz_driver ${TEST})
endforeach() endforeach()
# whitebox tests msan
if(MSAN_TOOLCHAIN_PATH)
add_executable(fuzz_driver_msan ConflictSet.cpp FuzzTestDriver.cpp)
target_compile_options(fuzz_driver_msan PRIVATE ${TEST_FLAGS})
if(NOT CMAKE_CROSSCOMPILING)
target_compile_options(
fuzz_driver_msan
PRIVATE -fsanitize=memory -fsanitize-memory-track-origins=2
$<$<COMPILE_LANGUAGE:CXX>:-stdlib=libc++>
-I${MSAN_TOOLCHAIN_PATH}/include/c++/v1)
target_link_options(
fuzz_driver_msan
PRIVATE
-fsanitize=memory
-fsanitize-memory-track-origins=2
-stdlib=libc++
-L${MSAN_TOOLCHAIN_PATH}/lib
LINKER:-rpath,${MSAN_TOOLCHAIN_PATH}/lib)
endif()
target_compile_definitions(fuzz_driver_msan PRIVATE ENABLE_FUZZ)
target_include_directories(fuzz_driver_msan
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
foreach(TEST ${CORPUS_TESTS})
get_filename_component(hash ${TEST} NAME)
add_test(NAME conflict_set_msan_${hash} COMMAND fuzz_driver_msan ${TEST})
endforeach()
endif()
# tsan tests # tsan tests
if(NOT CMAKE_CROSSCOMPILING AND NOT DISABLE_TSAN) if(NOT CMAKE_CROSSCOMPILING AND NOT DISABLE_TSAN)
add_executable(tsan_driver ConflictSet.cpp FuzzTestDriver.cpp) add_executable(tsan_driver ConflictSet.cpp FuzzTestDriver.cpp)
@@ -393,24 +355,6 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
PROPERTIES CXX_STANDARD_REQUIRED ON) PROPERTIES CXX_STANDARD_REQUIRED ON)
add_test(NAME conflict_set_cxx_api_test COMMAND conflict_set_cxx_api_test) add_test(NAME conflict_set_cxx_api_test COMMAND conflict_set_cxx_api_test)
# SHOW_MEMORY instrumentation, which the rest of the test suite doesn't
# compile with
add_executable(conflict_set_show_memory_test ConflictSet.cpp
ShowMemoryTest.cpp)
target_compile_options(conflict_set_show_memory_test PRIVATE ${TEST_FLAGS})
target_compile_definitions(conflict_set_show_memory_test
PRIVATE SHOW_MEMORY=1)
target_include_directories(conflict_set_show_memory_test
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
if(NOT CMAKE_CROSSCOMPILING)
target_compile_options(conflict_set_show_memory_test
PRIVATE -fsanitize=address,undefined)
target_link_options(conflict_set_show_memory_test PRIVATE
-fsanitize=address,undefined)
endif()
add_test(NAME conflict_set_show_memory_test
COMMAND conflict_set_show_memory_test)
# symbol visibility tests # symbol visibility tests
if(NOT CMAKE_BUILD_TYPE STREQUAL Debug) if(NOT CMAKE_BUILD_TYPE STREQUAL Debug)
if(APPLE) if(APPLE)
+114 -257
View File
@@ -80,19 +80,17 @@ using namespace weaselab;
#define assume assert #define assume assert
#endif #endif
// The SHOW_MEMORY instrumentation. Declared unconditionally so that each of #if SHOW_MEMORY
// these has a single definition in every configuration (see the definitions at
// the end of this file), rather than a no-op stub standing in for the real
// thing.
void addNode(struct Node *); void addNode(struct Node *);
void removeNode(struct Node *); void removeNode(struct Node *);
void addKey(struct Node *); void addKey(struct Node *);
void removeKey(struct Node *); void removeKey(struct Node *);
// Like `removeKey`, but with a caller-provided search path length. Used when #else
// the node's ancestors may already have been released, which makes walking constexpr void addNode(struct Node *) {}
// `parent` pointers invalid. constexpr void removeNode(struct Node *) {}
void removeKey(struct Node *, int64_t searchPathLength); constexpr void addKey(struct Node *) {}
int64_t getSearchPathLength(struct Node *); constexpr void removeKey(struct Node *) {}
#endif
// ==================== BEGIN IMPLEMENTATION ==================== // ==================== BEGIN IMPLEMENTATION ====================
@@ -118,40 +116,13 @@ struct InternalVersionT {
constexpr InternalVersionT() = default; constexpr InternalVersionT() = default;
constexpr explicit InternalVersionT(int64_t value) : value(value) {} constexpr explicit InternalVersionT(int64_t value) : value(value) {}
constexpr int64_t toInt64() const { return value; } // GCOVR_EXCL_LINE constexpr int64_t toInt64() const { return value; } // GCOVR_EXCL_LINE
constexpr bool operator<(const InternalVersionT &rhs) const { constexpr auto operator<=>(const InternalVersionT &rhs) const {
#if USE_64_BIT #if USE_64_BIT
return value < rhs.value; return value <=> rhs.value;
#else #else
// Maintains ordering after overflow, as long as the full-precision versions // Maintains ordering after overflow, as long as the full-precision versions
// are within `kMaxCorrectVersionWindow` of eachother. // are within `kMaxCorrectVersionWindow` of eachother.
return int32_t(value - rhs.value) < 0; return int32_t(value - rhs.value) <=> 0;
#endif
}
constexpr bool operator<=(const InternalVersionT &rhs) const {
#if USE_64_BIT
return value <= rhs.value;
#else
// Maintains ordering after overflow, as long as the full-precision versions
// are within `kMaxCorrectVersionWindow` of eachother.
return int32_t(value - rhs.value) <= 0;
#endif
}
constexpr bool operator>=(const InternalVersionT &rhs) const {
#if USE_64_BIT
return value >= rhs.value;
#else
// Maintains ordering after overflow, as long as the full-precision versions
// are within `kMaxCorrectVersionWindow` of eachother.
return int32_t(value - rhs.value) >= 0;
#endif
}
constexpr bool operator>(const InternalVersionT &rhs) const {
#if USE_64_BIT
return value > rhs.value;
#else
// Maintains ordering after overflow, as long as the full-precision versions
// are within `kMaxCorrectVersionWindow` of eachother.
return int32_t(value - rhs.value) > 0;
#endif #endif
} }
constexpr bool operator==(const InternalVersionT &) const = default; constexpr bool operator==(const InternalVersionT &) const = default;
@@ -336,9 +307,7 @@ struct Node {
uint8_t *partialKey(); uint8_t *partialKey();
Type getType() const { return type; } Type getType() const { return type; }
int32_t getCapacity() const { int32_t getCapacity() const {
// `partialKeyCapacity` is not repurposed by `deferRelease`, so this is also assert(!releaseDeferred);
// valid for nodes with `releaseDeferred` set. It's needed to account for
// the capacity of a deferred node when it's finally released.
return partialKeyCapacity; return partialKeyCapacity;
} }
@@ -519,7 +488,6 @@ inline void Node3::copyChildrenAndKeyFrom(const Node16 &other) {
inline void Node16::copyChildrenAndKeyFrom(const Node3 &other) { inline void Node16::copyChildrenAndKeyFrom(const Node3 &other) {
copyCommon(*this, other); copyCommon(*this, other);
memset(index, 0, sizeof(index));
memcpy(index, other.index, Node3::kMaxNodes); memcpy(index, other.index, Node3::kMaxNodes);
memcpy(children, other.children, memcpy(children, other.children,
Node3::kMaxNodes * sizeof(children[0])); // NOLINT Node3::kMaxNodes * sizeof(children[0])); // NOLINT
@@ -531,12 +499,6 @@ inline void Node16::copyChildrenAndKeyFrom(const Node3 &other) {
assert(children[i]->parent == &other); assert(children[i]->parent == &other);
children[i]->parent = this; children[i]->parent = this;
} }
// The index bytes beyond numChildren are initialized so that the SIMD loads
// of the full 16 bytes are well-defined (reading indeterminate bytes is UB
// in C++). valgrind then treats them as undefined so that it can still flag
// any read that relies on them.
VALGRIND_MAKE_MEM_UNDEFINED(index + numChildren,
sizeof(index) - numChildren * sizeof(index[0]));
} }
inline void Node16::copyChildrenAndKeyFrom(const Node16 &other) { inline void Node16::copyChildrenAndKeyFrom(const Node16 &other) {
@@ -553,7 +515,6 @@ inline void Node16::copyChildrenAndKeyFrom(const Node16 &other) {
inline void Node16::copyChildrenAndKeyFrom(const Node48 &other) { inline void Node16::copyChildrenAndKeyFrom(const Node48 &other) {
copyCommon(*this, other); copyCommon(*this, other);
memset(index, 0, sizeof(index));
int i = 0; int i = 0;
other.bitSet.forEachSet([&](int c) { other.bitSet.forEachSet([&](int c) {
// Suppress a false positive -Waggressive-loop-optimizations warning // Suppress a false positive -Waggressive-loop-optimizations warning
@@ -567,12 +528,6 @@ inline void Node16::copyChildrenAndKeyFrom(const Node48 &other) {
++i; ++i;
}); });
memcpy(partialKey(), &other + 1, partialKeyLen); memcpy(partialKey(), &other + 1, partialKeyLen);
// The index bytes beyond numChildren are initialized so that the SIMD loads
// of the full 16 bytes are well-defined (reading indeterminate bytes is UB
// in C++). valgrind then treats them as undefined so that it can still flag
// any read that relies on them.
VALGRIND_MAKE_MEM_UNDEFINED(index + numChildren,
sizeof(index) - numChildren * sizeof(index[0]));
} }
inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) { inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) {
@@ -580,7 +535,6 @@ inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) {
assert(numChildren == Node16::kMaxNodes); assert(numChildren == Node16::kMaxNodes);
memset(index, -1, sizeof(index)); memset(index, -1, sizeof(index));
memset(children, 0, sizeof(children)); memset(children, 0, sizeof(children));
memset(reverseIndex, 0, sizeof(reverseIndex));
const auto z = InternalVersionT::zero; const auto z = InternalVersionT::zero;
for (auto &v : childMaxVersion) { for (auto &v : childMaxVersion) {
v = z; v = z;
@@ -600,9 +554,6 @@ inline void Node48::copyChildrenAndKeyFrom(const Node16 &other) {
std::max(maxOfMax[i >> Node48::kMaxOfMaxShift], childMaxVersion[i]); std::max(maxOfMax[i >> Node48::kMaxOfMaxShift], childMaxVersion[i]);
++i; ++i;
} }
// The reverseIndex bytes beyond numChildren are initialized so that the
// SIMD loads of a full 16-byte page in scan16 are well-defined (reading
// indeterminate bytes is UB in C++).
} }
inline void Node48::copyChildrenAndKeyFrom(const Node48 &other) { inline void Node48::copyChildrenAndKeyFrom(const Node48 &other) {
@@ -629,7 +580,6 @@ inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) {
copyCommon(*this, other); copyCommon(*this, other);
memset(index, -1, sizeof(index)); memset(index, -1, sizeof(index));
memset(children, 0, sizeof(children)); memset(children, 0, sizeof(children));
memset(reverseIndex, 0, sizeof(reverseIndex));
const auto z = InternalVersionT::zero; const auto z = InternalVersionT::zero;
for (auto &v : childMaxVersion) { for (auto &v : childMaxVersion) {
v = z; v = z;
@@ -651,9 +601,6 @@ inline void Node48::copyChildrenAndKeyFrom(const Node256 &other) {
++i; ++i;
}); });
memcpy(partialKey(), &other + 1, partialKeyLen); memcpy(partialKey(), &other + 1, partialKeyLen);
// The reverseIndex bytes beyond numChildren are initialized so that the
// SIMD loads of a full 16-byte page in scan16 are well-defined (reading
// indeterminate bytes is UB in C++).
} }
inline void Node256::copyChildrenAndKeyFrom(const Node48 &other) { inline void Node256::copyChildrenAndKeyFrom(const Node48 &other) {
@@ -1385,8 +1332,8 @@ TaggedNodePointer getChildGeq(Node *self, int child) {
TaggedNodePointer getFirstChild(Node0 *) { return nullptr; } TaggedNodePointer getFirstChild(Node0 *) { return nullptr; }
TaggedNodePointer getFirstChild(Node3 *self) { TaggedNodePointer getFirstChild(Node3 *self) {
// Don't prefetch since self->children[1] might be uninitialized and msan // Improves scan performance
// doesn't like that self->children[1].prefetch();
return self->children[0]; return self->children[0];
} }
TaggedNodePointer getFirstChild(Node16 *self) { TaggedNodePointer getFirstChild(Node16 *self) {
@@ -2045,6 +1992,7 @@ Node *erase(Node *self, WriteContext *writeContext, bool logical) {
parent48->index[parentIndex] = toRemoveChildrenIndex; parent48->index[parentIndex] = toRemoveChildrenIndex;
parent48->reverseIndex[toRemoveChildrenIndex] = parentIndex; parent48->reverseIndex[toRemoveChildrenIndex] = parentIndex;
} }
parent48->childMaxVersion[lastChildrenIndex] = writeContext->zero;
VALGRIND_MAKE_MEM_UNDEFINED(parent48->children + lastChildrenIndex, VALGRIND_MAKE_MEM_UNDEFINED(parent48->children + lastChildrenIndex,
sizeof(parent48->children[0])); sizeof(parent48->children[0]));
@@ -2139,53 +2087,6 @@ compare16_avx512(const InternalVersionT *vs, InternalVersionT rv) {
} }
#endif #endif
#ifdef HAS_ARM_NEON
// Returns a uint8x16_t whose byte i is 0xff if vs[i] > readVersion and 0
// otherwise. When USE_64_BIT is set the comparison uses the full 64-bit value
// (via vcgtq_s64); otherwise it falls back to the 32-bit subtract-and-compare
// used by the original NEON fast path. The caller narrows this 16-byte vector
// into the nibble-packed (4 bits per version) mask expected by scan16 and
// checkMaxBetweenExclusiveImpl.
inline uint8x16_t conflictMask16(const InternalVersionT *vs,
InternalVersionT readVersion) {
#if USE_64_BIT
int64_t rv;
memcpy(&rv, &readVersion, sizeof(rv));
const auto rvVec = vdupq_n_s64(rv);
int32x2_t r32[8];
const auto *vsp = reinterpret_cast<const int64_t *>(vs);
for (int j = 0; j < 8; ++j) {
r32[j] = vmovn_s64(
vreinterpretq_s64_u64(vcgtq_s64(vld1q_s64(vsp + 2 * j), rvVec)));
}
uint32x4_t w4[4];
for (int k = 0; k < 4; ++k) {
w4[k] = vreinterpretq_u32_s32(vcombine_s32(r32[2 * k], r32[2 * k + 1]));
}
return vcombine_u8(
vmovn_u16(vcombine_u16(vmovn_u32(w4[0]), vmovn_u32(w4[1]))),
vmovn_u16(vcombine_u16(vmovn_u32(w4[2]), vmovn_u32(w4[3]))));
#else
uint32x4_t w4[4];
memcpy(w4, vs, sizeof(w4));
uint32_t rv;
memcpy(&rv, &readVersion, sizeof(rv));
const auto rvVec = vdupq_n_u32(rv);
int32x4_t z;
memset(&z, 0, sizeof(z));
uint16x4_t conflicting[4];
for (int i = 0; i < 4; ++i) {
conflicting[i] =
vmovn_u32(vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
}
return vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
vmovn_u16(vcombine_u16(conflicting[2], conflicting[3])));
#endif
}
#endif
// Returns true if v[i] <= readVersion for all i such that begin <= is[i] < end // Returns true if v[i] <= readVersion for all i such that begin <= is[i] < end
// Preconditions: begin <= end, end - begin < 256 // Preconditions: begin <= end, end - begin < 256
template <bool kAVX512> template <bool kAVX512>
@@ -2206,10 +2107,26 @@ bool scan16(const InternalVersionT *vs, const uint8_t *is, int begin, int end,
uint64_t mask = vget_lane_u64( uint64_t mask = vget_lane_u64(
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(results), 4)), 0); vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(results), 4)), 0);
uint32x4_t w4[4];
memcpy(w4, vs, sizeof(w4));
uint32_t rv;
memcpy(&rv, &readVersion, sizeof(rv));
const auto rvVec = vdupq_n_u32(rv);
int32x4_t z;
memset(&z, 0, sizeof(z));
uint16x4_t conflicting[4];
for (int i = 0; i < 4; ++i) {
conflicting[i] =
vmovn_u32(vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
}
auto combined =
vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
vmovn_u16(vcombine_u16(conflicting[2], conflicting[3])));
uint64_t compared = vget_lane_u64( uint64_t compared = vget_lane_u64(
vreinterpret_u64_u8(vshrn_n_u16( vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(combined), 4)), 0);
vreinterpretq_u16_u8(conflictMask16(vs, readVersion)), 4)),
0);
return !(compared & mask); return !(compared & mask);
@@ -2236,14 +2153,12 @@ bool scan16(const InternalVersionT *vs, const uint8_t *is, int begin, int end,
auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; }; auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
uint32_t compared = 0; uint32_t compared = 0;
for (int i = 0; i < 16; ++i) {
compared |= (vs[i] > readVersion) << i;
}
uint32_t mask = 0; uint32_t mask = 0;
for (int i = 0; i < 16; ++i) { for (int i = 0; i < 16; ++i) {
if (vs[i] > readVersion) { mask |= inBounds(is[i]) << i;
compared |= 1u << i;
if (inBounds(is[i])) {
mask |= 1u << i;
}
}
} }
return !(compared & mask); return !(compared & mask);
@@ -2259,10 +2174,26 @@ bool scan16(const InternalVersionT *vs, int begin, int end,
assert(begin <= end); assert(begin <= end);
#if defined(HAS_ARM_NEON) #if defined(HAS_ARM_NEON)
uint32x4_t w4[4];
memcpy(w4, vs, sizeof(w4));
uint32_t rv;
memcpy(&rv, &readVersion, sizeof(rv));
const auto rvVec = vdupq_n_u32(rv);
int32x4_t z;
memset(&z, 0, sizeof(z));
uint16x4_t conflicting[4];
for (int i = 0; i < 4; ++i) {
conflicting[i] =
vmovn_u32(vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
}
auto combined =
vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
vmovn_u16(vcombine_u16(conflicting[2], conflicting[3])));
uint64_t conflict = vget_lane_u64( uint64_t conflict = vget_lane_u64(
vreinterpret_u64_u8(vshrn_n_u16( vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(combined), 4)), 0);
vreinterpretq_u16_u8(conflictMask16(vs, readVersion)), 4)),
0);
conflict &= end == 16 ? -1 : (uint64_t(1) << (end << 2)) - 1; conflict &= end == 16 ? -1 : (uint64_t(1) << (end << 2)) - 1;
conflict >>= begin << 2; conflict >>= begin << 2;
@@ -2311,8 +2242,6 @@ bool checkMaxBetweenExclusiveImpl(Node3 *n, int begin, int end,
assert(!(begin == -1 && end == 256)); assert(!(begin == -1 && end == 256));
auto *self = static_cast<Node3 *>(n); auto *self = static_cast<Node3 *>(n);
assume(1 <= self->numChildren);
assume(self->numChildren <= 3);
++begin; ++begin;
@@ -2321,9 +2250,17 @@ bool checkMaxBetweenExclusiveImpl(Node3 *n, int begin, int end,
auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; }; auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
uint32_t mask = 0; uint32_t mask = 0;
for (int i = 0; i < self->numChildren; ++i) { for (int i = 0; i < Node3::kMaxNodes; ++i) {
mask |= inBounds(self->index[i]) << i; mask |= inBounds(self->index[i]) << i;
} }
mask &= (1 << self->numChildren) - 1;
#ifdef __aarch64__
// The bits surviving the mask above don't derive from uninitialized slots,
// but clang 21+ on aarch64 lowers inBounds through flags+csel, which
// memcheck models imprecisely, tainting bits the mask provably clears.
// https://git.weaselab.dev/weaselab/conflict-set/issues/39
VALGRIND_MAKE_MEM_DEFINED(&mask, sizeof(mask));
#endif
if (!mask) { if (!mask) {
return true; return true;
} }
@@ -2331,11 +2268,17 @@ bool checkMaxBetweenExclusiveImpl(Node3 *n, int begin, int end,
const bool firstRangeOk = const bool firstRangeOk =
!child->entryPresent || child->entry.rangeVersion <= readVersion; !child->entryPresent || child->entry.rangeVersion <= readVersion;
uint32_t compared = 0; uint32_t compared = 0;
for (int i = 0; i < self->numChildren; ++i) { for (int i = 0; i < Node3::kMaxNodes; ++i) {
compared |= (self->childMaxVersion[i] > readVersion) << i; compared |= (self->childMaxVersion[i] > readVersion) << i;
} }
return !(compared & mask) && firstRangeOk; uint32_t compared_masked = compared & mask;
#ifdef __aarch64__
// Same imprecise csel modeling as above.
// https://git.weaselab.dev/weaselab/conflict-set/issues/39
VALGRIND_MAKE_MEM_DEFINED(&compared_masked, sizeof(compared_masked));
#endif
return !compared_masked && firstRangeOk;
} }
template <bool kAVX512> template <bool kAVX512>
@@ -2378,11 +2321,26 @@ bool checkMaxBetweenExclusiveImpl(Node16 *n, int begin, int end,
const bool firstRangeOk = const bool firstRangeOk =
!child->entryPresent || child->entry.rangeVersion <= readVersion; !child->entryPresent || child->entry.rangeVersion <= readVersion;
uint32x4_t w4[4];
memcpy(w4, self->childMaxVersion, sizeof(w4));
uint32_t rv;
memcpy(&rv, &readVersion, sizeof(rv));
const auto rvVec = vdupq_n_u32(rv);
int32x4_t z;
memset(&z, 0, sizeof(z));
uint16x4_t conflicting[4];
for (int i = 0; i < 4; ++i) {
conflicting[i] =
vmovn_u32(vcgtq_s32(vreinterpretq_s32_u32(vsubq_u32(w4[i], rvVec)), z));
}
auto combined =
vcombine_u8(vmovn_u16(vcombine_u16(conflicting[0], conflicting[1])),
vmovn_u16(vcombine_u16(conflicting[2], conflicting[3])));
uint64_t compared = vget_lane_u64( uint64_t compared = vget_lane_u64(
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(conflictMask16( vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(combined), 4)), 0);
self->childMaxVersion, readVersion)),
4)),
0);
return !(compared & mask) && firstRangeOk; return !(compared & mask) && firstRangeOk;
@@ -2417,9 +2375,10 @@ bool checkMaxBetweenExclusiveImpl(Node16 *n, int begin, int end,
auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; }; auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; };
uint32_t mask = 0; uint32_t mask = 0;
for (int i = 0; i < self->numChildren; ++i) { for (int i = 0; i < 16; ++i) {
mask |= inBounds(self->index[i]) << i; mask |= inBounds(self->index[i]) << i;
} }
mask &= (1 << self->numChildren) - 1;
if (!mask) { if (!mask) {
return true; return true;
} }
@@ -2427,7 +2386,7 @@ bool checkMaxBetweenExclusiveImpl(Node16 *n, int begin, int end,
const bool firstRangeOk = const bool firstRangeOk =
!child->entryPresent || child->entry.rangeVersion <= readVersion; !child->entryPresent || child->entry.rangeVersion <= readVersion;
uint32_t compared = 0; uint32_t compared = 0;
for (int i = 0; i < self->numChildren; ++i) { for (int i = 0; i < 16; ++i) {
compared |= (self->childMaxVersion[i] > readVersion) << i; compared |= (self->childMaxVersion[i] > readVersion) << i;
} }
return !(compared & mask) && firstRangeOk; return !(compared & mask) && firstRangeOk;
@@ -2786,39 +2745,10 @@ checkMaxBetweenExclusiveImpl<true>(Node256 *n, int begin, int end,
return self; return self;
} }
#if SHOW_MEMORY
// Length of the key leading to `child`, given the length of the key leading to
// its parent. Consistent with `getSearchPathLength`.
int64_t childSearchPathLength(int64_t parentSearchPathLength, Node *child) {
return parentSearchPathLength + 1 + child->partialKeyLen;
}
#endif
void eraseTree(Node *root, WriteContext *writeContext) { void eraseTree(Node *root, WriteContext *writeContext) {
Arena arena; Arena arena;
auto toFree = vector<Node *>(arena); auto toFree = vector<Node *>(arena);
toFree.push_back(root); toFree.push_back(root);
// Length of the key leading to each node in `toFree`, on a parallel stack.
// A node is released before its children are visited, so the `parent`
// pointers that `getSearchPathLength` walks are dangling by the time a queued
// node is popped. Carry the lengths along instead. Only the SHOW_MEMORY
// accounting needs them.
#if SHOW_MEMORY
auto searchPathLengths = vector<int64_t>(arena);
searchPathLengths.push_back(getSearchPathLength(root));
#endif
// Queue `child` for release, where `parentSearchPathLength` is the length of
// the key leading to its parent.
auto queueChild = [&](Node *child, int64_t parentSearchPathLength) {
toFree.push_back(child);
#if SHOW_MEMORY
searchPathLengths.push_back(
childSearchPathLength(parentSearchPathLength, child));
#else
(void)parentSearchPathLength;
#endif
};
while (toFree.size() > 0) { while (toFree.size() > 0) {
auto *n = toFree.back(); auto *n = toFree.back();
@@ -2826,13 +2756,7 @@ void eraseTree(Node *root, WriteContext *writeContext) {
writeContext->accum.entries_erased += n->entryPresent; writeContext->accum.entries_erased += n->entryPresent;
++writeContext->accum.nodes_released; ++writeContext->accum.nodes_released;
#if SHOW_MEMORY removeKey(n);
const int64_t searchPathLength = searchPathLengths.back();
searchPathLengths.pop_back();
#else
const int64_t searchPathLength = 0;
#endif
removeKey(n, searchPathLength);
switch (n->getType()) { switch (n->getType()) {
case Type_Node0: { case Type_Node0: {
@@ -2842,39 +2766,28 @@ void eraseTree(Node *root, WriteContext *writeContext) {
case Type_Node3: { case Type_Node3: {
auto *n3 = static_cast<Node3 *>(n); auto *n3 = static_cast<Node3 *>(n);
for (int i = 0; i < n3->numChildren; ++i) { for (int i = 0; i < n3->numChildren; ++i) {
queueChild(n3->children[i], searchPathLength); toFree.push_back(n3->children[i]);
} }
writeContext->release(n3); writeContext->release(n3);
} break; } break;
case Type_Node16: { case Type_Node16: {
auto *n16 = static_cast<Node16 *>(n); auto *n16 = static_cast<Node16 *>(n);
for (int i = 0; i < n16->numChildren; ++i) { for (int i = 0; i < n16->numChildren; ++i) {
queueChild(n16->children[i], searchPathLength); toFree.push_back(n16->children[i]);
} }
writeContext->release(n16); writeContext->release(n16);
} break; } break;
case Type_Node48: { case Type_Node48: {
auto *n48 = static_cast<Node48 *>(n); auto *n48 = static_cast<Node48 *>(n);
for (int i = 0; i < n48->numChildren; ++i) { for (int i = 0; i < n48->numChildren; ++i) {
queueChild(n48->children[i], searchPathLength); toFree.push_back(n48->children[i]);
} }
writeContext->release(n48); writeContext->release(n48);
} break; } break;
case Type_Node256: { case Type_Node256: {
auto *n256 = static_cast<Node256 *>(n); auto *n256 = static_cast<Node256 *>(n);
auto *out = toFree.unsafePrepareAppend(n256->numChildren).data(); auto *out = toFree.unsafePrepareAppend(n256->numChildren).data();
#if SHOW_MEMORY n256->bitSet.forEachSet([&](int i) { *out++ = n256->children[i]; });
auto *outSearchPathLengths =
searchPathLengths.unsafePrepareAppend(n256->numChildren).data();
#endif
n256->bitSet.forEachSet([&](int i) {
Node *child = n256->children[i];
*out++ = child;
#if SHOW_MEMORY
*outSearchPathLengths++ =
childSearchPathLength(searchPathLength, child);
#endif
});
assert(out == toFree.end()); assert(out == toFree.end());
writeContext->release(n256); writeContext->release(n256);
} break; } break;
@@ -2971,11 +2884,7 @@ horizontalMax16(InternalVersionT *vs, InternalVersionT) {
return max; return max;
} }
/** @pre `node` is not the root, `node->entryPresent`, and `writeContext != // Precondition: `node->entryPresent`, and node is not the root
* nullptr`.
* @post `maxVersion(node)` is the max of `writeContext->zero` and the max
* version of all logical keys starting with the search path of `node`.
*/
void fixupMaxVersion(Node *node, WriteContext *writeContext) { void fixupMaxVersion(Node *node, WriteContext *writeContext) {
assert(node->parent); assert(node->parent);
InternalVersionT max; InternalVersionT max;
@@ -2986,13 +2895,9 @@ void fixupMaxVersion(Node *node, WriteContext *writeContext) {
break; break;
case Type_Node3: { case Type_Node3: {
auto *self3 = static_cast<Node3 *>(node); auto *self3 = static_cast<Node3 *>(node);
assume(1 <= self3->numChildren); max = std::max(max,
assume(self3->numChildren <= 3); horizontalMaxUpTo16(self3->childMaxVersion,
InternalVersionT m = self3->childMaxVersion[0]; writeContext->zero, self3->numChildren));
for (int i = 1; i < self3->numChildren; ++i) {
m = std::max(m, self3->childMaxVersion[i]);
}
max = std::max(max, m);
} break; } break;
case Type_Node16: { case Type_Node16: {
auto *self16 = static_cast<Node16 *>(node); auto *self16 = static_cast<Node16 *>(node);
@@ -3008,7 +2913,8 @@ void fixupMaxVersion(Node *node, WriteContext *writeContext) {
} break; } break;
case Type_Node256: { case Type_Node256: {
auto *self256 = static_cast<Node256 *>(node); auto *self256 = static_cast<Node256 *>(node);
max = std::max(max, horizontalMax16(self256->maxOfMax, writeContext->zero)); max = std::max(
max, horizontalMax16(self256->childMaxVersion, writeContext->zero));
} break; } break;
default: // GCOVR_EXCL_LINE default: // GCOVR_EXCL_LINE
__builtin_unreachable(); // GCOVR_EXCL_LINE __builtin_unreachable(); // GCOVR_EXCL_LINE
@@ -3958,17 +3864,17 @@ PRESERVE_NONE void right_side_iter(Job *job, Context *context) {
void Job::init(const ConflictSet::ReadRange *read, ConflictSet::Result *result, void Job::init(const ConflictSet::ReadRange *read, ConflictSet::Result *result,
Node *root, int64_t oldestVersionFullPrecision) { Node *root, int64_t oldestVersionFullPrecision) {
auto begin = TrivialSpan(read->begin.p, read->begin.len); auto begin = TrivialSpan(read->begin.p, read->begin.len);
auto end = TrivialSpan(read->end.p, read->end.len);
if (read->readVersion < oldestVersionFullPrecision) [[unlikely]] { if (read->readVersion < oldestVersionFullPrecision) [[unlikely]] {
*result = ConflictSet::TooOld; *result = ConflictSet::TooOld;
continuation = complete; continuation = complete;
} else if (read->end.len == 0) { } else if (end.size() == 0) {
this->begin = begin; this->begin = begin;
this->n = root; this->n = root;
this->readVersion = InternalVersionT(read->readVersion); this->readVersion = InternalVersionT(read->readVersion);
this->result = result; this->result = result;
continuation = check::point_read_state_machine::begin; continuation = check::point_read_state_machine::begin;
} else { } else {
auto end = TrivialSpan(read->end.p, read->end.len);
this->begin = begin; this->begin = begin;
this->end = end; this->end = end;
this->n = root; this->n = root;
@@ -4955,7 +4861,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
auto *results2 = new (arena) Result[count]; auto *results2 = new (arena) Result[count];
check::Context context2; check::Context context2;
useSequential(reads, results2, count, context2); useSequential(reads, results2, count, context2);
assert(memcmp(result, results2, count * sizeof(Result)) == 0); assert(memcmp(result, results2, count) == 0);
assert(context.readContext == context2.readContext); assert(context.readContext == context2.readContext);
#endif #endif
} }
@@ -5056,18 +4962,16 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
} }
if (e != nullptr) { if (e != nullptr) {
while (e->releaseDeferred) { while (e->releaseDeferred) {
e = e->forwardTo; e = b->forwardTo;
} }
} }
fprintf(stderr, "search path: %s, begin: %s\n", fprintf(stderr, "search path: %s, begin: %s\n",
getSearchPathPrintable(b).c_str(), getSearchPathPrintable(b).c_str(),
printable(context.results[i].remaining).c_str()); printable(context.results[i].remaining).c_str());
if (e != nullptr) {
fprintf(stderr, "search path: %s, end: %s\n", fprintf(stderr, "search path: %s, end: %s\n",
getSearchPathPrintable(e).c_str(), getSearchPathPrintable(e).c_str(),
printable(context.results[i].endRemaining).c_str()); printable(context.results[i].endRemaining).c_str());
} }
}
#endif #endif
while (context.results[i].insertionPoint->releaseDeferred) { while (context.results[i].insertionPoint->releaseDeferred) {
@@ -5142,8 +5046,8 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
for (int i = 0; i < count; ++i) { for (int i = 0; i < count; ++i) {
const auto &w = writes[i]; const auto &w = writes[i];
auto begin = TrivialSpan(w.begin.p, w.begin.len); auto begin = TrivialSpan(w.begin.p, w.begin.len);
if (w.end.len > 0) {
auto end = TrivialSpan(w.end.p, w.end.len); auto end = TrivialSpan(w.end.p, w.end.len);
if (w.end.len > 0) {
addWriteRange(rootParent->children[0], begin, end, addWriteRange(rootParent->children[0], begin, end,
InternalVersionT(writeVersion), &writeContext); InternalVersionT(writeVersion), &writeContext);
} else { } else {
@@ -5168,7 +5072,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
if (oldestExtantVersion < writeVersion - kMaxCorrectVersionWindow) if (oldestExtantVersion < writeVersion - kMaxCorrectVersionWindow)
[[unlikely]] { [[unlikely]] {
if (writeVersion - kNominalVersionWindow > newestVersionFullPrecision) { if (writeVersion > newestVersionFullPrecision + kNominalVersionWindow) {
eraseTree(rootParent->children[0], &writeContext); eraseTree(rootParent->children[0], &writeContext);
init(writeVersion - kNominalVersionWindow); init(writeVersion - kNominalVersionWindow);
} }
@@ -5546,9 +5450,8 @@ void internal_addWrites(ConflictSet::Impl *impl,
impl->totalBytes += mallocBytesDelta; impl->totalBytes += mallocBytesDelta;
impl->memory_bytes.set(impl->totalBytes); impl->memory_bytes.set(impl->totalBytes);
#if SHOW_MEMORY #if SHOW_MEMORY
// Testing-only cross-check of the accounting.
if (impl->totalBytes != mallocBytes) { if (impl->totalBytes != mallocBytes) {
abort(); // GCOVR_EXCL_LINE abort();
} }
#endif #endif
} }
@@ -5559,9 +5462,8 @@ void internal_setOldestVersion(ConflictSet::Impl *impl, int64_t oldestVersion) {
impl->totalBytes += mallocBytesDelta; impl->totalBytes += mallocBytesDelta;
impl->memory_bytes.set(impl->totalBytes); impl->memory_bytes.set(impl->totalBytes);
#if SHOW_MEMORY #if SHOW_MEMORY
// Testing-only cross-check of the accounting.
if (impl->totalBytes != mallocBytes) { if (impl->totalBytes != mallocBytes) {
abort(); // GCOVR_EXCL_LINE abort();
} }
#endif #endif
} }
@@ -6027,14 +5929,6 @@ checkMaxVersion(Node *root, Node *node, InternalVersionT oldestVersion,
} // namespace } // namespace
// The SHOW_MEMORY instrumentation. Each of these functions is defined exactly
// once, with the accounting compiled out when SHOW_MEMORY is disabled. Defining
// them once (rather than defining no-op stubs when SHOW_MEMORY is disabled)
// keeps every configuration of this file reporting each function on the same
// line, which merging coverage from objects compiled with different options
// requires: gcovr, merging the SHOW_MEMORY test with the rest of the build,
// otherwise aborts with "Got function ... on multiple lines". The optimizer
// removes the resulting empty calls.
#if SHOW_MEMORY #if SHOW_MEMORY
int64_t nodeBytes = 0; int64_t nodeBytes = 0;
@@ -6063,10 +5957,7 @@ int64_t getNodeSize(struct Node *n) {
} }
} }
#endif
int64_t getSearchPathLength(Node *n) { int64_t getSearchPathLength(Node *n) {
#if SHOW_MEMORY
assert(n != nullptr); assert(n != nullptr);
int64_t result = 0; int64_t result = 0;
for (;;) { for (;;) {
@@ -6078,14 +5969,9 @@ int64_t getSearchPathLength(Node *n) {
n = n->parent; n = n->parent;
} }
return result; return result;
#else
(void)n;
return 0;
#endif
} }
void addNode(Node *n) { void addNode(Node *n) {
#if SHOW_MEMORY
nodeBytes += getNodeSize(n); nodeBytes += getNodeSize(n);
partialCapacityBytes += n->getCapacity(); partialCapacityBytes += n->getCapacity();
if (nodeBytes > peakNodeBytes) { if (nodeBytes > peakNodeBytes) {
@@ -6094,22 +5980,14 @@ void addNode(Node *n) {
if (partialCapacityBytes > peakPartialCapacityBytes) { if (partialCapacityBytes > peakPartialCapacityBytes) {
peakPartialCapacityBytes = partialCapacityBytes; peakPartialCapacityBytes = partialCapacityBytes;
} }
#else
(void)n;
#endif
} }
void removeNode(Node *n) { void removeNode(Node *n) {
#if SHOW_MEMORY
nodeBytes -= getNodeSize(n); nodeBytes -= getNodeSize(n);
partialCapacityBytes -= n->getCapacity(); partialCapacityBytes -= n->getCapacity();
#else
(void)n;
#endif
} }
void addKey(Node *n) { void addKey(Node *n) {
#if SHOW_MEMORY
if (!n->entryPresent) { if (!n->entryPresent) {
++totalKeys; ++totalKeys;
keyBytes += getSearchPathLength(n); keyBytes += getSearchPathLength(n);
@@ -6120,36 +5998,15 @@ void addKey(Node *n) {
peakKeyBytes = keyBytes; peakKeyBytes = keyBytes;
} }
} }
#else
(void)n;
#endif
} }
void removeKey(Node *n) { void removeKey(Node *n) {
#if SHOW_MEMORY
if (n->entryPresent) { if (n->entryPresent) {
--totalKeys; --totalKeys;
keyBytes -= getSearchPathLength(n); keyBytes -= getSearchPathLength(n);
} }
#else
(void)n;
#endif
} }
void removeKey(Node *n, int64_t searchPathLength) {
#if SHOW_MEMORY
if (n->entryPresent) {
--totalKeys;
keyBytes -= searchPathLength;
}
#else
(void)n;
(void)searchPathLength;
#endif
}
#if SHOW_MEMORY
struct __attribute__((visibility("default"))) PeakPrinter { struct __attribute__((visibility("default"))) PeakPrinter {
~PeakPrinter() { ~PeakPrinter() {
printf("--- radix_tree ---\n"); printf("--- radix_tree ---\n");
+46
View File
@@ -0,0 +1,46 @@
FROM ubuntu:rolling
WORKDIR /tmp
ENV HOME=/tmp
RUN chmod -R 777 /tmp
# Install apt dependencies
RUN apt-get update
RUN apt-get upgrade -y
RUN TZ=America/Los_Angeles DEBIAN_FRONTEND=noninteractive apt-get install -y \
biber \
build-essential \
ccache \
clang \
cmake \
curl \
devscripts \
gcovr \
git \
latexmk \
libc6-dbg \
llvm \
mold \
ninja-build \
nodejs \
pre-commit \
python3-requests \
rpm \
texlive-bibtex-extra \
texlive-fonts-recommended \
texlive-latex-extra \
texlive-pictures \
valgrind \
wget \
zstd
# MinIO client, for uploading build artifacts
RUN curl -Ls "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/mc" \
-o /usr/local/bin/mc && chmod +x /usr/local/bin/mc
ENV CC=clang
ENV CXX=clang++
# Try to have all the pre-commit hooks we'll need already initialized
COPY .pre-commit-config.yaml /tmp/
RUN git init && pre-commit install-hooks
-6
View File
@@ -59,12 +59,6 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
while (keyUpdates > 0) { while (keyUpdates > 0) {
if (iter == map.end()) { if (iter == map.end()) {
iter = map.begin(); iter = map.begin();
// The map is empty, so there are no more entries to garbage-collect.
// Break rather than spinning forever (keyUpdates is charged at 2x
// the entry count, so it may still be > 0 after emptying the map).
if (iter == map.end()) {
break;
}
} }
for (; iter != map.end(); --keyUpdates) { for (; iter != map.end(); --keyUpdates) {
if (iter->second <= oldestVersion) { if (iter->second <= oldestVersion) {
-7
View File
@@ -23,9 +23,7 @@ using namespace weaselab;
#include <callgrind.h> #include <callgrind.h>
#define DEBUG_VERBOSE 0 #define DEBUG_VERBOSE 0
#ifndef SHOW_MEMORY
#define SHOW_MEMORY 0 #define SHOW_MEMORY 0
#endif
// std::span is not trivially constructible. We want a span that leaves its // std::span is not trivially constructible. We want a span that leaves its
// members uninitialized for performance reasons. // members uninitialized for performance reasons.
@@ -98,11 +96,6 @@ static_assert(std::is_trivial_v<TrivialSpan>);
#if SHOW_MEMORY #if SHOW_MEMORY
inline int64_t mallocBytes = 0; inline int64_t mallocBytes = 0;
inline int64_t peakMallocBytes = 0; inline int64_t peakMallocBytes = 0;
// Defined in ConflictSet.cpp
extern int64_t nodeBytes;
extern int64_t partialCapacityBytes;
extern int64_t totalKeys;
extern int64_t keyBytes;
#endif #endif
inline thread_local int64_t mallocBytesDelta = 0; inline thread_local int64_t mallocBytesDelta = 0;
+2 -70
View File
@@ -96,7 +96,7 @@ inline int firstNeqStride(const uint8_t *ap, const uint8_t *bp) {
// GCOVR_EXCL_START // GCOVR_EXCL_START
#if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__) #if defined(HAS_AVX) && !defined(__SANITIZE_THREAD__)
__attribute__((target("avx512f,avx512bw"))) inline int __attribute__((target("avx512f,avx512bw"))) inline int
longestCommonPrefixLong(const uint8_t *ap, const uint8_t *bp, int cl) { longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
int i = 0; int i = 0;
int end = cl & ~63; int end = cl & ~63;
while (i < end) { while (i < end) {
@@ -123,7 +123,7 @@ __attribute__((target("default")))
// GCOVR_EXCL_STOP // GCOVR_EXCL_STOP
inline int inline int
longestCommonPrefixLong(const uint8_t *ap, const uint8_t *bp, int cl) { longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
if (!(cl >= 0)) { if (!(cl >= 0)) {
__builtin_unreachable(); // GCOVR_EXCL_LINE __builtin_unreachable(); // GCOVR_EXCL_LINE
} }
@@ -183,71 +183,3 @@ longestCommonPrefixLong(const uint8_t *ap, const uint8_t *bp, int cl) {
return i; return i;
} }
// Use in-bounds native-word loads for short common lengths. The second load
// overlaps the first; because the first load matched, any mismatch found in
// the overlap is impossible and the returned absolute index is still correct.
// Precondition: cl <= 2 * sizeof(uint64_t)
__attribute__((always_inline)) static inline int
longestCommonPrefixShort(const uint8_t *ap, const uint8_t *bp, int cl) {
assert(cl <= 2 * int(sizeof(uint64_t)));
if (cl >= 8) {
uint64_t a0;
uint64_t b0;
memcpy(&a0, ap, sizeof(a0));
memcpy(&b0, bp, sizeof(b0));
if (uint64_t mismatched = a0 ^ b0) {
return std::countr_zero(mismatched) / 8;
}
if (cl == 8) {
return 8;
}
uint64_t a1;
uint64_t b1;
memcpy(&a1, ap + cl - 8, sizeof(a1));
memcpy(&b1, bp + cl - 8, sizeof(b1));
if (uint64_t mismatched = a1 ^ b1) {
return cl - 8 + std::countr_zero(mismatched) / 8;
}
return cl;
}
if (cl >= 4) {
uint32_t a0;
uint32_t b0;
memcpy(&a0, ap, sizeof(a0));
memcpy(&b0, bp, sizeof(b0));
if (uint32_t mismatched = a0 ^ b0) {
return std::countr_zero(mismatched) / 8;
}
if (cl == 4) {
return 4;
}
uint32_t a1;
uint32_t b1;
memcpy(&a1, ap + cl - 4, sizeof(a1));
memcpy(&b1, bp + cl - 4, sizeof(b1));
if (uint32_t mismatched = a1 ^ b1) {
return cl - 4 + std::countr_zero(mismatched) / 8;
}
return cl;
}
int i = 0;
while (i < cl && ap[i] == bp[i]) {
++i;
}
return i;
}
__attribute__((always_inline)) static inline int
longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
if (!(cl >= 0)) {
__builtin_unreachable(); // GCOVR_EXCL_LINE
}
if (cl <= 2 * int(sizeof(uint64_t))) {
return longestCommonPrefixShort(ap, bp, cl);
}
return longestCommonPrefixLong(ap, bp, cl);
}
+23 -23
View File
@@ -7,10 +7,10 @@ Hardware for all benchmarks is an AMD Ryzen 9 7900 with (2x32GB) 5600MT/s CL28-3
``` ```
$ clang++ --version $ clang++ --version
Ubuntu clang version 21.1.8 (6ubuntu1) Ubuntu clang version 20.0.0 (++20241120082228+86734c857724-1~exp1~20241120202359.554)
Target: x86_64-pc-linux-gnu Target: x86_64-pc-linux-gnu
Thread model: posix Thread model: posix
InstalledDir: /usr/lib/llvm-21/bin InstalledDir: /usr/lib/llvm-20/bin
``` ```
# Microbenchmark # Microbenchmark
@@ -19,30 +19,30 @@ InstalledDir: /usr/lib/llvm-21/bin
| ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark | ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark
|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:---------- |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
| 165.93 | 6,026,520.04 | 0.1% | 3,107.03 | 610.42 | 5.090 | 558.59 | 0.0% | 1.98 | `point reads` | 161.29 | 6,200,056.17 | 0.1% | 3,014.03 | 831.04 | 3.627 | 504.59 | 0.0% | 1.93 | `point reads`
| 163.08 | 6,132,046.57 | 0.1% | 3,036.76 | 599.68 | 5.064 | 539.35 | 0.0% | 1.95 | `prefix reads` | 158.32 | 6,316,160.64 | 0.1% | 2,954.16 | 815.80 | 3.621 | 490.17 | 0.0% | 1.89 | `prefix reads`
| 250.26 | 3,995,834.21 | 0.1% | 3,722.71 | 920.25 | 4.045 | 692.00 | 0.0% | 2.99 | `range reads` | 237.39 | 4,212,409.50 | 0.2% | 3,592.41 | 1,233.96 | 2.911 | 629.31 | 0.0% | 2.84 | `range reads`
| 359.64 | 2,780,581.63 | 0.1% | 4,523.64 | 1,322.63 | 3.420 | 720.22 | 2.1% | 4.30 | `point writes` | 442.11 | 2,261,878.94 | 0.0% | 4,450.57 | 2,314.25 | 1.923 | 707.92 | 2.1% | 5.28 | `point writes`
| 358.20 | 2,791,726.93 | 0.1% | 4,484.57 | 1,317.23 | 3.405 | 705.00 | 2.0% | 4.28 | `prefix writes` | 439.89 | 2,273,308.53 | 0.1% | 4,410.22 | 2,302.29 | 1.916 | 694.74 | 2.1% | 5.25 | `prefix writes`
| 193.91 | 5,157,081.29 | 0.1% | 2,224.10 | 713.33 | 3.118 | 377.17 | 3.3% | 2.32 | `range writes` | 290.96 | 3,436,936.78 | 0.0% | 2,315.38 | 1,528.68 | 1.515 | 396.69 | 3.3% | 3.49 | `range writes`
| 422.03 | 2,369,513.38 | 1.1% | 6,855.96 | 1,551.55 | 4.419 | 1,227.82 | 1.4% | 0.05 | `monotonic increasing point writes` | 476.93 | 2,096,762.02 | 0.6% | 6,999.33 | 2,484.94 | 2.817 | 1,251.73 | 1.3% | 0.06 | `monotonic increasing point writes`
| 142,467.00 | 7,019.17 | 3.8% | 812,047.33 | 508,535.40 | 1.597 | 151,246.50 | 1.0% | 0.01 | `worst case for radix tree` | 131,736.57 | 7,590.91 | 1.1% | 807,444.50 | 704,941.71 | 1.145 | 144,584.60 | 0.9% | 0.01 | `worst case for radix tree`
| 37.66 | 26,554,217.74 | 0.8% | 701.00 | 138.80 | 5.050 | 102.00 | 0.0% | 0.01 | `create and destroy` | 45.50 | 21,978,369.95 | 1.1% | 902.00 | 232.36 | 3.882 | 132.00 | 0.0% | 0.01 | `create and destroy`
## Radix tree (this implementation) ## Radix tree (this implementation)
| ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark | ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark
|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:---------- |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
| 12.25 | 81,664,669.29 | 0.4% | 233.49 | 44.94 | 5.196 | 31.23 | 0.7% | 0.15 | `point reads` | 12.36 | 80,885,626.43 | 0.2% | 243.56 | 63.62 | 3.828 | 31.07 | 0.6% | 0.15 | `point reads`
| 13.35 | 74,903,950.49 | 0.1% | 271.75 | 49.10 | 5.534 | 40.21 | 0.4% | 0.16 | `prefix reads` | 14.18 | 70,502,196.81 | 0.1% | 297.72 | 73.13 | 4.071 | 40.31 | 0.5% | 0.17 | `prefix reads`
| 31.37 | 31,874,583.62 | 0.1% | 700.39 | 115.43 | 6.067 | 100.83 | 0.2% | 0.37 | `range reads` | 33.44 | 29,901,623.04 | 0.1% | 767.90 | 172.42 | 4.454 | 101.32 | 0.2% | 0.40 | `range reads`
| 19.55 | 51,153,057.80 | 0.1% | 366.51 | 71.90 | 5.098 | 47.92 | 0.5% | 0.23 | `point writes` | 19.48 | 51,342,564.70 | 0.3% | 374.45 | 100.43 | 3.728 | 48.92 | 0.5% | 0.23 | `point writes`
| 36.92 | 27,088,344.17 | 0.1% | 645.11 | 135.72 | 4.753 | 100.31 | 0.4% | 0.44 | `prefix writes` | 37.46 | 26,694,471.44 | 0.1% | 672.00 | 193.14 | 3.479 | 101.28 | 0.3% | 0.45 | `prefix writes`
| 36.84 | 27,143,553.08 | 0.1% | 691.04 | 135.42 | 5.103 | 114.01 | 0.1% | 0.45 | `range writes` | 38.78 | 25,784,784.34 | 0.0% | 738.26 | 199.93 | 3.693 | 111.59 | 0.1% | 0.47 | `range writes`
| 74.62 | 13,400,467.72 | 0.9% | 1,418.37 | 274.20 | 5.173 | 273.39 | 0.1% | 0.01 | `monotonic increasing point writes` | 76.05 | 13,148,995.74 | 0.7% | 1,450.77 | 397.16 | 3.653 | 275.72 | 0.0% | 0.01 | `monotonic increasing point writes`
| 352,671.00 | 2,835.50 | 15.3% | 3,687,535.00 | 1,287,119.00 | 2.865 | 712,030.00 | 0.1% | 0.01 | :wavy_dash: `worst case for radix tree` (Unstable with ~1.0 iters. Increase `minEpochIterations` to e.g. 10) | 286,920.33 | 3,485.29 | 0.4% | 4,117,948.00 | 1,521,352.00 | 2.707 | 714,833.00 | 0.1% | 0.01 | `worst case for radix tree`
| 96.63 | 10,348,729.42 | 1.2% | 1,766.00 | 355.53 | 4.967 | 283.00 | 0.0% | 0.01 | `create and destroy` | 95.66 | 10,453,798.72 | 0.5% | 1,986.00 | 495.04 | 4.012 | 315.00 | 0.0% | 0.01 | `create and destroy`
# "Real data" test # "Real data" test
@@ -51,13 +51,13 @@ Point queries only. Gc ratio is the ratio of time spent doing garbage collection
## skip list ## skip list
``` ```
Check: 4.69875 seconds, 347.017 MB/s, Add: 3.39475 seconds, 165.153 MB/s, Gc ratio: 38.7414%, Peak idle memory: 5.51852e+06 Check: 4.53508 seconds, 371.81 MB/s, Add: 3.81222 seconds, 150.919 MB/s, Gc ratio: 33.66%, Peak idle memory: 5.61007e+06
``` ```
## radix tree ## radix tree
``` ```
Check: 0.976745 seconds, 1669.37 MB/s, Add: 1.20464 seconds, 465.411 MB/s, Gc ratio: 39.4158%, Peak idle memory: 2.0226e+06 Check: 0.957735 seconds, 1760.6 MB/s, Add: 1.19942 seconds, 479.678 MB/s, Gc ratio: 38.6069%, Peak idle memory: 2.05667e+06
``` ```
## hash table ## hash table
@@ -65,6 +65,6 @@ Check: 0.976745 seconds, 1669.37 MB/s, Add: 1.20464 seconds, 465.411 MB/s, Gc ra
(The hash table implementation doesn't work on range queries, and its purpose is to provide an idea of how fast point queries can be) (The hash table implementation doesn't work on range queries, and its purpose is to provide an idea of how fast point queries can be)
``` ```
Check: 0.864589 seconds, 1885.92 MB/s, Add: 0.634604 seconds, 883.47 MB/s, Gc ratio: 41.184%, Peak idle memory: 0 Check: 0.804598 seconds, 2095.69 MB/s, Add: 0.671221 seconds, 857.147 MB/s, Gc ratio: 35.0034%, Peak idle memory: 0
``` ```
-115
View File
@@ -1,115 +0,0 @@
// Exercises the SHOW_MEMORY instrumentation, which the other tests don't
// compile with (this target passes -DSHOW_MEMORY=1). The instrumentation has
// its own consistency checks, which abort on failure, and asserts are enabled
// by TEST_FLAGS. Only one ConflictSet may exist at a time, since the
// accounting is process-wide.
#include "ConflictSet.h"
#include "Internal.h"
#include <cassert>
#include <cstdint>
#include <string>
using namespace weaselab;
namespace {
void writePoint(ConflictSet &cs, const std::string &key, int64_t writeVersion) {
ConflictSet::WriteRange w{{(const uint8_t *)key.data(), int(key.size())},
{nullptr, 0}};
cs.addWrites(&w, 1, writeVersion);
}
ConflictSet::Result checkPoint(const ConflictSet &cs, const std::string &key,
int64_t readVersion) {
ConflictSet::ReadRange r{{(const uint8_t *)key.data(), int(key.size())},
{nullptr, 0},
readVersion};
ConflictSet::Result result;
cs.check(&r, &result, 1);
return result;
}
// Everything a ConflictSet allocated must have been accounted for and released
// by the time it's destroyed.
void checkFullyReleased() {
assert(nodeBytes == 0);
assert(partialCapacityBytes == 0);
assert(totalKeys == 0);
assert(keyBytes == 0);
assert(mallocBytes == 0);
}
} // namespace
int main() {
// Splitting a node replaces it, which defers the release of the old node.
// Releasing a deferred node must account for its capacity, and the key it
// held must be accounted for without walking `parent` pointers of nodes that
// have already been released.
{
ConflictSet cs{0};
writePoint(cs, "a", 1);
writePoint(cs, "ab", 2);
assert(checkPoint(cs, "a", 0) == ConflictSet::Conflict);
assert(checkPoint(cs, "ab", 0) == ConflictSet::Conflict);
assert(checkPoint(cs, "b", 0) == ConflictSet::Commit);
assert(nodeBytes > 0);
assert(keyBytes > 0);
}
checkFullyReleased();
// Exceeding the version window erases the tree mid-workload.
{
ConflictSet cs{0};
writePoint(cs, "abc", 1);
const int64_t version = int64_t(1) << 40;
writePoint(cs, "abc", version);
assert(checkPoint(cs, "abc", 0) == ConflictSet::TooOld);
assert(checkPoint(cs, "abc", version) == ConflictSet::Commit);
}
checkFullyReleased();
// Erase a tree with many nodes, both mid-workload and at destruction.
{
ConflictSet cs{0};
for (int i = 0; i < 1000; ++i) {
writePoint(cs, "prefix/" + std::to_string(i), i + 1);
}
const int64_t version = int64_t(1) << 40;
writePoint(cs, "prefix/1", version);
assert(checkPoint(cs, "prefix/1", version) == ConflictSet::Commit);
for (int i = 0; i < 1000; ++i) {
writePoint(cs, "other/" + std::to_string(i), version + i + 1);
}
assert(checkPoint(cs, "other/999", version) == ConflictSet::Conflict);
}
checkFullyReleased();
// Grow a node through every node type, so that the accounting for each node
// type's size is exercised, then erase the entries one at a time so that
// nodes are released individually rather than as part of a whole tree.
{
ConflictSet cs{0};
std::string key(1, '\0');
for (int i = 0; i < 256; ++i) {
key[0] = char(i);
writePoint(cs, key, i + 1);
}
for (int i = 0; i < 256; ++i) {
key[0] = char(i);
assert(checkPoint(cs, key, i) == ConflictSet::Conflict);
}
// Garbage collection erases entries that can only fail with TooOld anyway.
// Doing it one version at a time walks the tree, erasing, downsizing and
// releasing nodes as it goes.
for (int i = 0; i < 256; ++i) {
cs.setOldestVersion(i + 1);
}
key[0] = '\0';
assert(checkPoint(cs, key, 0) == ConflictSet::TooOld);
}
checkFullyReleased();
}
+10 -9
View File
@@ -99,8 +99,16 @@ force_inline bool getCharacter(const KeyInfo &ki, int character,
} }
bool operator<(const KeyInfo &lhs, const KeyInfo &rhs) { bool operator<(const KeyInfo &lhs, const KeyInfo &rhs) {
// An empty key is a prefix of every key, so it sorts before any non-empty
// key. Handle empty keys up front: an empty std::span may have
// data() == nullptr, which must never be passed to memcmp.
if (lhs.key.size() == 0 || rhs.key.size() == 0) {
if (lhs.key.size() != rhs.key.size())
return lhs.key.size() < rhs.key.size();
return extra_ordering(lhs) < extra_ordering(rhs);
}
int i = std::min(lhs.key.size(), rhs.key.size()); int i = std::min(lhs.key.size(), rhs.key.size());
int c = i > 0 ? memcmp(lhs.key.data(), rhs.key.data(), i) : 0; int c = memcmp(lhs.key.data(), rhs.key.data(), i);
if (c != 0) if (c != 0)
return c < 0; return c < 0;
@@ -819,17 +827,10 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
} }
void setOldestVersion(int64_t oldestVersion) { void setOldestVersion(int64_t oldestVersion) {
// oldestVersion becomes the max of its previous value and the argument, so
// an older version is a no-op. We still fall through in the equal case so
// that garbage collection can make progress without advancing the oldest
// version.
if (oldestVersion < this->oldestVersion) {
return;
}
// This isn't 100% accurate. It overcounts if you hit the end // This isn't 100% accurate. It overcounts if you hit the end
gc_iterations_total.add(keyUpdates); gc_iterations_total.add(keyUpdates);
assert(oldestVersion >= this->oldestVersion);
this->oldestVersion = oldestVersion; this->oldestVersion = oldestVersion;
SkipList::Finger finger; SkipList::Finger finger;
int temp; int temp;
+31
View File
@@ -35,5 +35,36 @@ int main(int argc, char **argv) {
break; break;
} }
} }
{
ConflictSet::MetricsV1 *metrics;
int metricsCount;
driver1.cs.getMetricsV1(&metrics, &metricsCount);
printf("#################### METRICS for ConflictSet 1 for %s "
"####################\n",
argv[i]);
for (int i = 0; i < metricsCount; ++i) {
printf("# HELP %s %s\n", metrics[i].name, metrics[i].help);
printf("# TYPE %s %s\n", metrics[i].name,
metrics[i].type == metrics[i].Counter ? "counter" : "gauge");
printf("%s %g\n", metrics[i].name, metrics[i].getValue());
}
puts("");
}
{
ConflictSet::MetricsV1 *metrics;
int metricsCount;
driver2.cs.getMetricsV1(&metrics, &metricsCount);
printf("#################### METRICS for ConflictSet 2 for %s "
"####################\n",
argv[i]);
for (int i = 0; i < metricsCount; ++i) {
printf("# HELP %s %s\n", metrics[i].name, metrics[i].help);
printf("# TYPE %s %s\n", metrics[i].name,
metrics[i].type == metrics[i].Counter ? "counter" : "gauge");
printf("%s %g\n", metrics[i].name, metrics[i].getValue());
}
puts("");
}
} }
} }
-43
View File
@@ -1,43 +0,0 @@
#!/bin/bash
set -euxo pipefail
LLVM_VERSION="${LLVM_VERSION:-21}"
MSAN_PREFIX="${MSAN_PREFIX:-$PWD/msan}"
JOBS="${JOBS:-$(nproc)}"
cd /tmp
rm -rf libcxx-msan
mkdir libcxx-msan
cd libcxx-msan
git clone --depth=1 "https://github.com/llvm/llvm-project.git" -b "release/${LLVM_VERSION}.x"
cmake -S llvm-project/runtimes -B build_msan \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_CXX_COMPILER="clang++-${LLVM_VERSION}" \
-DCMAKE_C_COMPILER="clang-${LLVM_VERSION}" \
-DCMAKE_INSTALL_PREFIX="${MSAN_PREFIX}" \
-DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
-DLLVM_USE_SANITIZER=MemoryWithOrigins
cmake --build build_msan -j"${JOBS}"
cmake --install build_msan
cmake -S llvm-project/runtimes -B build_nomsan \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_CXX_COMPILER="clang++-${LLVM_VERSION}" \
-DCMAKE_C_COMPILER="clang-${LLVM_VERSION}" \
-DCMAKE_INSTALL_PREFIX="${MSAN_PREFIX}" \
-DLLVM_ENABLE_RUNTIMES="libunwind"
cmake --build build_nomsan -j"${JOBS}"
cmake --install build_nomsan
VERSION="$(clang-"${LLVM_VERSION}" --version | head -n1 | sed -E 's/.*clang version ([0-9.]+).*/\1/')"
TARBALL="msan-toolchain-${VERSION}.tar.zst"
tar --zstd -cf "${TARBALL}" -C "${MSAN_PREFIX}" .
TARBALL_PATH="$(pwd)/${TARBALL}"
echo "Created: ${TARBALL_PATH}"
-23
View File
@@ -2,7 +2,6 @@
#include <cassert> #include <cassert>
#include <cstdio> #include <cstdio>
#include <limits>
using namespace weaselab; using namespace weaselab;
@@ -24,28 +23,6 @@ int main(void) {
int64_t bytes = cs.getBytes(); int64_t bytes = cs.getBytes();
assert(bytes > 0); assert(bytes > 0);
// Regression: a write just below INT64_MAX must not overflow the nominal
// version window comparison and erase entries that reads still observe.
// Versions are only required to be >= 0, so this is valid input.
{
const int64_t imax = std::numeric_limits<int64_t>::max();
const int64_t v0 = imax - int64_t(1000000000);
ConflictSet nearMax(0);
ConflictSet::WriteRange w2;
w2.begin.p = (const uint8_t *)"key";
w2.begin.len = 3;
w2.end.len = 0;
nearMax.addWrites(&w2, 1, v0);
nearMax.addWrites(&w2, 0, imax);
ConflictSet::ReadRange r2;
r2.begin.p = (const uint8_t *)"key";
r2.begin.len = 3;
r2.end.len = 0;
r2.readVersion = v0 - 1;
nearMax.check(&r2, &result, 1);
assert(result == ConflictSet::Conflict);
}
ConflictSet::MetricsV1 *metrics; ConflictSet::MetricsV1 *metrics;
int metricsCount; int metricsCount;
cs.getMetricsV1(&metrics, &metricsCount); cs.getMetricsV1(&metrics, &metricsCount);
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Loaded 100 of 104 files, more files were not shown because too many files have changed in this diff. Show more