Compare commits

..
Author SHA1 Message Date
weaselbot 0021c6e4e7 Fix release and coverage CI failures
* ConflictSet.cpp: enable the interleaved read/write path whenever
  musttail is available, falling back to the default calling convention
  when preserve_none is not supported. This prevents a large block of
  compiled-but-dead code from being counted in coverage.

* CMakeLists.txt: detect which arch-specific hardening-check options the
  installed hardening-check binary supports, including the newer
  hyphenated spellings, so the release test no longer fails with an
  unknown option.

* .gitea/workflows/ci.yml: build the coverage job with
  -DUSE_SIMD_FALLBACK=ON so AVX512-only functions that cannot execute on
  the CI runners are not counted against line coverage.
2026-06-22 14:09:04 -04:00
100 changed files with 137 additions and 341 deletions
+28 -110
View File
@@ -19,8 +19,15 @@ jobs:
- name: Install dependencies
run: |
. /etc/os-release
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
sudo apt-get update -qq
sudo apt-get install -y git nodejs pre-commit
sudo apt-get install -y \
clang-21 git nodejs pre-commit
for tool in clang clang++; do
sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
done
- uses: actions/cache@v4
with:
@@ -54,28 +61,28 @@ jobs:
path: /var/cache/apt/archives
key: apt-amd64-${{ hashFiles('.gitea/workflows/ci.yml') }}
- name: Install common dependencies
run: |
sudo apt-get update -qq
sudo apt-get install -y \
build-essential ccache cmake gcc g++ libc6-dbg \
ninja-build python3 valgrind zstd
sudo curl -Ls "https://minio.weaselab.dev/public/$(uname -m)/mc.RELEASE.2025-08-13T08-35-41Z" \
-o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
- name: Install LLVM toolchain
if: matrix.name != 'gcc'
- name: Install dependencies
run: |
. /etc/os-release
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
sudo apt-get update -qq
sudo apt-get install -y \
clang-21 llvm-21 lld-21 mold
build-essential ccache clang-21 cmake gcc g++ \
libc6-dbg llvm-21 lld-21 mold ninja-build python3 valgrind zstd
sudo curl -Ls "https://dl.min.io/client/mc/release/linux-amd64/mc" \
-o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do
sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
done
- uses: actions/cache@v4
with:
path: .ccache
key: ccache-${{ matrix.name }}-${{ gitea.sha }}
restore-keys: |
ccache-${{ matrix.name }}-
- name: Build
run: |
export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache"
@@ -134,23 +141,16 @@ jobs:
echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
sudo apt-get update -qq
sudo apt-get install -y \
build-essential ccache clang-21 cmake devscripts \
libc6-dbg llvm-21 lld-21 mold ninja-build rpm \
valgrind zstd
sudo curl -Ls "https://minio.weaselab.dev/public/$(uname -m)/mc.RELEASE.2025-08-13T08-35-41Z" \
biber build-essential ccache clang-21 cmake devscripts \
latexmk libc6-dbg llvm-21 lld-21 mold ninja-build rpm \
texlive-bibtex-extra texlive-fonts-recommended \
texlive-latex-extra texlive-pictures valgrind zstd
sudo curl -Ls "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/mc" \
-o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do
sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
done
- name: Install paper build dependencies
if: matrix.arch == 'amd64'
run: |
sudo apt-get install -y \
biber latexmk \
texlive-bibtex-extra texlive-fonts-recommended \
texlive-latex-extra texlive-pictures
- uses: actions/cache@v4
with:
path: .ccache
@@ -218,89 +218,6 @@ jobs:
--link "https://minio.weaselab.dev/jenkins/conflict-set/${{ gitea.run_number }}/release-${{ matrix.arch }}/Test.xml.zst" \
| tee -a "$GITHUB_STEP_SUMMARY"
msan:
runs-on: ubuntu-latest-amd64
env:
MSAN_VERSION: "21.1.8"
MSAN_URL: "https://minio.weaselab.dev/public/x86_64/msan-toolchain-21.1.8.tar.zst"
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
with:
path: /var/cache/apt/archives
key: apt-amd64-${{ hashFiles('.gitea/workflows/ci.yml') }}
- name: Install dependencies
run: |
. /etc/os-release
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
echo "deb http://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-21 main" | sudo tee /etc/apt/sources.list.d/llvm.list
sudo apt-get update -qq
sudo apt-get install -y \
build-essential ccache clang-21 cmake libc6-dbg \
llvm-21 lld-21 mold ninja-build python3 zstd
sudo curl -Ls "https://minio.weaselab.dev/public/$(uname -m)/mc.RELEASE.2025-08-13T08-35-41Z" \
-o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do
sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
done
- name: Download MSan toolchain
run: |
curl -Ls "${MSAN_URL}" -o /tmp/msan-toolchain.tar.zst
sudo mkdir -p /opt/msan
sudo tar --zstd -xf /tmp/msan-toolchain.tar.zst -C /opt/msan
- uses: actions/cache@v4
with:
path: .ccache
key: ccache-msan-${{ gitea.sha }}
restore-keys: |
ccache-msan-
- name: Build
run: |
export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache"
rm -rf build
cmake -S . -B build -G Ninja \
-DCMAKE_C_COMPILER=clang \
-DCMAKE_CXX_COMPILER=clang++ \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_BUILD_TYPE=Debug \
-DDISABLE_TSAN=ON \
-DUSE_MSAN=ON \
-DCMAKE_CXX_FLAGS="-fsanitize=memory -stdlib=libc++ -I/opt/msan/include/c++/v1 -L/opt/msan/lib -UNDEBUG" \
-DCMAKE_EXE_LINKER_FLAGS="-stdlib=libc++ -Wl,-rpath,/opt/msan/lib" \
-DCMAKE_SHARED_LINKER_FLAGS="-stdlib=libc++ -Wl,-rpath,/opt/msan/lib"
ninja -C build
ccache -s
- name: Test
run: |
cd build
ctest --no-compress-output --test-output-size-passed 100000 --test-output-size-failed 100000 -T Test -j "$(nproc)" --timeout 300 > /dev/null
- name: Upload test results to MinIO
if: always()
env:
MINIO_ACCESS_KEY: ${{ secrets.MINIO_ACCESS_KEY }}
MC_HOST_minio: https://${{ secrets.MINIO_ACCESS_KEY }}:${{ secrets.MINIO_SECRET_KEY }}@minio.weaselab.dev
run: |
if [ -z "$MINIO_ACCESS_KEY" ]; then
echo "MinIO credentials not configured; skipping upload"
exit 0
fi
zstd build/Testing/*/Test.xml
mc cp build/Testing/*/Test.xml.zst "minio/jenkins/conflict-set/${{ gitea.run_number }}/msan/"
- name: Test summary
if: always()
run: |
python3 ctest_summary.py build/Testing/*/Test.xml \
--link "https://minio.weaselab.dev/jenkins/conflict-set/${{ gitea.run_number }}/msan/Test.xml.zst" \
| tee -a "$GITHUB_STEP_SUMMARY"
coverage:
runs-on: ubuntu-latest-amd64
steps:
@@ -320,7 +237,7 @@ jobs:
sudo apt-get install -y \
build-essential ccache clang-21 cmake gcovr \
libc6-dbg llvm-21 lld-21 mold ninja-build python3 valgrind zstd
sudo curl -Ls "https://minio.weaselab.dev/public/$(uname -m)/mc.RELEASE.2025-08-13T08-35-41Z" \
sudo curl -Ls "https://dl.min.io/client/mc/release/linux-amd64/mc" \
-o /usr/local/bin/mc && sudo chmod +x /usr/local/bin/mc
for tool in clang clang++ llvm-ar llvm-nm llvm-ranlib llvm-objcopy llvm-cov llvm-symbolizer lld ld.lld; do
sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100
@@ -339,7 +256,8 @@ jobs:
rm -rf build
cmake -S . -B build -G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage \
-DCMAKE_BUILD_TYPE=Debug -DDISABLE_TSAN=ON
-DCMAKE_BUILD_TYPE=Debug -DDISABLE_TSAN=ON \
-DUSE_SIMD_FALLBACK=ON
ninja -C build
ccache -s
+4 -5
View File
@@ -37,16 +37,15 @@ ConflictSet::ReadRange singleton(Arena &arena, TrivialSpan key) {
}
ConflictSet::ReadRange prefixRange(Arena &arena, TrivialSpan key) {
int index = key.size() - 1;
for (; index >= 0; index--)
if (key[index] != 255)
int index;
for (index = key.size() - 1; index >= 0; index--)
if ((key[index]) != 255)
break;
// Must not be called with a string that consists only of zero or more '\xff'
// bytes, or with an empty string (which has no finite upper bound).
// bytes.
if (index < 0) {
assert(false);
std::abort();
}
uint8_t *buf = new (arena) uint8_t[index + 1];
+6 -34
View File
@@ -1,7 +1,7 @@
cmake_minimum_required(VERSION 3.18)
project(
conflict-set
VERSION 0.1.0
VERSION 0.0.14
DESCRIPTION
"A data structure for optimistic concurrency control on ranges of bitwise-lexicographically-ordered keys."
HOMEPAGE_URL "https://git.weaselab.dev/weaselab/conflict-set"
@@ -49,18 +49,6 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
${LLVM_OBJCOPY}
CACHE FILEPATH "path to objcopy binary" FORCE)
endif()
if(USE_MSAN)
find_program(LLD_LINKER lld)
if(LLD_LINKER)
set(CMAKE_LINKER_TYPE
"LLD"
CACHE STRING "Use LLD linker" FORCE)
set(CMAKE_LINKER
${LLD_LINKER}
CACHE FILEPATH "path to linker binary" FORCE)
add_link_options("-fuse-ld=lld")
endif()
endif()
endif()
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
@@ -115,8 +103,6 @@ option(USE_SIMD_FALLBACK
option(DISABLE_TSAN "Disable TSAN" OFF)
option(USE_MSAN "Build with MemorySanitizer (disables ASan/UBSan)" OFF)
# This is encouraged according to
# https://valgrind.org/docs/manual/manual-core-adv.html#manual-core-adv.clientreq
include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/third_party/valgrind)
@@ -246,10 +232,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
target_compile_definitions(conflict_set_main PRIVATE ENABLE_MAIN)
target_link_libraries(conflict_set_main PRIVATE nanobench)
if(NOT APPLE AND NOT USE_MSAN)
# libfuzzer target, to generate/manage corpus. MSan requires an instrumented
# libfuzzer runtime, which is not shipped with the compiler, so skip this
# target when building with MSan.
if(NOT APPLE)
# libfuzzer target, to generate/manage corpus
set(FUZZ_FLAGS "-fsanitize=fuzzer-no-link,address,undefined")
include(CheckCXXCompilerFlag)
cmake_push_check_state()
@@ -273,13 +257,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
add_executable(fuzz_driver ConflictSet.cpp FuzzTestDriver.cpp)
target_compile_options(fuzz_driver PRIVATE ${TEST_FLAGS})
if(NOT CMAKE_CROSSCOMPILING)
if(USE_MSAN)
target_compile_options(fuzz_driver PRIVATE -fsanitize=memory)
target_link_options(fuzz_driver PRIVATE -fsanitize=memory)
else()
target_compile_options(fuzz_driver PRIVATE -fsanitize=address,undefined)
target_link_options(fuzz_driver PRIVATE -fsanitize=address,undefined)
endif()
target_compile_options(fuzz_driver PRIVATE -fsanitize=address,undefined)
target_link_options(fuzz_driver PRIVATE -fsanitize=address,undefined)
endif()
target_compile_definitions(fuzz_driver PRIVATE ENABLE_FUZZ)
target_include_directories(fuzz_driver
@@ -290,9 +269,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
endforeach()
# tsan tests
if(NOT CMAKE_CROSSCOMPILING
AND NOT DISABLE_TSAN
AND NOT USE_MSAN)
if(NOT CMAKE_CROSSCOMPILING AND NOT DISABLE_TSAN)
add_executable(tsan_driver ConflictSet.cpp FuzzTestDriver.cpp)
target_compile_options(tsan_driver PRIVATE ${TEST_FLAGS} -fsanitize=thread)
target_link_options(tsan_driver PRIVATE -fsanitize=thread)
@@ -514,11 +491,6 @@ target_include_directories(
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}>)
if(USE_MSAN)
target_compile_options(${PROJECT_NAME} PUBLIC -fsanitize=memory)
target_link_options(${PROJECT_NAME} PUBLIC -fsanitize=memory)
endif()
set_target_properties(
${PROJECT_NAME} PROPERTIES VERSION ${PROJECT_VERSION}
SOVERSION ${PROJECT_VERSION_MAJOR})
+26 -29
View File
@@ -3080,7 +3080,7 @@ Node *firstGeqPhysical(Node *n, const TrivialSpan key) {
#define PRESERVE_NONE
#endif
#if __has_attribute(musttail) && __has_attribute(preserve_none)
#if __has_attribute(musttail)
constexpr bool kEnableInterleaved = true;
#else
constexpr bool kEnableInterleaved = false;
@@ -3864,10 +3864,11 @@ PRESERVE_NONE void right_side_iter(Job *job, Context *context) {
void Job::init(const ConflictSet::ReadRange *read, ConflictSet::Result *result,
Node *root, int64_t oldestVersionFullPrecision) {
auto begin = TrivialSpan(read->begin.p, read->begin.len);
auto end = TrivialSpan(read->end.p, read->end.len);
if (read->readVersion < oldestVersionFullPrecision) [[unlikely]] {
*result = ConflictSet::TooOld;
continuation = complete;
} else if (read->end.len == 0) {
} else if (end.size() == 0) {
this->begin = begin;
this->n = root;
this->readVersion = InternalVersionT(read->readVersion);
@@ -3875,7 +3876,7 @@ void Job::init(const ConflictSet::ReadRange *read, ConflictSet::Result *result,
continuation = check::point_read_state_machine::begin;
} else {
this->begin = begin;
this->end = TrivialSpan(read->end.p, read->end.len);
this->end = end;
this->n = root;
this->readVersion = InternalVersionT(read->readVersion);
this->result = result;
@@ -5039,20 +5040,22 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
assert(allPointWrites || sorted);
#endif
if (kEnableInterleaved && count > 1) {
interleavedWrites(writes, count, InternalVersionT(writeVersion));
} else {
for (int i = 0; i < count; ++i) {
const auto &w = writes[i];
auto begin = TrivialSpan(w.begin.p, w.begin.len);
if (w.end.len > 0) {
auto end = TrivialSpan(w.end.p, w.end.len);
addWriteRange(rootParent->children[0], begin, end,
InternalVersionT(writeVersion), &writeContext);
} else {
addPointWrite(rootParent->children[0], begin,
InternalVersionT(writeVersion), &writeContext);
}
if constexpr (kEnableInterleaved) {
if (count > 1) {
interleavedWrites(writes, count, InternalVersionT(writeVersion));
return;
}
}
for (int i = 0; i < count; ++i) {
const auto &w = writes[i];
auto begin = TrivialSpan(w.begin.p, w.begin.len);
auto end = TrivialSpan(w.end.p, w.end.len);
if (w.end.len > 0) {
addWriteRange(rootParent->children[0], begin, end,
InternalVersionT(writeVersion), &writeContext);
} else {
addPointWrite(rootParent->children[0], begin,
InternalVersionT(writeVersion), &writeContext);
}
}
}
@@ -5588,12 +5591,7 @@ ConflictSet::ConflictSet(ConflictSet &&other) noexcept
: impl(std::exchange(other.impl, nullptr)) {}
ConflictSet &ConflictSet::operator=(ConflictSet &&other) noexcept {
if (this != &other) {
if (impl) {
internal_destroy(impl);
}
impl = std::exchange(other.impl, nullptr);
}
impl = std::exchange(other.impl, nullptr);
return *this;
}
@@ -5678,13 +5676,13 @@ std::string getPartialKeyPrintable(Node *n) {
}
std::string strinc(std::string_view str, bool &ok) {
int index = static_cast<int>(str.size()) - 1;
for (; index >= 0; index--)
if (static_cast<uint8_t>(str[index]) != 255)
int index;
for (index = str.size() - 1; index >= 0; index--)
if ((uint8_t &)(str[index]) != 255)
break;
// Must not be called with a string that consists only of zero or more
// '\xff' bytes, and the empty string has no successor.
// '\xff' bytes.
if (index < 0) {
ok = false;
return {};
@@ -5692,8 +5690,7 @@ std::string strinc(std::string_view str, bool &ok) {
ok = true;
auto r = std::string(str.substr(0, index + 1));
auto &last = r[r.size() - 1];
last = static_cast<char>(static_cast<uint8_t>(last) + 1);
((uint8_t &)r[r.size() - 1])++;
return r;
}
+46
View File
@@ -0,0 +1,46 @@
FROM ubuntu:rolling
WORKDIR /tmp
ENV HOME=/tmp
RUN chmod -R 777 /tmp
# Install apt dependencies
RUN apt-get update
RUN apt-get upgrade -y
RUN TZ=America/Los_Angeles DEBIAN_FRONTEND=noninteractive apt-get install -y \
biber \
build-essential \
ccache \
clang \
cmake \
curl \
devscripts \
gcovr \
git \
latexmk \
libc6-dbg \
llvm \
mold \
ninja-build \
nodejs \
pre-commit \
python3-requests \
rpm \
texlive-bibtex-extra \
texlive-fonts-recommended \
texlive-latex-extra \
texlive-pictures \
valgrind \
wget \
zstd
# MinIO client, for uploading build artifacts
RUN curl -Ls "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/mc" \
-o /usr/local/bin/mc && chmod +x /usr/local/bin/mc
ENV CC=clang
ENV CXX=clang++
# Try to have all the pre-commit hooks we'll need already initialized
COPY .pre-commit-config.yaml /tmp/
RUN git init && pre-commit install-hooks
+4 -15
View File
@@ -96,9 +96,7 @@ void ConflictSet::setOldestVersion(int64_t oldestVersion) {
return impl->setOldestVersion(oldestVersion);
}
// The hash_table implementation does not track memory usage, so return 0 to
// satisfy the API contract that getBytes() returns a non-negative value.
int64_t ConflictSet::getBytes() const { return 0; }
int64_t ConflictSet::getBytes() const { return -1; }
void ConflictSet::getMetricsV1(MetricsV1 **metrics, int *count) const {
*metrics = nullptr;
@@ -121,13 +119,7 @@ ConflictSet::ConflictSet(ConflictSet &&other) noexcept
: impl(std::exchange(other.impl, nullptr)) {}
ConflictSet &ConflictSet::operator=(ConflictSet &&other) noexcept {
if (this != &other) {
if (impl) {
impl->~Impl();
safe_free(impl, sizeof(Impl));
}
impl = std::exchange(other.impl, nullptr);
}
impl = std::exchange(other.impl, nullptr);
return *this;
}
@@ -163,10 +155,7 @@ __attribute__((__visibility__("default"))) void ConflictSet_destroy(void *cs) {
}
__attribute__((__visibility__("default"))) int64_t
ConflictSet_getBytes(void *cs) {
(void)cs;
// The hash_table implementation does not track memory usage, so return 0 to
// satisfy the API contract that ConflictSet_getBytes returns a non-negative
// value.
return 0;
using Impl = ConflictSet::Impl;
return -1;
}
}
+6 -7
View File
@@ -1,6 +1,5 @@
#include <ConflictSet.h>
#include <algorithm>
#include <cerrno>
#include <chrono>
#include <cstdio>
@@ -78,10 +77,10 @@ int main(int argc, const char **argv) {
begin = end + 1;
end = (uint8_t *)memchr(begin, '\n', size);
if (line.size() >= 2 && line[0] == 'P') {
write = line.subspan(2, line.size() - 2);
} else if (line.size() >= 2 && line[0] == 'L') {
reads.push_back(line.subspan(2, line.size() - 2));
if (line.size() > 0 && line[0] == 'P') {
write = line.subspan(2, line.size());
} else if (line.size() > 0 && line[0] == 'L') {
reads.push_back(line.subspan(2, line.size()));
} else if (line.empty()) {
{
readRanges.resize(reads.size());
@@ -91,7 +90,7 @@ int main(int argc, const char **argv) {
iter->begin.len = read.size();
checkBytes += read.size();
iter->end.len = 0;
iter->readVersion = std::max<int64_t>(0, version - 100);
iter->readVersion = version - 100;
++iter;
}
}
@@ -122,7 +121,7 @@ int main(int argc, const char **argv) {
}
timer = now();
cs.setOldestVersion(std::max<int64_t>(0, version - 10000));
cs.setOldestVersion(version - 10000);
gcTime += now() - timer;
}
}
-1
View File
@@ -12,7 +12,6 @@
#include <sys/ioctl.h>
#include <sys/resource.h>
#include <sys/socket.h>
#include <sys/syscall.h>
#include <sys/types.h>
#include <sys/uio.h>
#include <thread>
+4 -14
View File
@@ -38,9 +38,6 @@ std::span<const uint8_t> keyAfter(Arena &arena, std::span<const uint8_t> key) {
std::span<const uint8_t> copyToArena(Arena &arena,
std::span<const uint8_t> key) {
if (key.size() == 0) {
return std::span<const uint8_t>();
}
auto result = std::span<uint8_t>(new (arena) uint8_t[key.size()], key.size());
memcpy(result.data(), key.data(), key.size());
return result;
@@ -100,7 +97,7 @@ force_inline bool getCharacter(const KeyInfo &ki, int character,
bool operator<(const KeyInfo &lhs, const KeyInfo &rhs) {
int i = std::min(lhs.key.size(), rhs.key.size());
int c = i > 0 ? memcmp(lhs.key.data(), rhs.key.data(), i) : 0;
int c = memcmp(lhs.key.data(), rhs.key.data(), i);
if (c != 0)
return c < 0;
@@ -290,8 +287,7 @@ private:
static force_inline bool less(const uint8_t *a, int aLen, const uint8_t *b,
int bLen) {
int n = std::min(aLen, bLen);
int c = n > 0 ? memcmp(a, b, n) : 0;
int c = memcmp(a, b, std::min(aLen, bLen));
if (c < 0)
return true;
if (c > 0)
@@ -758,8 +754,7 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
void addWrites(const ConflictSet::WriteRange *writes, int count,
int64_t writeVersion) {
auto points = std::vector<KeyInfo>();
points.reserve(count * 2);
auto points = std::vector<KeyInfo>(count * 2);
Arena arena;
for (int r = 0; r < count; r++) {
@@ -986,12 +981,7 @@ ConflictSet::ConflictSet(ConflictSet &&other) noexcept
: impl(std::exchange(other.impl, nullptr)) {}
ConflictSet &ConflictSet::operator=(ConflictSet &&other) noexcept {
if (this != &other) {
if (impl) {
internal_destroy(impl);
}
impl = std::exchange(other.impl, nullptr);
}
impl = std::exchange(other.impl, nullptr);
return *this;
}
-43
View File
@@ -1,43 +0,0 @@
#!/bin/bash
set -euxo pipefail
LLVM_VERSION="${LLVM_VERSION:-21}"
MSAN_PREFIX="${MSAN_PREFIX:-$PWD/msan}"
JOBS="${JOBS:-$(nproc)}"
cd /tmp
rm -rf libcxx-msan
mkdir libcxx-msan
cd libcxx-msan
git clone --depth=1 "https://github.com/llvm/llvm-project.git" -b "release/${LLVM_VERSION}.x"
cmake -S llvm-project/runtimes -B build \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER="clang-${LLVM_VERSION}" \
-DCMAKE_CXX_COMPILER="clang++-${LLVM_VERSION}" \
-DLLVM_USE_SANITIZER=MemoryWithOrigins \
-DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
-DLLVM_TARGETS_TO_BUILD=Native \
-DCMAKE_INSTALL_PREFIX="${MSAN_PREFIX}" \
-DLIBCXXABI_USE_LLVM_UNWINDER=ON \
-DLIBCXXABI_ENABLE_STATIC_UNWINDER=ON \
-DLIBCXX_USE_COMPILER_RT=ON \
-DLIBCXXABI_USE_COMPILER_RT=ON \
-DLIBUNWIND_USE_COMPILER_RT=ON \
-DLIBCXX_INCLUDE_TESTS=OFF \
-DLIBCXXABI_INCLUDE_TESTS=OFF \
-DLIBUNWIND_INCLUDE_TESTS=OFF \
-DLIBCXX_ENABLE_CLANG_TIDY=OFF
cmake --build build -j"${JOBS}"
cmake --install build
VERSION="$(clang-"${LLVM_VERSION}" --version | head -n1 | sed -E 's/.*clang version ([0-9.]+).*/\1/')"
TARBALL="msan-toolchain-${VERSION}.tar.zst"
tar --zstd -cf "${TARBALL}" -C "${MSAN_PREFIX}" .
TARBALL_PATH="$(pwd)/${TARBALL}"
echo "Created: ${TARBALL_PATH}"
+9 -23
View File
@@ -27,37 +27,23 @@ class Result(enum.Enum):
TOO_OLD = 2
def _make_key(buf: bytes) -> tuple[_Key, bytearray]:
"""Create a _Key and a backing bytearray that must be kept alive."""
backing = bytearray(buf)
array = (ctypes.c_ubyte * len(backing)).from_buffer(backing)
return _Key(array, len(array)), backing
def write(begin: bytes, end: Optional[bytes] = None) -> WriteRange:
begin_key, begin_buf = _make_key(begin)
b = (ctypes.c_ubyte * len(begin)).from_buffer(bytearray(begin))
if end is None:
end_key = _Key((ctypes.c_ubyte * 0)(), 0)
end_buf = None
e = (ctypes.c_ubyte * 0)()
else:
end_key, end_buf = _make_key(end)
result = WriteRange(begin_key, end_key)
result._begin_buf = begin_buf
result._end_buf = end_buf
return result
e = (ctypes.c_ubyte * len(end)).from_buffer(bytearray(end))
return WriteRange(_Key(b, len(b)), _Key(e, len(e)))
def read(version: int, begin: bytes, end: Optional[bytes] = None) -> ReadRange:
begin_key, begin_buf = _make_key(begin)
b = (ctypes.c_ubyte * len(begin)).from_buffer(bytearray(begin))
if end is None:
end_key = _Key((ctypes.c_ubyte * 0)(), 0)
end_buf = None
e = (ctypes.c_ubyte * 0)()
else:
end_key, end_buf = _make_key(end)
result = ReadRange(begin_key, end_key, version)
result._begin_buf = begin_buf
result._end_buf = end_buf
return result
e = (ctypes.c_ubyte * len(end)).from_buffer(bytearray(end))
return ReadRange(_Key(b, len(b)), _Key(e, len(e)), version)
class ConflictSet:
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+2 -11
View File
@@ -88,8 +88,7 @@ struct __attribute__((__visibility__("default"))) ConflictSet {
~ConflictSet();
/** Returns the total bytes in use by this ConflictSet. Implementations that
* do not track memory usage return 0. */
/** Returns the total bytes in use by this ConflictSet */
int64_t getBytes() const;
/** Experimental! */
@@ -133,13 +132,6 @@ struct __attribute__((__visibility__("default"))) ConflictSet {
private:
Impl *impl;
#if __cplusplus <= 199711L
/* Declared private and left undefined to prevent copying in C++98/C++03.
The compiler would otherwise implicitly generate public copy operations,
which share the opaque Impl* and cause a double-free. */
ConflictSet(const ConflictSet &);
ConflictSet &operator=(const ConflictSet &);
#endif
};
} /* namespace weaselab */
@@ -219,8 +211,7 @@ ConflictSet *ConflictSet_create(int64_t oldestVersion);
void ConflictSet_destroy(ConflictSet *cs);
/** Returns the total bytes in use by this ConflictSet. Implementations that
* do not track memory usage return 0. */
/** Returns the total bytes in use by this ConflictSet */
int64_t ConflictSet_getBytes(const ConflictSet *cs);
#endif
+2 -49
View File
@@ -57,53 +57,6 @@ def test_conflict_set():
assert cs.check(read(0, key), read(1, key)) == [Result.TOO_OLD, Result.COMMIT]
def test_hash_table_getBytes():
# Regression test for issue #62: the hash_table implementation is
# point-query only and does not track memory usage, but getBytes() must
# still return a non-negative value rather than -1.
with ConflictSet(0, build_dir=build_dir, implementation="hash_table") as cs:
assert cs.getBytes() == 0
cs.addWrites(1, write(b"key"))
assert cs.getBytes() >= 0
assert cs.check(read(0, b"key")) == [Result.CONFLICT]
def test_write_read_without_outer_reference():
# Regression test for issue #42: WriteRange/ReadRange must keep their
# backing key buffers alive, because the C library reads the pointer
# stored in _Key while addWrites/check run.
with DebugConflictSet() as cs:
# The bytes literal is not referenced after this expression.
cs.addWrites(1, write(b"key"))
assert cs.check(read(0, b"key")) == [Result.CONFLICT]
cs.addWrites(2, write(b"a", b"z"))
assert cs.check(read(1, b"a", b"z")) == [Result.CONFLICT]
assert cs.check(read(1, b"b")) == [Result.CONFLICT]
assert cs.check(read(1, b"0")) == [Result.COMMIT]
def test_range_keeps_key_buffers_alive():
# Verify the fix for issue #42: returned range objects must retain a
# reference to the backing bytearray so the C pointer stays valid after
# the helper returns.
w = write(b"key")
assert w._begin_buf == bytearray(b"key")
assert w._end_buf is None
w2 = write(b"a", b"z")
assert w2._begin_buf == bytearray(b"a")
assert w2._end_buf == bytearray(b"z")
r = read(0, b"key")
assert r._begin_buf == bytearray(b"key")
assert r._end_buf is None
r2 = read(1, b"a", b"z")
assert r2._begin_buf == bytearray(b"a")
assert r2._end_buf == bytearray(b"z")
def test_update_zero_should_commit():
with DebugConflictSet() as cs1:
with DebugConflictSet() as cs2:
@@ -115,7 +68,7 @@ def test_update_zero_should_commit():
for i in range(256 - 17, 256):
cs2.addWrites(int(1), write(bytes([i])))
# Scan until first point write
assert cs2.check(read(0, b"\x00", bytes([256 - 17]))) == [Result.COMMIT]
cs2.check(read(0, b"\x00", bytes([256 - 17])))
def test_update_zero_should_conflict():
@@ -128,7 +81,7 @@ def test_update_zero_should_conflict():
# "zero" is now 2**31 + 100
cs1.addWrites(2**32 + 101, write(b"", b"\x02"), write(b"\x01"))
# rangeVersion of \x01 is now 2**31 + 100 ("max" of (2**31 + 100, 2**32 + 101))
assert cs1.check(read(2**32 + 1, b"\x00")) == [Result.CONFLICT]
cs1.check(read(2**32 + 1, b"\x00"))
# but 2**32 + 1 ">" 2**31 + 100 , and it incorrectly commits