diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 5b846de..aa19795 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: - name: 64-bit-versions cmake_args: -DCMAKE_CXX_FLAGS=-DUSE_64_BIT=1 - name: debug - cmake_args: -DCMAKE_BUILD_TYPE=Debug + cmake_args: -DCMAKE_BUILD_TYPE=Debug -DMSAN_TOOLCHAIN_PATH=/opt/msan - name: simd-fallback cmake_args: -DUSE_SIMD_FALLBACK=ON - name: gcc @@ -76,6 +76,13 @@ jobs: sudo update-alternatives --install /usr/bin/${tool} ${tool} /usr/bin/${tool}-21 100 done + - name: Download MSan toolchain + if: matrix.name == 'debug' + run: | + curl -Ls "https://minio.weaselab.dev/public/x86_64/msan-toolchain-21.1.8.tar.zst" -o /tmp/msan-toolchain.tar.zst + sudo mkdir -p /opt/msan + sudo tar --zstd -xf /tmp/msan-toolchain.tar.zst -C /opt/msan + - name: Build run: | export CCACHE_DIR="$GITHUB_WORKSPACE/.ccache" diff --git a/CMakeLists.txt b/CMakeLists.txt index 4dfdd73..bd433a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ project( DESCRIPTION "A data structure for optimistic concurrency control on ranges of bitwise-lexicographically-ordered keys." HOMEPAGE_URL "https://git.weaselab.dev/weaselab/conflict-set" - LANGUAGES C CXX) + LANGUAGES C CXX ASM) set(CMAKE_CXX_STANDARD 20) file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/version.txt ${PROJECT_VERSION}) @@ -130,7 +130,15 @@ endif() set(CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "") -add_library(${PROJECT_NAME}-object OBJECT ConflictSet.cpp) +# Architecture-specific SIMD assembly. These functions operate on +# potentially-indeterminate memory, which is UB in C++ but well-defined in +# assembly. +set(SIMD_ASM_FILES) +if(CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64 AND NOT USE_SIMD_FALLBACK) + set(SIMD_ASM_FILES ${CMAKE_CURRENT_SOURCE_DIR}/simd_x86_64.S) +endif() + +add_library(${PROJECT_NAME}-object OBJECT ConflictSet.cpp ${SIMD_ASM_FILES}) target_compile_options(${PROJECT_NAME}-object PRIVATE -fno-exceptions -fvisibility=hidden) target_include_directories(${PROJECT_NAME}-object @@ -233,7 +241,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING) endif() # ad hoc testing - add_executable(conflict_set_main ConflictSet.cpp) + add_executable(conflict_set_main ConflictSet.cpp ${SIMD_ASM_FILES}) target_include_directories(conflict_set_main PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) target_compile_definitions(conflict_set_main PRIVATE ENABLE_MAIN) @@ -249,7 +257,7 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING) cmake_pop_check_state() if(HAS_LIB_FUZZER) - add_executable(conflict_set_fuzz_test ConflictSet.cpp) + add_executable(conflict_set_fuzz_test ConflictSet.cpp ${SIMD_ASM_FILES}) target_include_directories(conflict_set_fuzz_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) target_compile_definitions(conflict_set_fuzz_test PRIVATE ENABLE_FUZZ) @@ -261,7 +269,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING) endif() # whitebox tests asan+ubsan - add_executable(fuzz_driver ConflictSet.cpp FuzzTestDriver.cpp) + add_executable(fuzz_driver ConflictSet.cpp FuzzTestDriver.cpp + ${SIMD_ASM_FILES}) target_compile_options(fuzz_driver PRIVATE ${TEST_FLAGS}) if(NOT CMAKE_CROSSCOMPILING) target_compile_options(fuzz_driver PRIVATE -fsanitize=address,undefined) @@ -277,7 +286,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING) # whitebox tests msan if(MSAN_TOOLCHAIN_PATH) - add_executable(fuzz_driver_msan ConflictSet.cpp FuzzTestDriver.cpp) + add_executable(fuzz_driver_msan ConflictSet.cpp FuzzTestDriver.cpp + ${SIMD_ASM_FILES}) target_compile_options(fuzz_driver_msan PRIVATE ${TEST_FLAGS}) if(NOT CMAKE_CROSSCOMPILING) target_compile_options( @@ -304,7 +314,8 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING) # tsan tests if(NOT CMAKE_CROSSCOMPILING AND NOT DISABLE_TSAN) - add_executable(tsan_driver ConflictSet.cpp FuzzTestDriver.cpp) + add_executable(tsan_driver ConflictSet.cpp FuzzTestDriver.cpp + ${SIMD_ASM_FILES}) target_compile_options(tsan_driver PRIVATE ${TEST_FLAGS} -fsanitize=thread) target_link_options(tsan_driver PRIVATE -fsanitize=thread) target_compile_definitions(tsan_driver PRIVATE ENABLE_FUZZ THREAD_TEST) diff --git a/ConflictSet.cpp b/ConflictSet.cpp index a3b9143..8ebab6c 100644 --- a/ConflictSet.cpp +++ b/ConflictSet.cpp @@ -28,6 +28,7 @@ limitations under the License. #include "Internal.h" #include "LongestCommonPrefix.h" #include "Metrics.h" +#include "simd.h" #include #include @@ -910,34 +911,11 @@ int getNodeIndexExists(Node3 *self, uint8_t index) { int getNodeIndex(Node16 *self, uint8_t index) { -#ifdef HAS_AVX - // Based on https://www.the-paper-trail.org/post/art-paper-notes/ - - // key_vec is 16 repeated copies of the searched-for byte, one for every - // possible position in child_keys that needs to be searched. - __m128i key_vec = _mm_set1_epi8(index); - - // Compare all child_keys to 'index' in parallel. Don't worry if some of the - // keys aren't valid, we'll mask the results to only consider the valid ones - // below. - __m128i indices; - memcpy(&indices, self->index, Node16::kMaxNodes); - __m128i results = _mm_cmpeq_epi8(key_vec, indices); - - // Build a mask to select only the first node->num_children values from the - // comparison (because the other values are meaningless) - uint32_t mask = (1 << self->numChildren) - 1; - - // Change the results of the comparison into a bitfield, masking off any - // invalid comparisons. - uint32_t bitfield = _mm_movemask_epi8(results) & mask; - - // No match if there are no '1's in the bitfield. +#if defined(__x86_64__) && !defined(USE_SIMD_FALLBACK) + uint32_t bitfield = + find_eq_16(self->index, index) & ((1 << self->numChildren) - 1); if (bitfield == 0) return -1; - - // Find the index of the first '1' in the bitfield by counting the leading - // zeros. return std::countr_zero(bitfield); #elif defined(HAS_ARM_NEON) // Based on @@ -970,13 +948,9 @@ int getNodeIndex(Node16 *self, uint8_t index) { int getNodeIndexExists(Node16 *self, uint8_t index) { -#ifdef HAS_AVX - __m128i key_vec = _mm_set1_epi8(index); - __m128i indices; - memcpy(&indices, self->index, Node16::kMaxNodes); - __m128i results = _mm_cmpeq_epi8(key_vec, indices); - uint32_t mask = (1 << self->numChildren) - 1; - uint32_t bitfield = _mm_movemask_epi8(results) & mask; +#if defined(__x86_64__) && !defined(USE_SIMD_FALLBACK) + uint32_t bitfield = + find_eq_16(self->index, index) & ((1 << self->numChildren) - 1); assume(bitfield != 0); return std::countr_zero(bitfield); #elif defined(HAS_ARM_NEON) @@ -1266,13 +1240,9 @@ TaggedNodePointer getChildGeq(Node16 *self, int child) { return nullptr; } -#ifdef HAS_AVX - __m128i key_vec = _mm_set1_epi8(child); - __m128i indices; - memcpy(&indices, self->index, Node16::kMaxNodes); - __m128i results = _mm_cmpeq_epi8(key_vec, _mm_min_epu8(key_vec, indices)); - int mask = (1 << self->numChildren) - 1; - uint32_t bitfield = _mm_movemask_epi8(results) & mask; +#if defined(__x86_64__) && !defined(USE_SIMD_FALLBACK) + uint32_t bitfield = + find_ge_16(self->index, child) & ((1 << self->numChildren) - 1); return bitfield == 0 ? nullptr : self->children[std::countr_zero(bitfield)]; #elif defined(HAS_ARM_NEON) uint8x16_t indices; @@ -2130,13 +2100,9 @@ bool scan16(const InternalVersionT *vs, const uint8_t *is, int begin, int end, return !(compared & mask); -#elif defined(HAS_AVX) +#elif defined(__x86_64__) && !defined(USE_SIMD_FALLBACK) - __m128i indices; - memcpy(&indices, is, 16); - indices = _mm_sub_epi8(indices, _mm_set1_epi8(begin)); - uint32_t mask = ~_mm_movemask_epi8(_mm_cmpeq_epi8( - indices, _mm_max_epu8(indices, _mm_set1_epi8(end - begin)))); + uint32_t mask = mask_in_range_16(is, begin, end); uint32_t compared = 0; if constexpr (kAVX512) { @@ -2153,12 +2119,14 @@ bool scan16(const InternalVersionT *vs, const uint8_t *is, int begin, int end, auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; }; uint32_t compared = 0; - for (int i = 0; i < 16; ++i) { - compared |= (vs[i] > readVersion) << i; - } uint32_t mask = 0; for (int i = 0; i < 16; ++i) { - mask |= inBounds(is[i]) << i; + if (vs[i] > readVersion) { + compared |= 1u << i; + if (inBounds(is[i])) { + mask |= 1u << i; + } + } } return !(compared & mask); @@ -2250,17 +2218,9 @@ bool checkMaxBetweenExclusiveImpl(Node3 *n, int begin, int end, auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; }; uint32_t mask = 0; - for (int i = 0; i < Node3::kMaxNodes; ++i) { + for (int i = 0; i < self->numChildren; ++i) { mask |= inBounds(self->index[i]) << i; } - mask &= (1 << self->numChildren) - 1; -#ifdef __aarch64__ - // The bits surviving the mask above don't derive from uninitialized slots, - // but clang 21+ on aarch64 lowers inBounds through flags+csel, which - // memcheck models imprecisely, tainting bits the mask provably clears. - // https://git.weaselab.dev/weaselab/conflict-set/issues/39 - VALGRIND_MAKE_MEM_DEFINED(&mask, sizeof(mask)); -#endif if (!mask) { return true; } @@ -2268,17 +2228,11 @@ bool checkMaxBetweenExclusiveImpl(Node3 *n, int begin, int end, const bool firstRangeOk = !child->entryPresent || child->entry.rangeVersion <= readVersion; uint32_t compared = 0; - for (int i = 0; i < Node3::kMaxNodes; ++i) { + for (int i = 0; i < self->numChildren; ++i) { compared |= (self->childMaxVersion[i] > readVersion) << i; } - uint32_t compared_masked = compared & mask; -#ifdef __aarch64__ - // Same imprecise csel modeling as above. - // https://git.weaselab.dev/weaselab/conflict-set/issues/39 - VALGRIND_MAKE_MEM_DEFINED(&compared_masked, sizeof(compared_masked)); -#endif - return !compared_masked && firstRangeOk; + return !(compared & mask) && firstRangeOk; } template @@ -2344,15 +2298,10 @@ bool checkMaxBetweenExclusiveImpl(Node16 *n, int begin, int end, return !(compared & mask) && firstRangeOk; -#elif defined(HAS_AVX) +#elif defined(__x86_64__) && !defined(USE_SIMD_FALLBACK) - __m128i indices; - memcpy(&indices, self->index, 16); - indices = _mm_sub_epi8(indices, _mm_set1_epi8(begin)); - uint32_t mask = - 0xffff & ~_mm_movemask_epi8(_mm_cmpeq_epi8( - indices, _mm_max_epu8(indices, _mm_set1_epi8(end - begin)))); - mask &= (1 << self->numChildren) - 1; + uint32_t mask = mask_in_range_16(self->index, begin, end) & + ((1 << self->numChildren) - 1); if (!mask) { return true; } @@ -2375,10 +2324,9 @@ bool checkMaxBetweenExclusiveImpl(Node16 *n, int begin, int end, auto inBounds = [&](unsigned c) { return c - shiftAmount < shiftUpperBound; }; uint32_t mask = 0; - for (int i = 0; i < 16; ++i) { + for (int i = 0; i < self->numChildren; ++i) { mask |= inBounds(self->index[i]) << i; } - mask &= (1 << self->numChildren) - 1; if (!mask) { return true; } @@ -2386,7 +2334,7 @@ bool checkMaxBetweenExclusiveImpl(Node16 *n, int begin, int end, const bool firstRangeOk = !child->entryPresent || child->entry.rangeVersion <= readVersion; uint32_t compared = 0; - for (int i = 0; i < 16; ++i) { + for (int i = 0; i < self->numChildren; ++i) { compared |= (self->childMaxVersion[i] > readVersion) << i; } return !(compared & mask) && firstRangeOk; @@ -3864,17 +3812,17 @@ PRESERVE_NONE void right_side_iter(Job *job, Context *context) { void Job::init(const ConflictSet::ReadRange *read, ConflictSet::Result *result, Node *root, int64_t oldestVersionFullPrecision) { auto begin = TrivialSpan(read->begin.p, read->begin.len); - auto end = TrivialSpan(read->end.p, read->end.len); if (read->readVersion < oldestVersionFullPrecision) [[unlikely]] { *result = ConflictSet::TooOld; continuation = complete; - } else if (end.size() == 0) { + } else if (read->end.len == 0) { this->begin = begin; this->n = root; this->readVersion = InternalVersionT(read->readVersion); this->result = result; continuation = check::point_read_state_machine::begin; } else { + auto end = TrivialSpan(read->end.p, read->end.len); this->begin = begin; this->end = end; this->n = root; @@ -5046,8 +4994,8 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl { for (int i = 0; i < count; ++i) { const auto &w = writes[i]; auto begin = TrivialSpan(w.begin.p, w.begin.len); - auto end = TrivialSpan(w.end.p, w.end.len); if (w.end.len > 0) { + auto end = TrivialSpan(w.end.p, w.end.len); addWriteRange(rootParent->children[0], begin, end, InternalVersionT(writeVersion), &writeContext); } else { diff --git a/simd.h b/simd.h new file mode 100644 index 0000000..2af1289 --- /dev/null +++ b/simd.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +#if defined(__x86_64__) && !defined(USE_SIMD_FALLBACK) + +// SIMD operations on potentially-indeterminate Node16::index[16] bytes. +// Implemented in file-level assembly (simd_x86_64.S) because loading and +// operating on indeterminate values is UB in C++ but well-defined in +// assembly. The caller must mask the returned bitfield to +// [0, numChildren) before using it. +// +// Each function returns a 16-bit bitmask in the low 16 bits of a uint32_t +// (upper 16 bits are zero). Bit i is set iff the condition holds at index i. + +extern "C" { +// Returns bit i set iff idx[i] == key +uint32_t find_eq_16(const uint8_t idx[16], uint8_t key); +// Returns bit i set iff idx[i] >= child +uint32_t find_ge_16(const uint8_t idx[16], uint8_t child); +// Returns bit i set iff begin <= idx[i] < end +uint32_t mask_in_range_16(const uint8_t idx[16], uint8_t begin, uint8_t end); +} + +#endif \ No newline at end of file diff --git a/simd_x86_64.S b/simd_x86_64.S new file mode 100644 index 0000000..38ed58b --- /dev/null +++ b/simd_x86_64.S @@ -0,0 +1,74 @@ +// SIMD operations on potentially-indeterminate Node16::index[16] bytes. +// Written in assembly so msan doesn't track the loads. The caller is +// responsible for masking the returned bitfield to [0, numChildren) before +// using it. +// +// All functions return a 16-bit bitmask in %eax (bit i set = condition true +// at index i). The upper 16 bits of %eax are zero. +// +// System V AMD64 ABI: +// %rdi = const uint8_t *idx (16 bytes) +// %esi = uint8_t key (find_eq_16, find_ge_16) +// %sil = uint8_t begin (mask_in_range_16) +// %dl = uint8_t end (mask_in_range_16) + + .text + +// uint32_t find_eq_16(const uint8_t idx[16], uint8_t key) +// Returns bit i set if idx[i] == key + .globl find_eq_16 + .type find_eq_16, @function +find_eq_16: + vmovd %esi, %xmm1 // broadcast key + vpbroadcastb %xmm1, %xmm1 + vmovdqu (%rdi), %xmm0 // load 16 bytes (may contain indeterminate data) + vpcmpeqb %xmm0, %xmm1, %xmm0 // 0xff for each match + vpmovmskb %xmm0, %eax // 16-bit bitmask + movzwl %ax, %eax // zero-extend to 32 bits + ret + .size find_eq_16, .-find_eq_16 + +// uint32_t find_ge_16(const uint8_t idx[16], uint8_t child) +// Returns bit i set if idx[i] >= child +// x86 doesn't have a "compare unsigned >=" for bytes directly, so we use: +// min(key, idx[i]) == key iff idx[i] >= key + .globl find_ge_16 + .type find_ge_16, @function +find_ge_16: + vmovd %esi, %xmm1 + vpbroadcastb %xmm1, %xmm1 // key broadcast + vmovdqu (%rdi), %xmm0 // load 16 bytes + vpminub %xmm0, %xmm1, %xmm2 // min(key, idx[i]) + vpcmpeqb %xmm2, %xmm1, %xmm0 // 0xff where min == key, i.e. idx[i] >= key + vpmovmskb %xmm0, %eax + movzwl %ax, %eax + ret + .size find_ge_16, .-find_ge_16 + +// uint32_t mask_in_range_16(const uint8_t idx[16], uint8_t begin, uint8_t end) +// Returns bit i set if begin <= idx[i] < end +// Logic: (idx[i] - begin) < (end - begin) [unsigned wrapping arithmetic] +// Equivalently: idx[i] - begin != max(idx[i] - begin, end - begin) +// i.e. idx[i] - begin is NOT equal to the saturated value. +// We compute: sub = idx - begin; result = (sub < (end-begin)) for each byte. +// Using: sub == max(sub, end-begin) means NOT in range. +// So: in_range = ~(movemask(cmpeq(sub, max(sub, range_size)))) + .globl mask_in_range_16 + .type mask_in_range_16, @function +mask_in_range_16: + vmovd %esi, %xmm1 // begin + vpbroadcastb %xmm1, %xmm1 + vmovd %edx, %xmm2 // end + vpbroadcastb %xmm2, %xmm2 + vmovdqu (%rdi), %xmm0 // load 16 bytes + vpsubb %xmm1, %xmm0, %xmm0 // idx - begin (wrapping) + vpsubb %xmm1, %xmm2, %xmm2 // end - begin (range size) + vpmaxub %xmm0, %xmm2, %xmm3 // max(idx-begin, range_size) + vpcmpeqb %xmm3, %xmm0, %xmm0 // 0xff where NOT in range + vpmovmskb %xmm0, %eax + not %eax // invert: 1 = in range + movzwl %ax, %eax + ret + .size mask_in_range_16, .-mask_in_range_16 + + .section .note.GNU-stack,"",@progbits \ No newline at end of file