Reordering seems to improve codegen

.pre-commit-config.yaml

@@ -24,6 +24,14 @@ repos:
         entry: "^#define SHOW_MEMORY 1$"
         language: pygrep
         types: [c++]
+  - repo: local
+    hooks:
+      - id: sim cache misses check
+        name: disallow checking in SIM_CACHE_MISSES=1
+        description: disallow checking in SIM_CACHE_MISSES=1
+        entry: "^#define SIM_CACHE_MISSES 1$"
+        language: pygrep
+        types: [c++]
   - repo: https://github.com/shellcheck-py/shellcheck-py
     rev: a23f6b85d0fdd5bb9d564e2579e678033debbdff # frozen: v0.10.0.1
     hooks:

Bench.cpp

@@ -332,24 +332,18 @@ void benchWorstCaseForRadixRangeRead() {
   auto end = std::vector<uint8_t>(kKeyLenForWorstCase - 1, 255);
   end.push_back(254);
 
-  weaselab::ConflictSet::ReadRange r[] = {
+  weaselab::ConflictSet::Result result;
-      {{begin.data(), int(begin.size())}, {end.data(), int(end.size())}, 0},
+  weaselab::ConflictSet::ReadRange r{
-  };
+      {begin.data(), int(begin.size())}, {end.data(), int(end.size())}, 0};
-  weaselab::ConflictSet::Result results[sizeof(r) / sizeof(r[0])];
-  for (auto &result : results) {
-    result = weaselab::ConflictSet::TooOld;
-  }
-  bench.batch(sizeof(r) / sizeof(r[0]));
 
   bench.run("worst case for radix tree", [&]() {
     for (int i = 0; i < 256; ++i) {
-      cs[i]->check(r, results, sizeof(r) / sizeof(r[0]));
+      result = weaselab::ConflictSet::TooOld;
-      for (auto result : results) {
+      cs[i]->check(&r, &result, 1);
       if (result != weaselab::ConflictSet::Commit) {
         abort();
       }
     }
-    }
   });
 
   // for (int i = 0; i < 256; ++i) {

CMakeLists.txt

@@ -33,15 +33,6 @@ endif()
 
 add_compile_options(-fdata-sections -ffunction-sections -Wswitch-enum
                     -Werror=switch-enum -fPIC)
-
-if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-  add_link_options("-Wno-unused-command-line-argument")
-endif()
-
-if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-  add_compile_options("-Wno-maybe-uninitialized")
-endif()
-
 if(NOT APPLE)
   # This causes some versions of clang to crash on macos
   add_compile_options(-g -fno-omit-frame-pointer)
@@ -104,23 +95,12 @@ target_compile_options(${PROJECT_NAME}-object PRIVATE -fno-exceptions
                                                       -fvisibility=hidden)
 target_include_directories(${PROJECT_NAME}-object
                            PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include)
-if(NOT LD_EXE)
-  set(LD_EXE ld)
-endif()
-add_custom_command(
-  OUTPUT ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o
-  COMMAND ${LD_EXE} -r $<TARGET_OBJECTS:${PROJECT_NAME}-object> -o
-          ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o
-  DEPENDS $<TARGET_OBJECTS:${PROJECT_NAME}-object>
-  COMMAND_EXPAND_LISTS)
 
-add_library(${PROJECT_NAME} SHARED ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o)
+add_library(${PROJECT_NAME} SHARED $<TARGET_OBJECTS:${PROJECT_NAME}-object>)
 set_target_properties(
   ${PROJECT_NAME} PROPERTIES LIBRARY_OUTPUT_DIRECTORY
                              "${CMAKE_CURRENT_BINARY_DIR}/radix_tree")
-if(CMAKE_BUILD_TYPE STREQUAL Debug)
+if(NOT CMAKE_BUILD_TYPE STREQUAL Debug)
-  set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX)
-else()
   set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE C)
 endif()
 
@@ -130,13 +110,19 @@ if(HAS_VERSION_SCRIPT)
     LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/linker.map)
 endif()
 
-add_library(${PROJECT_NAME}-static STATIC ${CMAKE_BINARY_DIR}/${PROJECT_NAME}.o)
+add_library(${PROJECT_NAME}-static STATIC
-if(CMAKE_BUILD_TYPE STREQUAL Debug)
+            $<TARGET_OBJECTS:${PROJECT_NAME}-object>)
-  set_target_properties(${PROJECT_NAME}-static PROPERTIES LINKER_LANGUAGE CXX)
+if(NOT CMAKE_BUILD_TYPE STREQUAL Debug)
-else()
   set_target_properties(${PROJECT_NAME}-static PROPERTIES LINKER_LANGUAGE C)
 endif()
-if(NOT APPLE)
+
+if(APPLE)
+  add_custom_command(
+    TARGET ${PROJECT_NAME}-static
+    PRE_LINK
+    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/privatize_symbols_macos.sh
+            $<TARGET_OBJECTS:${PROJECT_NAME}-object>)
+else()
   add_custom_command(
     TARGET ${PROJECT_NAME}-static
     POST_BUILD
@@ -191,13 +177,10 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
   target_compile_options(driver_skip_list PRIVATE ${TEST_FLAGS})
   target_link_libraries(driver_skip_list PRIVATE skip_list)
 
-  # enable to test skip list
-  if(0)
   foreach(TEST ${CORPUS_TESTS})
     get_filename_component(hash ${TEST} NAME)
     add_test(NAME skip_list_${hash} COMMAND driver_skip_list ${TEST})
   endforeach()
-  endif()
 
   # ad hoc testing
   add_executable(conflict_set_main ConflictSet.cpp)
@@ -265,19 +248,6 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
     add_test(NAME conflict_set_blackbox_${hash} COMMAND driver ${TEST})
   endforeach()
 
-  find_program(VALGRIND_EXE valgrind)
-  if(VALGRIND_EXE AND NOT CMAKE_CROSSCOMPILING)
-    list(LENGTH CORPUS_TESTS len)
-    math(EXPR last "${len} - 1")
-    set(partition_size 100)
-    foreach(i RANGE 0 ${last} ${partition_size})
-      list(SUBLIST CORPUS_TESTS ${i} ${partition_size} partition)
-      add_test(NAME conflict_set_blackbox_valgrind_${i}
-               COMMAND ${VALGRIND_EXE} --error-exitcode=99 --
-                       $<TARGET_FILE:driver> ${partition})
-    endforeach()
-  endif()
-
   # scripted tests. Written manually to fill in anything libfuzzer couldn't
   # find.
   if(NOT CMAKE_CROSSCOMPILING)
@@ -298,14 +268,19 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING)
           ${Python3_EXECUTABLE}
           ${CMAKE_CURRENT_SOURCE_DIR}/test_conflict_set.py test ${TEST}
           --build-dir ${CMAKE_CURRENT_BINARY_DIR})
-      if(VALGRIND_EXE AND NOT CMAKE_CROSSCOMPILING)
+    endforeach()
-        add_test(
-          NAME script_test_${TEST}_valgrind
-          COMMAND
-            ${VALGRIND_EXE} ${Python3_EXECUTABLE}
-            ${CMAKE_CURRENT_SOURCE_DIR}/test_conflict_set.py test ${TEST}
-            --build-dir ${CMAKE_CURRENT_BINARY_DIR})
   endif()
+
+  find_program(VALGRIND_EXE valgrind)
+  if(VALGRIND_EXE AND NOT CMAKE_CROSSCOMPILING)
+    list(LENGTH CORPUS_TESTS len)
+    math(EXPR last "${len} - 1")
+    set(partition_size 100)
+    foreach(i RANGE 0 ${last} ${partition_size})
+      list(SUBLIST CORPUS_TESTS ${i} ${partition_size} partition)
+      add_test(NAME conflict_set_blackbox_valgrind_${i}
+               COMMAND ${VALGRIND_EXE} --error-exitcode=99 --
+                       $<TARGET_FILE:driver> ${partition})
     endforeach()
   endif()
 

Dockerfile

@@ -8,25 +8,25 @@ RUN chmod -R 777 /tmp
 RUN apt-get update
 RUN apt-get upgrade -y
 RUN TZ=America/Los_Angeles DEBIAN_FRONTEND=noninteractive apt-get install -y \
-    binutils-aarch64-linux-gnu \
     build-essential \
     ccache \
+    clang \
     cmake \
     curl \
+    doxygen \
+    file \
     g++-aarch64-linux-gnu \
     gcovr \
     git \
-    gnupg \
+    gperf \
+    graphviz \
     libc6-dbg \
-    lsb-release \
     ninja-build \
     pre-commit \
     python3-requests \
     qemu-user \
     rpm \
-    software-properties-common \
     texlive-full \
-    wget \
     zstd
 
 # Install recent valgrind from source
@@ -42,11 +42,6 @@ RUN curl -Ls https://sourceware.org/pub/valgrind/valgrind-3.22.0.tar.bz2 -o valg
     cd .. && \
     rm -rf /tmp/*
 
-# Recent clang
-RUN wget https://apt.llvm.org/llvm.sh && chmod +x ./llvm.sh && ./llvm.sh 20
-
-RUN apt-get -y install clang llvm
-
 # Set after building valgrind, which doesn't build with clang for some reason
 ENV CC=clang
 ENV CXX=clang++

Jenkinsfile

@@ -36,6 +36,18 @@ pipeline {
                 sh 'pre-commit run --all-files --show-diff-on-failure'
             }
         }
+        stage('Clang') {
+            agent {
+                dockerfile {
+                    args '-v /home/jenkins/ccache:/ccache'
+                    reuseNode true
+                }
+            }
+            steps {
+                CleanBuildAndTest("")
+                recordIssues(tools: [clang()])
+            }
+        }
         stage('64 bit versions') {
             agent {
                 dockerfile {
@@ -69,7 +81,7 @@ pipeline {
                 CleanBuildAndTest("-DUSE_SIMD_FALLBACK=ON")
             }
         }
-        stage('Release [clang]') {
+        stage('Release [gcc]') {
             agent {
                 dockerfile {
                     args '-v /home/jenkins/ccache:/ccache'
@@ -77,8 +89,8 @@ pipeline {
                 }
             }
             steps {
-                CleanBuildAndTest("-DCMAKE_CXX_FLAGS=-DNVALGRIND")
+                CleanBuildAndTest("-DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_CXX_FLAGS=-DNVALGRIND")
-                recordIssues(tools: [clang()])
+                recordIssues(tools: [gcc()])
                 sh '''
                 cd build
                 cpack -G DEB
@@ -91,19 +103,7 @@ pipeline {
                 minio bucket: 'jenkins', credentialsId: 'jenkins-minio', excludes: '', host: 'minio.weaselab.dev', includes: 'build/*.deb,build/*.rpm,paper/*.pdf', targetFolder: '${JOB_NAME}/${BUILD_NUMBER}/${STAGE_NAME}/'
             }
         }
-        stage('Release [gcc]') {
+        stage('Release [gcc,aarch64]') {
-            agent {
-                dockerfile {
-                    args '-v /home/jenkins/ccache:/ccache'
-                    reuseNode true
-                }
-            }
-            steps {
-                CleanBuildAndTest("-DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_CXX_FLAGS=-DNVALGRIND")
-                recordIssues(tools: [gcc()])
-            }
-        }
-        stage('Release [clang,aarch64]') {
             agent {
                 dockerfile {
                     args '-v /home/jenkins/ccache:/ccache'
@@ -129,16 +129,16 @@ pipeline {
             }
             steps {
                 script {
-                    gcov_args = "-f ConflictSet.cpp -f LongestCommonPrefix.h -f Metrics.h --gcov-executable 'llvm-cov gcov' --exclude-noncode-lines"
+                    filter_args = "-f ConflictSet.cpp -f LongestCommonPrefix.h -f Metrics.h"
                 }
-                CleanBuildAndTest("-DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DCMAKE_BUILD_TYPE=Debug -DDISABLE_TSAN=ON")
+                CleanBuildAndTest("-DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DCMAKE_BUILD_TYPE=Debug -DDISABLE_TSAN=ON")
                 sh """
-                gcovr ${gcov_args} --cobertura > build/coverage.xml
+                gcovr ${filter_args} --cobertura > build/coverage.xml
                 """
                 recordCoverage qualityGates: [[criticality: 'NOTE', metric: 'MODULE']], tools: [[parser: 'COBERTURA', pattern: 'build/coverage.xml']]
                 sh """
-                gcovr ${gcov_args}
+                gcovr ${filter_args}
-                gcovr ${gcov_args} --fail-under-line 100 > /dev/null
+                gcovr ${filter_args} --fail-under-line 100 > /dev/null
                 """
             }
         }

LongestCommonPrefix.h

@@ -129,7 +129,7 @@ longestCommonPrefix(const uint8_t *ap, const uint8_t *bp, int cl) {
   }
 
   int i = 0;
-  int end; // GCOVR_EXCL_LINE
+  int end;
 
   // kStride * kUnrollCount at a time
   end = cl & ~(kStride * kUnrollFactor - 1);

ServerBench.cpp

@@ -57,21 +57,20 @@ void workload(weaselab::ConflictSet *cs) {
     cs->addWrites(&wr, 1, version);
   }
 
-  constexpr int kNumReads = 1;
+  std::vector<weaselab::ConflictSet::Result> results(10);
-  std::vector<weaselab::ConflictSet::Result> results(kNumReads);
   for (;; transactions.fetch_add(1, std::memory_order_relaxed)) {
-    std::vector<std::string> keys(kNumReads);
+    std::vector<std::string> keys(10);
     for (auto &k : keys) {
       k = makeKey(rand() % kNumPrefixes, 49);
     }
-    std::vector<weaselab::ConflictSet::ReadRange> reads(kNumReads);
+    std::vector<weaselab::ConflictSet::ReadRange> reads(10);
     for (int i = 0; i < reads.size(); ++i) {
       reads[i].begin.p = (const uint8_t *)(keys[i].data());
       reads[i].begin.len = keys[i].size();
       reads[i].end.len = 0;
       reads[i].readVersion = version - 1;
     }
-    cs->check(reads.data(), results.data(), kNumReads);
+    cs->check(reads.data(), results.data(), 10);
   }
 }
 

aarch64-toolchain.cmake

@@ -1,8 +1,7 @@
 set(CMAKE_SYSTEM_NAME Linux)
 set(CMAKE_SYSTEM_PROCESSOR aarch64)
-set(CMAKE_C_COMPILER "clang;--target=aarch64-linux-gnu")
+set(CMAKE_C_COMPILER "/usr/bin/aarch64-linux-gnu-gcc")
-set(CMAKE_CXX_COMPILER "clang++;--target=aarch64-linux-gnu")
+set(CMAKE_CXX_COMPILER "/usr/bin/aarch64-linux-gnu-g++")
 set(CMAKE_FIND_ROOT_PATH /usr/aarch64-linux-gnu)
 set(CMAKE_CROSSCOMPILING_EMULATOR "qemu-aarch64;-L;/usr/aarch64-linux-gnu/")
 set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE arm64)
-set(LD_EXE "/usr/bin/aarch64-linux-gnu-ld")