Compare commits
19 Commits
v0.0.3
...
d2e1863593
| Author | SHA1 | Date | |
|---|---|---|---|
| d2e1863593 | |||
| bf91bca16d | |||
| 08ed17f47b | |||
| 76a45f16ad | |||
| c15d296432 | |||
| 64a98c529c | |||
| ed1388ed21 | |||
| 309d315956 | |||
| eab2e46a56 | |||
| 85db1a8786 | |||
| 717f9d6829 | |||
| fd93300ce8 | |||
| b7e16b31ff | |||
| a324d31518 | |||
| fdb05e0e33 | |||
| 7c27d4a972 | |||
| 738de01cb4 | |||
| 325cab6a95 | |||
| 0b2821941a |
+18
-9
@@ -1,7 +1,7 @@
|
|||||||
cmake_minimum_required(VERSION 3.18)
|
cmake_minimum_required(VERSION 3.18)
|
||||||
project(
|
project(
|
||||||
conflict-set
|
conflict-set
|
||||||
VERSION 0.0.3
|
VERSION 0.0.4
|
||||||
DESCRIPTION
|
DESCRIPTION
|
||||||
"A data structure for optimistic concurrency control on ranges of bitwise-lexicographically-ordered keys."
|
"A data structure for optimistic concurrency control on ranges of bitwise-lexicographically-ordered keys."
|
||||||
HOMEPAGE_URL "https://git.weaselab.dev/weaselab/conflict-set"
|
HOMEPAGE_URL "https://git.weaselab.dev/weaselab/conflict-set"
|
||||||
@@ -261,14 +261,22 @@ if(BUILD_TESTING)
|
|||||||
|
|
||||||
# scripted tests. Written manually to fill in anything libfuzzer couldn't
|
# scripted tests. Written manually to fill in anything libfuzzer couldn't
|
||||||
# find.
|
# find.
|
||||||
add_executable(script_test ScriptTest.cpp)
|
if(NOT CMAKE_CROSSCOMPILING)
|
||||||
target_compile_options(script_test PRIVATE ${TEST_FLAGS})
|
find_package(Python3 REQUIRED COMPONENTS Interpreter)
|
||||||
target_link_libraries(script_test PRIVATE ${PROJECT_NAME})
|
set_property(
|
||||||
file(GLOB SCRIPT_TESTS ${CMAKE_SOURCE_DIR}/script_tests/*)
|
DIRECTORY
|
||||||
foreach(TEST ${SCRIPT_TESTS})
|
APPEND
|
||||||
get_filename_component(name ${TEST} NAME)
|
PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/test_conflict_set.py)
|
||||||
add_test(NAME conflict_set_script_${name} COMMAND script_test ${TEST})
|
execute_process(
|
||||||
endforeach()
|
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/test_conflict_set.py
|
||||||
|
list OUTPUT_VARIABLE SCRIPT_TESTS)
|
||||||
|
foreach(TEST ${SCRIPT_TESTS})
|
||||||
|
add_test(
|
||||||
|
NAME script_test_${TEST}
|
||||||
|
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/test_conflict_set.py
|
||||||
|
test ${TEST} --build-dir ${CMAKE_BINARY_DIR})
|
||||||
|
endforeach()
|
||||||
|
endif()
|
||||||
|
|
||||||
find_program(VALGRIND_EXE valgrind)
|
find_program(VALGRIND_EXE valgrind)
|
||||||
if(VALGRIND_EXE AND NOT CMAKE_CROSSCOMPILING)
|
if(VALGRIND_EXE AND NOT CMAKE_CROSSCOMPILING)
|
||||||
@@ -357,6 +365,7 @@ else()
|
|||||||
endif()
|
endif()
|
||||||
|
|
||||||
# macos
|
# macos
|
||||||
|
set(CMAKE_OSX_DEPLOYMENT_TARGET 11.0)
|
||||||
if(APPLE)
|
if(APPLE)
|
||||||
find_program(PANDOC_EXE pandoc)
|
find_program(PANDOC_EXE pandoc)
|
||||||
if(PANDOC_EXE)
|
if(PANDOC_EXE)
|
||||||
|
|||||||
+35
-5
@@ -770,7 +770,7 @@ template <class NodeT> int getChildGeqSimd(NodeT *self, int child) {
|
|||||||
|
|
||||||
// cachegrind says the plain loop is fewer instructions and more mis-predicted
|
// cachegrind says the plain loop is fewer instructions and more mis-predicted
|
||||||
// branches. Microbenchmark says plain loop is faster. It's written in this
|
// branches. Microbenchmark says plain loop is faster. It's written in this
|
||||||
// weird "generic" way though in case someday we can use the simd
|
// weird "generic" way though so that someday we can use the simd
|
||||||
// implementation easily if we want.
|
// implementation easily if we want.
|
||||||
if constexpr (std::is_same_v<NodeT, Node3>) {
|
if constexpr (std::is_same_v<NodeT, Node3>) {
|
||||||
Node3 *n = (Node3 *)self;
|
Node3 *n = (Node3 *)self;
|
||||||
@@ -1146,8 +1146,14 @@ void freeAndMakeCapacityAtLeast(Node *&self, int capacity,
|
|||||||
// capacity.
|
// capacity.
|
||||||
void maybeDecreaseCapacity(Node *&self, NodeAllocators *allocators,
|
void maybeDecreaseCapacity(Node *&self, NodeAllocators *allocators,
|
||||||
ConflictSet::Impl *impl) {
|
ConflictSet::Impl *impl) {
|
||||||
|
|
||||||
const int maxCapacity =
|
const int maxCapacity =
|
||||||
(self->numChildren + int(self->entryPresent)) * (self->partialKeyLen + 1);
|
(self->numChildren + int(self->entryPresent)) * (self->partialKeyLen + 1);
|
||||||
|
#if DEBUG_VERBOSE && !defined(NDEBUG)
|
||||||
|
fprintf(stderr, "maybeDecreaseCapacity: current: %d, max: %d, key: %s\n",
|
||||||
|
self->getCapacity(), maxCapacity,
|
||||||
|
getSearchPathPrintable(self).c_str());
|
||||||
|
#endif
|
||||||
if (self->getCapacity() <= maxCapacity) {
|
if (self->getCapacity() <= maxCapacity) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -2389,7 +2395,11 @@ Iterator firstGeq(Node *n, const std::span<const uint8_t> key) {
|
|||||||
} else {
|
} else {
|
||||||
n = nextSibling(n);
|
n = nextSibling(n);
|
||||||
if (n == nullptr) {
|
if (n == nullptr) {
|
||||||
return {nullptr, 1};
|
// This line is genuinely unreachable from any entry point of the
|
||||||
|
// final library, since we can't remove a key without introducing a
|
||||||
|
// key after it, and the only production caller of firstGeq is for
|
||||||
|
// resuming the setOldestVersion scan.
|
||||||
|
return {nullptr, 1}; // GCOVR_EXCL_LINE
|
||||||
}
|
}
|
||||||
goto downLeftSpine;
|
goto downLeftSpine;
|
||||||
}
|
}
|
||||||
@@ -2510,6 +2520,10 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
|
|||||||
}
|
}
|
||||||
|
|
||||||
explicit Impl(int64_t oldestVersion) : oldestVersion(oldestVersion) {
|
explicit Impl(int64_t oldestVersion) : oldestVersion(oldestVersion) {
|
||||||
|
#if DEBUG_VERBOSE
|
||||||
|
fprintf(stderr, "radix_tree: create\n");
|
||||||
|
#endif
|
||||||
|
|
||||||
// Insert ""
|
// Insert ""
|
||||||
root = allocators.node0.allocate(0);
|
root = allocators.node0.allocate(0);
|
||||||
root->numChildren = 0;
|
root->numChildren = 0;
|
||||||
@@ -2524,7 +2538,12 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
|
|||||||
root->entry.pointVersion = oldestVersion;
|
root->entry.pointVersion = oldestVersion;
|
||||||
root->entry.rangeVersion = oldestVersion;
|
root->entry.rangeVersion = oldestVersion;
|
||||||
}
|
}
|
||||||
~Impl() { destroyTree(root); }
|
~Impl() {
|
||||||
|
#if DEBUG_VERBOSE
|
||||||
|
fprintf(stderr, "radix_tree: destroy\n");
|
||||||
|
#endif
|
||||||
|
destroyTree(root);
|
||||||
|
}
|
||||||
|
|
||||||
NodeAllocators allocators;
|
NodeAllocators allocators;
|
||||||
|
|
||||||
@@ -2647,17 +2666,27 @@ ConflictSet_check(void *cs, const ConflictSet_ReadRange *reads,
|
|||||||
__attribute__((__visibility__("default"))) void
|
__attribute__((__visibility__("default"))) void
|
||||||
ConflictSet_addWrites(void *cs, const ConflictSet_WriteRange *writes, int count,
|
ConflictSet_addWrites(void *cs, const ConflictSet_WriteRange *writes, int count,
|
||||||
int64_t writeVersion) {
|
int64_t writeVersion) {
|
||||||
auto *impl = ((ConflictSet::Impl *)cs);
|
auto *impl = (ConflictSet::Impl *)cs;
|
||||||
mallocBytesDelta = 0;
|
mallocBytesDelta = 0;
|
||||||
impl->addWrites(writes, count, writeVersion);
|
impl->addWrites(writes, count, writeVersion);
|
||||||
impl->totalBytes += mallocBytesDelta;
|
impl->totalBytes += mallocBytesDelta;
|
||||||
|
#if SHOW_MEMORY
|
||||||
|
if (impl->totalBytes != mallocBytes) {
|
||||||
|
abort();
|
||||||
|
}
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
__attribute__((__visibility__("default"))) void
|
__attribute__((__visibility__("default"))) void
|
||||||
ConflictSet_setOldestVersion(void *cs, int64_t oldestVersion) {
|
ConflictSet_setOldestVersion(void *cs, int64_t oldestVersion) {
|
||||||
auto *impl = ((ConflictSet::Impl *)cs);
|
auto *impl = (ConflictSet::Impl *)cs;
|
||||||
mallocBytesDelta = 0;
|
mallocBytesDelta = 0;
|
||||||
impl->setOldestVersion(oldestVersion);
|
impl->setOldestVersion(oldestVersion);
|
||||||
impl->totalBytes += mallocBytesDelta;
|
impl->totalBytes += mallocBytesDelta;
|
||||||
|
#if SHOW_MEMORY
|
||||||
|
if (impl->totalBytes != mallocBytes) {
|
||||||
|
abort();
|
||||||
|
}
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
__attribute__((__visibility__("default"))) void *
|
__attribute__((__visibility__("default"))) void *
|
||||||
ConflictSet_create(int64_t oldestVersion) {
|
ConflictSet_create(int64_t oldestVersion) {
|
||||||
@@ -2995,6 +3024,7 @@ void removeKey(Node *n) {
|
|||||||
|
|
||||||
struct __attribute__((visibility("default"))) PeakPrinter {
|
struct __attribute__((visibility("default"))) PeakPrinter {
|
||||||
~PeakPrinter() {
|
~PeakPrinter() {
|
||||||
|
printf("--- radix_tree ---\n");
|
||||||
printf("malloc bytes: %g\n", double(mallocBytes));
|
printf("malloc bytes: %g\n", double(mallocBytes));
|
||||||
printf("Peak malloc bytes: %g\n", double(peakMallocBytes));
|
printf("Peak malloc bytes: %g\n", double(peakMallocBytes));
|
||||||
printf("Node bytes: %g\n", double(nodeBytes));
|
printf("Node bytes: %g\n", double(nodeBytes));
|
||||||
|
|||||||
+10
-5
@@ -530,10 +530,13 @@ template <class ConflictSetImpl> struct TestDriver {
|
|||||||
ConflictSetImpl cs{oldestVersion};
|
ConflictSetImpl cs{oldestVersion};
|
||||||
ReferenceImpl refImpl{oldestVersion};
|
ReferenceImpl refImpl{oldestVersion};
|
||||||
|
|
||||||
constexpr static auto kMaxKeyLen = 8;
|
constexpr static auto kMaxKeySuffixLen = 8;
|
||||||
|
|
||||||
bool ok = true;
|
bool ok = true;
|
||||||
|
|
||||||
|
const int prefixLen = arbitrary.bounded(512);
|
||||||
|
const int prefixByte = arbitrary.randT<uint8_t>();
|
||||||
|
|
||||||
// Call until it returns true, for "done". Check internal invariants etc
|
// Call until it returns true, for "done". Check internal invariants etc
|
||||||
// between calls to next.
|
// between calls to next.
|
||||||
bool next() {
|
bool next() {
|
||||||
@@ -552,9 +555,10 @@ template <class ConflictSetImpl> struct TestDriver {
|
|||||||
if (!arbitrary.hasEntropy()) {
|
if (!arbitrary.hasEntropy()) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
int keyLen = arbitrary.bounded(kMaxKeyLen);
|
int keyLen = prefixLen + arbitrary.bounded(kMaxKeySuffixLen);
|
||||||
auto *begin = new (arena) uint8_t[keyLen];
|
auto *begin = new (arena) uint8_t[keyLen];
|
||||||
arbitrary.randomBytes(begin, keyLen);
|
memset(begin, prefixByte, prefixLen);
|
||||||
|
arbitrary.randomBytes(begin + prefixLen, keyLen - prefixLen);
|
||||||
keys.insert(std::string_view((const char *)begin, keyLen));
|
keys.insert(std::string_view((const char *)begin, keyLen));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -619,9 +623,10 @@ template <class ConflictSetImpl> struct TestDriver {
|
|||||||
if (!arbitrary.hasEntropy()) {
|
if (!arbitrary.hasEntropy()) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
int keyLen = arbitrary.bounded(kMaxKeyLen);
|
int keyLen = prefixLen + arbitrary.bounded(kMaxKeySuffixLen);
|
||||||
auto *begin = new (arena) uint8_t[keyLen];
|
auto *begin = new (arena) uint8_t[keyLen];
|
||||||
arbitrary.randomBytes(begin, keyLen);
|
memset(begin, prefixByte, prefixLen);
|
||||||
|
arbitrary.randomBytes(begin + prefixLen, keyLen - prefixLen);
|
||||||
keys.insert(std::string_view((const char *)begin, keyLen));
|
keys.insert(std::string_view((const char *)begin, keyLen));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Vendored
+3
@@ -111,6 +111,9 @@ pipeline {
|
|||||||
gcovr -f ConflictSet.cpp --cobertura > build/coverage.xml
|
gcovr -f ConflictSet.cpp --cobertura > build/coverage.xml
|
||||||
'''
|
'''
|
||||||
cobertura autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: 'build/coverage.xml', conditionalCoverageTargets: '70, 0, 0', failUnhealthy: false, failUnstable: false, lineCoverageTargets: '80, 0, 0', maxNumberOfBuilds: 0, methodCoverageTargets: '80, 0, 0', onlyStable: false, sourceEncoding: 'ASCII', zoomCoverageChart: false
|
cobertura autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: 'build/coverage.xml', conditionalCoverageTargets: '70, 0, 0', failUnhealthy: false, failUnstable: false, lineCoverageTargets: '80, 0, 0', maxNumberOfBuilds: 0, methodCoverageTargets: '80, 0, 0', onlyStable: false, sourceEncoding: 'ASCII', zoomCoverageChart: false
|
||||||
|
sh '''
|
||||||
|
gcovr -f ConflictSet.cpp --fail-under-line 100 > /dev/null
|
||||||
|
'''
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
-155
@@ -1,155 +0,0 @@
|
|||||||
#include "Internal.h"
|
|
||||||
#include <ConflictSet.h>
|
|
||||||
|
|
||||||
#include <chrono>
|
|
||||||
#include <cstring>
|
|
||||||
#include <fcntl.h>
|
|
||||||
#include <sys/mman.h>
|
|
||||||
#include <sys/stat.h>
|
|
||||||
#include <unistd.h>
|
|
||||||
#include <vector>
|
|
||||||
|
|
||||||
inline size_t getPageSize() {
|
|
||||||
static size_t kPageSize = sysconf(_SC_PAGESIZE);
|
|
||||||
return kPageSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Helper for rounding up to page size (or some other alignment)
|
|
||||||
constexpr inline size_t rightAlign(size_t offset, size_t alignment) {
|
|
||||||
return offset % alignment == 0 ? offset
|
|
||||||
: ((offset / alignment) + 1) * alignment;
|
|
||||||
}
|
|
||||||
|
|
||||||
using StringView = std::basic_string_view<uint8_t>;
|
|
||||||
|
|
||||||
inline StringView operator"" _v(const char *str, size_t size) {
|
|
||||||
return {reinterpret_cast<const uint8_t *>(str), size};
|
|
||||||
}
|
|
||||||
|
|
||||||
int main(int argc, const char **argv) {
|
|
||||||
|
|
||||||
ConflictSet cs{0};
|
|
||||||
ReferenceImpl ref{0};
|
|
||||||
|
|
||||||
for (int i = 1; i < argc; ++i) {
|
|
||||||
int fd = open(argv[i], O_RDONLY);
|
|
||||||
struct stat st;
|
|
||||||
if (fstat(fd, &st) == -1) {
|
|
||||||
int err = errno;
|
|
||||||
fprintf(stderr, "stat error %s - %s\n", argv[i], strerror(err));
|
|
||||||
fflush(stderr);
|
|
||||||
abort();
|
|
||||||
}
|
|
||||||
|
|
||||||
int64_t size = rightAlign(st.st_size, getPageSize());
|
|
||||||
const uint8_t *begin =
|
|
||||||
(uint8_t *)mmap(0, size, PROT_READ, MAP_PRIVATE, fd, 0);
|
|
||||||
madvise((void *)begin, size, MADV_SEQUENTIAL);
|
|
||||||
auto *const mapOriginal = begin;
|
|
||||||
const auto sizeOriginal = size;
|
|
||||||
|
|
||||||
StringView b;
|
|
||||||
StringView e;
|
|
||||||
int64_t v = 0;
|
|
||||||
int64_t lastWriteVersion = 0;
|
|
||||||
int64_t lastOldestVersion = 0;
|
|
||||||
std::vector<ConflictSet::WriteRange> writeRanges;
|
|
||||||
std::vector<ConflictSet::ReadRange> readRanges;
|
|
||||||
std::vector<ConflictSet::Result> results;
|
|
||||||
|
|
||||||
for (uint8_t *end = (uint8_t *)memchr(begin, '\n', size); end != nullptr;) {
|
|
||||||
StringView line{begin, static_cast<size_t>(end - begin)};
|
|
||||||
size -= end - begin + 1;
|
|
||||||
begin = end + 1;
|
|
||||||
end = (uint8_t *)memchr(begin, '\n', size);
|
|
||||||
|
|
||||||
if (line.starts_with("begin"_v)) {
|
|
||||||
b = line.substr("begin "_v.size(), line.size());
|
|
||||||
printf("b <- %.*s\n", int(b.size()), b.data());
|
|
||||||
} else if (line.starts_with("end"_v)) {
|
|
||||||
e = line.substr("end "_v.size(), line.size());
|
|
||||||
printf("e <- %.*s\n", int(e.size()), e.data());
|
|
||||||
} else if (line.starts_with("version"_v)) {
|
|
||||||
line = line.substr("version "_v.size(), line.size());
|
|
||||||
v = 0;
|
|
||||||
for (auto c : line) {
|
|
||||||
v = v * 10 + int(c) - int('0');
|
|
||||||
}
|
|
||||||
printf("v <- %" PRId64 "\n", v);
|
|
||||||
} else if (line.starts_with("pointread"_v)) {
|
|
||||||
printf("pointread\n");
|
|
||||||
ConflictSet::ReadRange r;
|
|
||||||
r.begin.p = b.data();
|
|
||||||
r.begin.len = b.size();
|
|
||||||
r.end.len = 0;
|
|
||||||
r.readVersion = v;
|
|
||||||
readRanges.push_back(r);
|
|
||||||
} else if (line.starts_with("pointwrite"_v)) {
|
|
||||||
printf("pointwrite\n");
|
|
||||||
assert(writeRanges.empty() ||
|
|
||||||
(writeRanges.back().end.len == 0 ? writeRanges.back().begin
|
|
||||||
: writeRanges.back().end) < b);
|
|
||||||
ConflictSet::WriteRange w;
|
|
||||||
w.begin.p = b.data();
|
|
||||||
w.begin.len = b.size();
|
|
||||||
w.end.len = 0;
|
|
||||||
writeRanges.push_back(w);
|
|
||||||
} else if (line.starts_with("rangeread"_v)) {
|
|
||||||
printf("rangeread\n");
|
|
||||||
ConflictSet::ReadRange r;
|
|
||||||
r.begin.p = b.data();
|
|
||||||
r.begin.len = b.size();
|
|
||||||
r.end.p = e.data();
|
|
||||||
r.end.len = e.size();
|
|
||||||
r.readVersion = v;
|
|
||||||
readRanges.push_back(r);
|
|
||||||
} else if (line.starts_with("rangewrite"_v)) {
|
|
||||||
printf("rangewrite\n");
|
|
||||||
assert(b < e);
|
|
||||||
assert(writeRanges.empty() ||
|
|
||||||
(writeRanges.back().end.len == 0 ? writeRanges.back().begin
|
|
||||||
: writeRanges.back().end) < b);
|
|
||||||
ConflictSet::WriteRange w;
|
|
||||||
w.begin.p = b.data();
|
|
||||||
w.begin.len = b.size();
|
|
||||||
w.end.p = e.data();
|
|
||||||
w.end.len = e.size();
|
|
||||||
writeRanges.push_back(w);
|
|
||||||
} else if (line.starts_with("check"_v)) {
|
|
||||||
printf("check\n");
|
|
||||||
Arena arena;
|
|
||||||
auto *expected = new (arena) ConflictSet::Result[readRanges.size()];
|
|
||||||
auto *actual = new (arena) ConflictSet::Result[readRanges.size()];
|
|
||||||
ref.check(readRanges.data(), expected, readRanges.size());
|
|
||||||
cs.check(readRanges.data(), actual, readRanges.size());
|
|
||||||
for (int i = 0; i < int(readRanges.size()); ++i) {
|
|
||||||
if (expected[i] != actual[i]) {
|
|
||||||
fprintf(stderr, "Expected %s, got %s at index %d\n",
|
|
||||||
resultToStr(expected[i]), resultToStr(actual[i]), i);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
readRanges = {};
|
|
||||||
} else if (line.starts_with("addwrites"_v)) {
|
|
||||||
printf("addwrites\n");
|
|
||||||
assert(v > lastWriteVersion);
|
|
||||||
lastWriteVersion = v;
|
|
||||||
cs.addWrites(writeRanges.data(), writeRanges.size(), v);
|
|
||||||
ref.addWrites(writeRanges.data(), writeRanges.size(), v);
|
|
||||||
writeRanges = {};
|
|
||||||
} else if (line.starts_with("setoldest"_v)) {
|
|
||||||
printf("setoldest\n");
|
|
||||||
assert(v > lastOldestVersion);
|
|
||||||
lastOldestVersion = v;
|
|
||||||
cs.setOldestVersion(v);
|
|
||||||
ref.setOldestVersion(v);
|
|
||||||
} else if (line.empty() || line.starts_with(";"_v)) {
|
|
||||||
// skip
|
|
||||||
} else {
|
|
||||||
printf("Unrecognized line: %.*s\n", int(line.size()), line.data());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
munmap((void *)mapOriginal, sizeOriginal);
|
|
||||||
close(fd);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+38
-4
@@ -16,6 +16,8 @@
|
|||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
* See the License for the specific language governing permissions and
|
* See the License for the specific language governing permissions and
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
|
*
|
||||||
|
* This source code is modified to compile outside of FoundationDB
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include "ConflictSet.h"
|
#include "ConflictSet.h"
|
||||||
@@ -268,13 +270,21 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
explicit SkipList(Version version = 0) {
|
explicit SkipList(Version version = 0) {
|
||||||
|
#if DEBUG_VERBOSE
|
||||||
|
fprintf(stderr, "skip_list: create\n");
|
||||||
|
#endif
|
||||||
header = Node::create(StringRef(), MaxLevels - 1);
|
header = Node::create(StringRef(), MaxLevels - 1);
|
||||||
for (int l = 0; l < MaxLevels; l++) {
|
for (int l = 0; l < MaxLevels; l++) {
|
||||||
header->setNext(l, nullptr);
|
header->setNext(l, nullptr);
|
||||||
header->setMaxVersion(l, version);
|
header->setMaxVersion(l, version);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
~SkipList() { destroy(); }
|
~SkipList() {
|
||||||
|
#if DEBUG_VERBOSE
|
||||||
|
fprintf(stderr, "skip_list: destroy\n");
|
||||||
|
#endif
|
||||||
|
destroy();
|
||||||
|
}
|
||||||
SkipList(SkipList &&other) noexcept : header(other.header) {
|
SkipList(SkipList &&other) noexcept : header(other.header) {
|
||||||
other.header = nullptr;
|
other.header = nullptr;
|
||||||
}
|
}
|
||||||
@@ -603,6 +613,10 @@ struct __attribute__((visibility("hidden"))) ConflictSet::Impl {
|
|||||||
for (int s = stripes - 1; s >= 0; s--) {
|
for (int s = stripes - 1; s >= 0; s--) {
|
||||||
for (int i = 0; i * 2 < ss; ++i) {
|
for (int i = 0; i * 2 < ss; ++i) {
|
||||||
const auto &w = writes[s * stripeSize / 2 + i];
|
const auto &w = writes[s * stripeSize / 2 + i];
|
||||||
|
#if DEBUG_VERBOSE
|
||||||
|
printf("Write begin: %s\n", printable(w.begin).c_str());
|
||||||
|
fflush(stdout);
|
||||||
|
#endif
|
||||||
values[i * 2] = {w.begin.p, size_t(w.begin.len)};
|
values[i * 2] = {w.begin.p, size_t(w.begin.len)};
|
||||||
values[i * 2 + 1] = w.end.len > 0
|
values[i * 2 + 1] = w.end.len > 0
|
||||||
? StringRef{w.end.p, size_t(w.end.len)}
|
? StringRef{w.end.p, size_t(w.end.len)}
|
||||||
@@ -702,16 +716,35 @@ ConflictSet_check(void *cs, const ConflictSet_ReadRange *reads,
|
|||||||
__attribute__((__visibility__("default"))) void
|
__attribute__((__visibility__("default"))) void
|
||||||
ConflictSet_addWrites(void *cs, const ConflictSet_WriteRange *writes, int count,
|
ConflictSet_addWrites(void *cs, const ConflictSet_WriteRange *writes, int count,
|
||||||
int64_t writeVersion) {
|
int64_t writeVersion) {
|
||||||
((ConflictSet::Impl *)cs)->addWrites(writes, count, writeVersion);
|
auto *impl = (ConflictSet::Impl *)cs;
|
||||||
|
mallocBytesDelta = 0;
|
||||||
|
impl->addWrites(writes, count, writeVersion);
|
||||||
|
impl->totalBytes += mallocBytesDelta;
|
||||||
|
#if SHOW_MEMORY
|
||||||
|
if (impl->totalBytes != mallocBytes) {
|
||||||
|
abort();
|
||||||
|
}
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
__attribute__((__visibility__("default"))) void
|
__attribute__((__visibility__("default"))) void
|
||||||
ConflictSet_setOldestVersion(void *cs, int64_t oldestVersion) {
|
ConflictSet_setOldestVersion(void *cs, int64_t oldestVersion) {
|
||||||
((ConflictSet::Impl *)cs)->setOldestVersion(oldestVersion);
|
auto *impl = (ConflictSet::Impl *)cs;
|
||||||
|
mallocBytesDelta = 0;
|
||||||
|
impl->setOldestVersion(oldestVersion);
|
||||||
|
impl->totalBytes += mallocBytesDelta;
|
||||||
|
#if SHOW_MEMORY
|
||||||
|
if (impl->totalBytes != mallocBytes) {
|
||||||
|
abort();
|
||||||
|
}
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
__attribute__((__visibility__("default"))) void *
|
__attribute__((__visibility__("default"))) void *
|
||||||
ConflictSet_create(int64_t oldestVersion) {
|
ConflictSet_create(int64_t oldestVersion) {
|
||||||
return new (safe_malloc(sizeof(ConflictSet::Impl)))
|
mallocBytesDelta = 0;
|
||||||
|
auto *result = new (safe_malloc(sizeof(ConflictSet::Impl)))
|
||||||
ConflictSet::Impl{oldestVersion};
|
ConflictSet::Impl{oldestVersion};
|
||||||
|
result->totalBytes += mallocBytesDelta;
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
__attribute__((__visibility__("default"))) void ConflictSet_destroy(void *cs) {
|
__attribute__((__visibility__("default"))) void ConflictSet_destroy(void *cs) {
|
||||||
using Impl = ConflictSet::Impl;
|
using Impl = ConflictSet::Impl;
|
||||||
@@ -728,6 +761,7 @@ ConflictSet_getBytes(void *cs) {
|
|||||||
#if SHOW_MEMORY
|
#if SHOW_MEMORY
|
||||||
struct __attribute__((visibility("default"))) PeakPrinter {
|
struct __attribute__((visibility("default"))) PeakPrinter {
|
||||||
~PeakPrinter() {
|
~PeakPrinter() {
|
||||||
|
printf("--- skip_list ---\n");
|
||||||
printf("malloc bytes: %g\n", double(mallocBytes));
|
printf("malloc bytes: %g\n", double(mallocBytes));
|
||||||
printf("Peak malloc bytes: %g\n", double(peakMallocBytes));
|
printf("Peak malloc bytes: %g\n", double(peakMallocBytes));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,4 +4,5 @@ _bzero
|
|||||||
_free
|
_free
|
||||||
_malloc
|
_malloc
|
||||||
_memcpy
|
_memcpy
|
||||||
_memmove
|
_memmove
|
||||||
|
dyld_stub_binder
|
||||||
+73
-57
@@ -4,22 +4,6 @@ import os
|
|||||||
|
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
_lib = None
|
|
||||||
for f in (
|
|
||||||
os.path.dirname(__file__) + "/build/radix_tree/libconflict-set.so.0",
|
|
||||||
os.path.dirname(__file__) + "/build/radix_tree/libconflict-set.0.dylib",
|
|
||||||
):
|
|
||||||
try:
|
|
||||||
_lib = ctypes.cdll.LoadLibrary(f)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
if _lib is None:
|
|
||||||
import sys
|
|
||||||
|
|
||||||
print("Could not find libconflict-set", file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
class _Key(ctypes.Structure):
|
class _Key(ctypes.Structure):
|
||||||
_fields_ = [("p", ctypes.POINTER(ctypes.c_ubyte)), ("len", ctypes.c_int)]
|
_fields_ = [("p", ctypes.POINTER(ctypes.c_ubyte)), ("len", ctypes.c_int)]
|
||||||
@@ -37,28 +21,6 @@ class WriteRange(ctypes.Structure):
|
|||||||
_fields_ = [("begin", _Key), ("end", _Key)]
|
_fields_ = [("begin", _Key), ("end", _Key)]
|
||||||
|
|
||||||
|
|
||||||
_lib.ConflictSet_create.argtypes = (ctypes.c_int64,)
|
|
||||||
_lib.ConflictSet_create.restype = ctypes.c_void_p
|
|
||||||
|
|
||||||
_lib.ConflictSet_check.argtypes = (
|
|
||||||
ctypes.c_void_p,
|
|
||||||
ctypes.POINTER(ReadRange),
|
|
||||||
ctypes.POINTER(ctypes.c_int),
|
|
||||||
ctypes.c_int,
|
|
||||||
)
|
|
||||||
|
|
||||||
_lib.ConflictSet_addWrites.argtypes = (
|
|
||||||
ctypes.c_void_p,
|
|
||||||
ctypes.POINTER(WriteRange),
|
|
||||||
ctypes.c_int,
|
|
||||||
ctypes.c_int64,
|
|
||||||
)
|
|
||||||
|
|
||||||
_lib.ConflictSet_setOldestVersion.argtypes = (ctypes.c_void_p, ctypes.c_int64)
|
|
||||||
|
|
||||||
_lib.ConflictSet_destroy.argtypes = (ctypes.c_void_p,)
|
|
||||||
|
|
||||||
|
|
||||||
class Result(enum.Enum):
|
class Result(enum.Enum):
|
||||||
COMMIT = 0
|
COMMIT = 0
|
||||||
CONFLICT = 1
|
CONFLICT = 1
|
||||||
@@ -66,55 +28,109 @@ class Result(enum.Enum):
|
|||||||
|
|
||||||
|
|
||||||
def write(begin: bytes, end: Optional[bytes] = None) -> WriteRange:
|
def write(begin: bytes, end: Optional[bytes] = None) -> WriteRange:
|
||||||
b = (ctypes.c_ubyte * len(begin))()
|
b = (ctypes.c_ubyte * len(begin)).from_buffer(bytearray(begin))
|
||||||
b.value = begin
|
|
||||||
if end is None:
|
if end is None:
|
||||||
e = (ctypes.c_ubyte * 0)()
|
e = (ctypes.c_ubyte * 0)()
|
||||||
e.value = b""
|
|
||||||
else:
|
else:
|
||||||
e = (ctypes.c_ubyte * len(end))()
|
e = (ctypes.c_ubyte * len(end)).from_buffer(bytearray(end))
|
||||||
e.value = end
|
|
||||||
return WriteRange(_Key(b, len(b)), _Key(e, len(e)))
|
return WriteRange(_Key(b, len(b)), _Key(e, len(e)))
|
||||||
|
|
||||||
|
|
||||||
def read(version: int, begin: bytes, end: Optional[bytes] = None) -> ReadRange:
|
def read(version: int, begin: bytes, end: Optional[bytes] = None) -> ReadRange:
|
||||||
b = (ctypes.c_ubyte * len(begin))()
|
b = (ctypes.c_ubyte * len(begin)).from_buffer(bytearray(begin))
|
||||||
b.value = begin
|
|
||||||
if end is None:
|
if end is None:
|
||||||
e = (ctypes.c_ubyte * 0)()
|
e = (ctypes.c_ubyte * 0)()
|
||||||
e.value = b""
|
|
||||||
else:
|
else:
|
||||||
e = (ctypes.c_ubyte * len(end))()
|
e = (ctypes.c_ubyte * len(end)).from_buffer(bytearray(end))
|
||||||
e.value = end
|
|
||||||
return ReadRange(_Key(b, len(b)), _Key(e, len(e)), version)
|
return ReadRange(_Key(b, len(b)), _Key(e, len(e)), version)
|
||||||
|
|
||||||
|
|
||||||
class ConflictSet:
|
class ConflictSet:
|
||||||
def __init__(self, version: int = 0) -> None:
|
def __init__(
|
||||||
self.p = _lib.ConflictSet_create(version)
|
self,
|
||||||
|
version: int = 0,
|
||||||
|
build_dir: Optional[str] = None,
|
||||||
|
implementation: Optional[str] = None,
|
||||||
|
) -> None:
|
||||||
|
self._lib = None
|
||||||
|
if build_dir is None:
|
||||||
|
build_dir = os.path.dirname(__file__) + "/build"
|
||||||
|
if implementation is None:
|
||||||
|
implementation = "radix_tree"
|
||||||
|
for f in (
|
||||||
|
build_dir + "/" + implementation + "/libconflict-set.so.0",
|
||||||
|
os.path.dirname(__file__)
|
||||||
|
+ "/build/"
|
||||||
|
+ implementation
|
||||||
|
+ "/libconflict-set.0.dylib",
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
self._lib = ctypes.cdll.LoadLibrary(f)
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if self._lib is None:
|
||||||
|
import sys
|
||||||
|
|
||||||
|
print(
|
||||||
|
"Could not find libconflict-set implementation " + implementation,
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
self._lib.ConflictSet_create.argtypes = (ctypes.c_int64,)
|
||||||
|
self._lib.ConflictSet_create.restype = ctypes.c_void_p
|
||||||
|
|
||||||
|
self._lib.ConflictSet_check.argtypes = (
|
||||||
|
ctypes.c_void_p,
|
||||||
|
ctypes.POINTER(ReadRange),
|
||||||
|
ctypes.POINTER(ctypes.c_int),
|
||||||
|
ctypes.c_int,
|
||||||
|
)
|
||||||
|
|
||||||
|
self._lib.ConflictSet_addWrites.argtypes = (
|
||||||
|
ctypes.c_void_p,
|
||||||
|
ctypes.POINTER(WriteRange),
|
||||||
|
ctypes.c_int,
|
||||||
|
ctypes.c_int64,
|
||||||
|
)
|
||||||
|
|
||||||
|
self._lib.ConflictSet_setOldestVersion.argtypes = (
|
||||||
|
ctypes.c_void_p,
|
||||||
|
ctypes.c_int64,
|
||||||
|
)
|
||||||
|
|
||||||
|
self._lib.ConflictSet_destroy.argtypes = (ctypes.c_void_p,)
|
||||||
|
|
||||||
|
self._lib.ConflictSet_getBytes.argtypes = (ctypes.c_void_p,)
|
||||||
|
self._lib.ConflictSet_getBytes.restype = ctypes.c_int64
|
||||||
|
|
||||||
|
self.p = self._lib.ConflictSet_create(version)
|
||||||
|
|
||||||
def addWrites(self, version: int, *writes: WriteRange):
|
def addWrites(self, version: int, *writes: WriteRange):
|
||||||
_lib.ConflictSet_addWrites(
|
self._lib.ConflictSet_addWrites(
|
||||||
self.p, (WriteRange * len(writes))(*writes), len(writes), version
|
self.p, (WriteRange * len(writes))(*writes), len(writes), version
|
||||||
)
|
)
|
||||||
|
|
||||||
def check(self, *reads: ReadRange) -> list[Result]:
|
def check(self, *reads: ReadRange) -> list[Result]:
|
||||||
r = (ctypes.c_int * len(reads))()
|
r = (ctypes.c_int * len(reads))()
|
||||||
_lib.ConflictSet_check(self.p, *reads, r, 1)
|
self._lib.ConflictSet_check(self.p, *reads, r, 1)
|
||||||
return [Result(x) for x in r]
|
return [Result(x) for x in r]
|
||||||
|
|
||||||
def setOldestVersion(self, version: int) -> None:
|
def setOldestVersion(self, version: int) -> None:
|
||||||
_lib.ConflictSet_setOldestVersion(self.p, version)
|
self._lib.ConflictSet_setOldestVersion(self.p, version)
|
||||||
|
|
||||||
|
def getBytes(self) -> int:
|
||||||
|
return self._lib.ConflictSet_getBytes(self.p)
|
||||||
|
|
||||||
def __enter__(self):
|
def __enter__(self):
|
||||||
return self
|
return self
|
||||||
|
|
||||||
def close(self) -> None:
|
def close(self) -> None:
|
||||||
if self.p is not None:
|
if self.p is not None:
|
||||||
_lib.ConflictSet_destroy(self.p)
|
self._lib.ConflictSet_destroy(self.p)
|
||||||
self.p = None
|
self.p = None
|
||||||
|
|
||||||
def __exit__(self, exception_type, exception_value, exception_traceback):
|
def __exit__(self, exception_type, exception_value, exception_traceback):
|
||||||
if self.p is not None:
|
self.close()
|
||||||
_lib.ConflictSet_destroy(self.p)
|
|
||||||
self.p = None
|
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user