weaselbot
dee3a8f640
CI / release (arm64, ubuntu-latest-arm64) (pull_request) Successful in 3m25s
CI / pre-commit (pull_request) Successful in 3m40s
CI / test (-DCMAKE_BUILD_TYPE=Debug, debug) (pull_request) Successful in 5m8s
CI / test (-DCMAKE_CXX_FLAGS=-DUSE_64_BIT=1, 64-bit-versions) (pull_request) Successful in 4m50s
CI / test (-DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++, gcc) (pull_request) Successful in 4m57s
CI / test (-DUSE_SIMD_FALLBACK=ON, simd-fallback) (pull_request) Successful in 4m42s
CI / release (amd64, ubuntu-latest-amd64) (pull_request) Successful in 6m31s
CI / coverage (pull_request) Successful in 4m51s
The hash_table implementation cannot accurately track memory usage and was returning -1 from ConflictSet::getBytes() and ConflictSet_getBytes(). That violates the API contract that getBytes() returns a non-negative byte count. Change both entry points to return 0 and document in ConflictSet.h that implementations which do not track memory usage may return 0. Add a regression test in test_conflict_set.py that loads the hash_table implementation and verifies getBytes() is non-negative. Closes #62
253 lines
8.2 KiB
Python
253 lines
8.2 KiB
Python
import struct
|
|
from conflict_set import *
|
|
|
|
|
|
build_dir = None
|
|
|
|
|
|
class DebugConflictSet:
|
|
"""
|
|
Bisimulates the skip list and radix tree conflict sets for testing purposes
|
|
"""
|
|
|
|
def __init__(self, version: int = 0) -> None:
|
|
self.skip_list = ConflictSet(
|
|
version, build_dir=build_dir, implementation="skip_list"
|
|
)
|
|
self.radix_tree = ConflictSet(
|
|
version, build_dir=build_dir, implementation="radix_tree"
|
|
)
|
|
|
|
def addWrites(self, version: int, *writes: WriteRange):
|
|
self.skip_list.addWrites(version, *writes)
|
|
self.radix_tree.addWrites(version, *writes)
|
|
|
|
def check(self, *reads: ReadRange) -> list[Result]:
|
|
expected = self.skip_list.check(*reads)
|
|
actual = self.radix_tree.check(*reads)
|
|
assert expected == actual
|
|
return actual
|
|
|
|
def setOldestVersion(self, version: int) -> None:
|
|
self.skip_list.setOldestVersion(version)
|
|
self.radix_tree.setOldestVersion(version)
|
|
|
|
def getBytes(self) -> int:
|
|
return self.radix_tree.getBytes()
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def close(self) -> None:
|
|
self.skip_list.close()
|
|
self.radix_tree.close()
|
|
|
|
def __exit__(self, exception_type, exception_value, exception_traceback):
|
|
self.close()
|
|
|
|
|
|
def test_conflict_set():
|
|
with DebugConflictSet() as cs:
|
|
before = cs.getBytes()
|
|
key = b"a key"
|
|
cs.addWrites(1, write(key))
|
|
assert cs.getBytes() - before > 0
|
|
assert cs.check(read(0, key)) == [Result.CONFLICT]
|
|
cs.setOldestVersion(1)
|
|
assert cs.check(read(0, key), read(1, key)) == [Result.TOO_OLD, Result.COMMIT]
|
|
|
|
|
|
def test_hash_table_getBytes():
|
|
# Regression test for issue #62: the hash_table implementation is
|
|
# point-query only and does not track memory usage, but getBytes() must
|
|
# still return a non-negative value rather than -1.
|
|
with ConflictSet(0, build_dir=build_dir, implementation="hash_table") as cs:
|
|
assert cs.getBytes() == 0
|
|
cs.addWrites(1, write(b"key"))
|
|
assert cs.getBytes() >= 0
|
|
assert cs.check(read(0, b"key")) == [Result.CONFLICT]
|
|
|
|
|
|
def test_write_read_without_outer_reference():
|
|
# Regression test for issue #42: WriteRange/ReadRange must keep their
|
|
# backing key buffers alive, because the C library reads the pointer
|
|
# stored in _Key while addWrites/check run.
|
|
with DebugConflictSet() as cs:
|
|
# The bytes literal is not referenced after this expression.
|
|
cs.addWrites(1, write(b"key"))
|
|
assert cs.check(read(0, b"key")) == [Result.CONFLICT]
|
|
|
|
cs.addWrites(2, write(b"a", b"z"))
|
|
assert cs.check(read(1, b"a", b"z")) == [Result.CONFLICT]
|
|
assert cs.check(read(1, b"b")) == [Result.CONFLICT]
|
|
assert cs.check(read(1, b"0")) == [Result.COMMIT]
|
|
|
|
|
|
def test_range_keeps_key_buffers_alive():
|
|
# Verify the fix for issue #42: returned range objects must retain a
|
|
# reference to the backing bytearray so the C pointer stays valid after
|
|
# the helper returns.
|
|
w = write(b"key")
|
|
assert w._begin_buf == bytearray(b"key")
|
|
assert w._end_buf is None
|
|
|
|
w2 = write(b"a", b"z")
|
|
assert w2._begin_buf == bytearray(b"a")
|
|
assert w2._end_buf == bytearray(b"z")
|
|
|
|
r = read(0, b"key")
|
|
assert r._begin_buf == bytearray(b"key")
|
|
assert r._end_buf is None
|
|
|
|
r2 = read(1, b"a", b"z")
|
|
assert r2._begin_buf == bytearray(b"a")
|
|
assert r2._end_buf == bytearray(b"z")
|
|
|
|
|
|
def test_update_zero_should_commit():
|
|
with DebugConflictSet() as cs1:
|
|
with DebugConflictSet() as cs2:
|
|
cs1.addWrites(2, write(b""))
|
|
cs1.setOldestVersion(2)
|
|
# "zero" is now 2
|
|
|
|
# make a Node48
|
|
for i in range(256 - 17, 256):
|
|
cs2.addWrites(int(1), write(bytes([i])))
|
|
# Scan until first point write
|
|
assert cs2.check(read(0, b"\x00", bytes([256 - 17]))) == [Result.COMMIT]
|
|
|
|
|
|
def test_update_zero_should_conflict():
|
|
with DebugConflictSet() as cs1:
|
|
with DebugConflictSet() as cs2:
|
|
cs1.addWrites(2**32)
|
|
cs1.setOldestVersion(2**32)
|
|
cs2.addWrites(2**31 + 100)
|
|
cs2.setOldestVersion(2**31 + 100)
|
|
# "zero" is now 2**31 + 100
|
|
cs1.addWrites(2**32 + 101, write(b"", b"\x02"), write(b"\x01"))
|
|
# rangeVersion of \x01 is now 2**31 + 100 ("max" of (2**31 + 100, 2**32 + 101))
|
|
assert cs1.check(read(2**32 + 1, b"\x00")) == [Result.CONFLICT]
|
|
# but 2**32 + 1 ">" 2**31 + 100 , and it incorrectly commits
|
|
|
|
|
|
def test_inner_full_words():
|
|
with DebugConflictSet() as cs:
|
|
cs.addWrites(1, write(b"\x3f\x61"), write(b"\x81\x61"))
|
|
writes = []
|
|
for i in range(0x40, 0x81):
|
|
writes.append(write(bytes([i, 0x61])))
|
|
cs.addWrites(2, *writes)
|
|
cs.check(read(1, b"\x21", b"\xc2"))
|
|
|
|
|
|
def test_internal_version_zero():
|
|
with DebugConflictSet() as cs:
|
|
cs.addWrites(0xFFFFFFF0)
|
|
cs.setOldestVersion(0xFFFFFFF0)
|
|
for i in range(24):
|
|
cs.addWrites(0xFFFFFFF1, write(bytes([i])))
|
|
for i in range(256 - 25, 256):
|
|
cs.addWrites(0xFFFFFFF1, write(bytes([i])))
|
|
cs.addWrites(0x100000000, write(b"\xff"))
|
|
cs.check(read(0xFFFFFFF1, b"\x00", b"\xff"))
|
|
|
|
|
|
def test_two_billion_versions():
|
|
with DebugConflictSet() as cs:
|
|
cs.addWrites(int(2e9) + 1)
|
|
cs.check(read(0, b"\x00", b"\xff"))
|
|
cs.check(read(1, b"\x00", b"\xff"))
|
|
|
|
|
|
def test_positive_bytes():
|
|
with DebugConflictSet() as cs:
|
|
cs.addWrites(1, write(b"hello"))
|
|
assert cs.getBytes() > 0
|
|
cs.addWrites(1 + 2**32)
|
|
assert cs.getBytes() > 0
|
|
|
|
|
|
def test_decrease_capacity():
|
|
# make a Node48, then a Node256
|
|
for count in (17, 49):
|
|
with DebugConflictSet() as cs:
|
|
for i in range(count):
|
|
cs.addWrites(1, write(bytes(([0] * 99) + [i])))
|
|
# lower its partial key length
|
|
cs.addWrites(2, write(bytes([0] * 98)))
|
|
# create work for setOldestVersion
|
|
for i in range(3, 1000):
|
|
cs.addWrites(i)
|
|
# setOldestVersion should decrease the capacity
|
|
cs.setOldestVersion(1)
|
|
|
|
|
|
def test_large():
|
|
with DebugConflictSet() as cs:
|
|
end = 100000
|
|
for i in range(end):
|
|
cs.addWrites(1, write(i.to_bytes(8, byteorder="big")))
|
|
cs.addWrites(
|
|
2,
|
|
write((0).to_bytes(8, byteorder="big"), (end).to_bytes(8, byteorder="big")),
|
|
)
|
|
|
|
|
|
def test_merge_child_node48():
|
|
with DebugConflictSet() as cs:
|
|
cs.addWrites(1, write(b"\x00" * 9))
|
|
for i in range(17):
|
|
cs.addWrites(1, write(b"\x00" * 10 + bytes([i])))
|
|
cs.addWrites(1, write(b"\x00" * 8, b"\x00" * 10))
|
|
|
|
|
|
def test_fixup_256():
|
|
with DebugConflictSet() as cs:
|
|
cs.addWrites(0, write(bytes([1])))
|
|
for i in range(256):
|
|
cs.addWrites(1, write(bytes([1, i])))
|
|
cs.addWrites(2, write(bytes([0]), bytes([1])))
|
|
cs.check(read(0, bytes([1]), bytes([2])))
|
|
|
|
|
|
def test_large_removal_buffer():
|
|
with DebugConflictSet() as cs:
|
|
for i in range(1000):
|
|
# create extra gc work
|
|
for j in range(100):
|
|
cs.addWrites(1000 + i)
|
|
cs.addWrites(1000 + i, write(struct.pack(">l", i) + bytes([0] * 100000)))
|
|
cs.setOldestVersion(i)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# budget "pytest" for ctest integration without pulling in a dependency. You can of course still use pytest in local development.
|
|
import argparse
|
|
import inspect
|
|
import sys
|
|
|
|
parser = argparse.ArgumentParser()
|
|
subparsers = parser.add_subparsers(dest="command")
|
|
|
|
list_parser = subparsers.add_parser("list")
|
|
test_parser = subparsers.add_parser("test")
|
|
test_parser.add_argument("test")
|
|
test_parser.add_argument("--build-dir")
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.command == "list":
|
|
sys.stdout.write(
|
|
";".join(
|
|
name[5:]
|
|
for name in dir()
|
|
if name.startswith("test_")
|
|
and inspect.isfunction(getattr(sys.modules[__name__], name))
|
|
)
|
|
)
|
|
elif args.command == "test":
|
|
build_dir = args.build_dir
|
|
globals()["test_" + args.test]()
|