diff --git a/README.md b/README.md index e5cf66b..f33cc0c 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ proof. search material; those sections are not the route taken by the final closed-form proof. * The C++ tree is validated against Lean-generated golden vectors, by - differential fuzzing, and by a deterministic exhaustive sweep of small + differential fuzzing, and by a deterministic finite sweep of small readable instances; `cpp/README.md` describes the model, the tests, the coverage check, and the benchmark. The summarised implementation has 100% line and branch coverage via `cpp/run-coverage.sh`. diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 725941d..d12c3e3 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -36,18 +36,21 @@ target_include_directories(extreme_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests) target_link_libraries(extreme_test PRIVATE cr_literal cr_fast) -# Deterministic exhaustive sweep of small readable instances. This is the -# driver for the line-coverage check in `run-coverage.sh`. -add_executable(exhaustive_test tests/exhaustive.cpp) -target_include_directories(exhaustive_test PRIVATE +# Deterministic sweep over a finite class of small readable instances. This is +# the driver for the line-coverage check in `run-coverage.sh`. +add_executable(sweep_test tests/sweep.cpp) +target_include_directories(sweep_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests) -target_link_libraries(exhaustive_test PRIVATE cr_literal cr_fast) +target_link_libraries(sweep_test PRIVATE cr_literal cr_fast) set(CR_GOLDEN_VECTORS "${CMAKE_CURRENT_SOURCE_DIR}/oracle/golden.txt") add_test(NAME golden COMMAND golden_test "${CR_GOLDEN_VECTORS}") add_test(NAME diff_corpus COMMAND diff_corpus "${CR_GOLDEN_VECTORS}") +# Curated readable cases found by SMT-assisted mutation analysis. +add_test(NAME smt_cases COMMAND diff_corpus + "${CMAKE_CURRENT_SOURCE_DIR}/tests/smt_cases.txt") add_test(NAME extreme COMMAND extreme_test) -add_test(NAME exhaustive COMMAND exhaustive_test) +add_test(NAME sweep COMMAND sweep_test) if(CR_BUILD_BENCH) add_executable(cr_bench tests/bench.cpp) diff --git a/cpp/README.md b/cpp/README.md index 62c912d..0612632 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -16,10 +16,12 @@ The Lean build is untouched. | `oracle/gen.lean` | Deterministic generator of golden vectors by evaluating the real Lean `closedForm`. | | `tests/golden.cpp` | Checks the literal translation against the Lean golden vectors. | | `tests/diff_corpus.cpp` | Compares literal and fast on the sorted subset of the golden corpus. | +| `tests/smt_cases.txt` | Curated readable cases found by SMT-assisted mutation analysis; run by the `smt_cases` CTest. | | `tests/readable_gen.h` | Readable-by-construction instance generator (owns the selector keys and reported keys, with borrowed views into them). | | `tests/extreme.cpp` | Regression test for offsets at the ends of the 32-bit input range. | -| `tests/exhaustive.cpp` | Deterministic exhaustive sweep of small readable instances (also the coverage driver). | +| `tests/sweep.cpp` | Deterministic sweep over a finite class of small readable instances (also the coverage driver). | | `tests/fuzz_readable.cpp` | libFuzzer differential test: literal vs. fast on readable instances. | +| `tests/corpus/readable/` | Committed read-only seeds for `fuzz_readable`. | | `tests/fuzz_raw.cpp` | libFuzzer safety test on arbitrary instances under ASan/UBSan. | | `tests/bench.cpp` | nanobench literal-vs-fast benchmark. | | `run-fuzz.sh` | Rebuilds and runs both fuzzers for a fixed time. | @@ -50,7 +52,7 @@ integer and reject values outside the 32-bit range before narrowing to and `golden_test` compares it with Lean over the whole generated corpus, most of which is not readable. `fast.cpp` is the readable-only implementation: `diff_corpus` filters the corpus to readable cases, and the fuzzers and the -exhaustive sweep generate readable instances. The Lean anchor therefore covers +finite sweep generate readable instances. The Lean anchor therefore covers `literal` on all inputs, while `fast` is anchored to `literal` on the readable subset. @@ -69,7 +71,8 @@ ctest --test-dir build --output-on-failure ``` `golden` and `diff_corpus` read `oracle/golden.txt`; regenerate it first if it -is missing (see below). +is missing (see below). `smt_cases` reads the committed +`tests/smt_cases.txt` and needs no generated input. ## Golden vectors from Lean @@ -98,20 +101,21 @@ cases that satisfy the readable sorting invariant: # extreme: 7574 cases, 0 mismatches ``` -`exhaustive_test` enumerates a specific finite class rather than sampling it: -every sorted database of at most four stored keys drawn from a fixed universe of +`sweep_test` is exhaustive over a specific bounded class rather than sampling +it: every sorted database of at most four stored keys drawn from a fixed universe of boundary keys, every selector key (including `⊥` and `⊤`) with offsets in `[-3, 5]`, both traversal directions, and every readable result shape (the complete traversal, each nonempty truncated prefix, or the empty result when the traversal is empty): ```sh -./cpp/build/exhaustive_test -# exhaustive: 13565772 cases, 0 mismatches +./cpp/build/sweep_test +# sweep: 13565772 cases, 0 mismatches ``` -It runs in about a second and is also the coverage driver. The class is not all -readable inputs; the guarantee is that every instance in this family agrees. In +It runs in about a second and is also the coverage driver. It is exhaustive +only within those bounds; the class is not all readable inputs, and the +guarantee is that every instance in this family agrees. In particular, branch coverage cannot exercise a predicate operand that the implementation no longer contains: the coverage result below describes the code as written, it is not a proof of equivalence with `literal.cpp`. @@ -123,8 +127,8 @@ as written, it is not a proof of equivalence with `literal.cpp`. ``` The script configures a Clang source-based-coverage build, runs -`exhaustive_test`, and fails unless `src/fast.cpp` has 100% line and branch -coverage. `exhaustive_test` alone covers every region, function, line, and +`sweep_test`, and fails unless `src/fast.cpp` has 100% line and branch +coverage. `sweep_test` alone covers every region, function, line, and branch side of the summarised implementation: ```text @@ -149,11 +153,13 @@ translation can still be audited line by line. ```sh cd cpp mkdir -p /tmp/cr-corpus/readable /tmp/cr-corpus/raw -./build/fuzz_readable -max_total_time=300 /tmp/cr-corpus/readable +./build/fuzz_readable -max_total_time=300 /tmp/cr-corpus/readable tests/corpus/readable ./build/fuzz_raw -max_total_time=300 /tmp/cr-corpus/raw ``` or simply `./cpp/run-fuzz.sh` (optionally `./cpp/run-fuzz.sh build 600`). +`tests/corpus/readable` holds committed seeds; the first directory above is the +writable corpus where libFuzzer stores newly discovered units. The CMake fuzz targets compile the implementation sources directly so that the decoder code, not just the harness, is covered by ASan/UBSan. @@ -173,7 +179,7 @@ One representative validation run on this machine: | literal vs. Lean golden vectors | 40,000 | 0 mismatches (exact endpoints) | | fast vs. literal on sorted corpus | 18,971 | 0 mismatches | | extreme-offset regression (`extreme_test`) | 7,574 | 0 mismatches | -| exhaustive readable sweep (`exhaustive_test`) | 13,565,772 | 0 mismatches | +| finite readable sweep (`sweep_test`) | 13,565,772 | 0 mismatches | | readable differential fuzzing (60 s) | 1,182,424 | 0 mismatches | | raw safety fuzzing (60 s) | 1,905,270 | no ASan/UBSan findings | diff --git a/cpp/run-coverage.sh b/cpp/run-coverage.sh index 40f253b..e430491 100755 --- a/cpp/run-coverage.sh +++ b/cpp/run-coverage.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Build with Clang source-based coverage and require 100% line and branch -# coverage of the summarised implementation (`src/fast.cpp`). `exhaustive_test` -# enumerates small readable instances deterministically and is sufficient on its -# own. +# coverage of the summarised implementation (`src/fast.cpp`). `sweep_test` +# deterministically enumerates a finite class of small readable instances and is +# sufficient on its own. set -euo pipefail here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -74,18 +74,18 @@ cmake -S "$here" -B "$build" -G Ninja \ -DCR_BUILD_BENCH=OFF \ -DCMAKE_CXX_FLAGS="-fprofile-instr-generate -fcoverage-mapping" \ -DCMAKE_EXE_LINKER_FLAGS="-fprofile-instr-generate" -cmake --build "$build" --target exhaustive_test +cmake --build "$build" --target sweep_test profdir="$build/prof" rm -rf "$profdir" mkdir -p "$profdir" cd "$here" -LLVM_PROFILE_FILE="$profdir/exhaustive-%p.profraw" "$build/exhaustive_test" +LLVM_PROFILE_FILE="$profdir/sweep-%p.profraw" "$build/sweep_test" "$profdata" merge -sparse "$profdir"/*.profraw -o "$profdir/cov.profdata" report() { - "$cov" report "$build/exhaustive_test" \ + "$cov" report "$build/sweep_test" \ -instr-profile="$profdir/cov.profdata" src/fast.cpp } output="$(report)" diff --git a/cpp/run-fuzz.sh b/cpp/run-fuzz.sh index 49eb39b..da8737c 100755 --- a/cpp/run-fuzz.sh +++ b/cpp/run-fuzz.sh @@ -15,9 +15,13 @@ cmake --build "$build" --target fuzz_readable fuzz_raw corpus="${TMPDIR:-/tmp}/cr-fuzz-corpus" mkdir -p "$corpus/readable" "$corpus/raw" +# Committed read-only seeds (cases found by SMT/mutation analysis) are +# passed after the writable corpus, which is where libFuzzer stores new units. +seeds="$here/tests/corpus/readable" echo "== differential fuzzing (readable instances) ==" -"$build/fuzz_readable" -max_total_time="$seconds" -max_len=1024 "$corpus/readable" +"$build/fuzz_readable" -max_total_time="$seconds" -max_len=1024 \ + "$corpus/readable" "$seeds" echo "== safety fuzzing (arbitrary instances) ==" "$build/fuzz_raw" -max_total_time="$seconds" -max_len=1024 "$corpus/raw" diff --git a/cpp/src/fast.cpp b/cpp/src/fast.cpp index 9e9d930..61aecef 100644 --- a/cpp/src/fast.cpp +++ b/cpp/src/fast.cpp @@ -703,17 +703,11 @@ struct Impl { return c.r->end_.offset >= 2 && allBelowEnd(c); } - bool zeroCrossOvershootComplete(const Ctx& c, const Obs& o) const { - return c.r->begin.offset == 0 && overshoot(c); - } - - bool negCrossOvershoot(const Ctx& c, const Obs& o) const { - return c.r->begin.offset < 0 && overshoot(c); - } - - bool crossedCompleteOvershoot(const Ctx& c, const Obs& o) const { - return zeroCrossOvershootComplete(c, o) || negCrossOvershoot(c, o); - } + // `closedFormCrossedComplete` is reached only from `closedFormCrossed`, + // which the entry point enters only when `begin.offset <= 0`; the + // `begin.offset >= 1` cases go to the positive subtree. So the mode does + // not need to be retested here. + bool crossedCompleteOvershoot(const Ctx& c) const { return overshoot(c); } bool crossedCompleteEndLeOne(const Ctx& c) const { if (c.r->end_.offset == 1 && bSlack(c) < c.below) @@ -722,8 +716,8 @@ struct Impl { !(c.below <= bSlack(c) && clampNonposBottom(c)); } - // The three `zeroCross*` predicates above cover every `begin.offset == 0` - // case of the complete-crossed subtree, so the mode alone decides here. + // The overshoot branch above already covered the offset-zero overshoot + // cases, so the mode alone decides here. bool crossedCompleteBeginZero(const Ctx& c, const Obs& o) const { return c.r->begin.offset == 0; } @@ -735,7 +729,7 @@ struct Impl { } ConflictRange closedFormCrossedComplete(const Ctx& c, const Obs& o) const { - if (crossedCompleteOvershoot(c, o)) + if (crossedCompleteOvershoot(c)) return closedFormNegCrossTopUnified(c, o); if (crossedCompleteEndLeOne(c)) return closedFormNegCrossNonclampUnified(c, o); @@ -851,7 +845,10 @@ struct Impl { } bool closedFormForwardSaturated(const Ctx& c, const Obs& o) const { - if (!(rE(c) > 0 && oLtK(c.r->end_.key, o.firstK()))) return false; + // The caller has `end.offset > 0`. If `end.offset == 1` then `rE == 0`, + // and readability makes `end.key < firstKey` false, so the `rE > 0` + // guard adds nothing. + if (!oLtK(c.r->end_.key, o.firstK())) return false; NatInf rhs = niAdd(cardOK(c.r->end_.key, o.firstK()), NatInf::of(static_cast(c.n))); return niLt(NatInf::of(rE(c)), rhs); diff --git a/cpp/tests/corpus/readable/cardko-empty-interval b/cpp/tests/corpus/readable/cardko-empty-interval new file mode 100644 index 0000000..ab342e8 Binary files /dev/null and b/cpp/tests/corpus/readable/cardko-empty-interval differ diff --git a/cpp/tests/smt_cases.txt b/cpp/tests/smt_cases.txt new file mode 100644 index 0000000..8d1c14c --- /dev/null +++ b/cpp/tests/smt_cases.txt @@ -0,0 +1,10 @@ +# Hand-picked readable instances that the deterministic sweep and the +# Lean-generated corpus did not cover. Format matches `oracle/golden.txt` and +# is consumed by `diff_corpus`. +# +# z3 model for the `cardKO` empty-interval mutation (`NatInf::of(0)` -> +# `NatInf::of(1)` in `fast.cpp`): a reversed, truncated read whose reported +# prefix contains keys at or above `begin.key`. The mutated cardinality flips +# `finiteUnderCond` and moves the upper endpoint from `after(firstKey)` to +# `begin.key`. +610100 -1 610080 3 1 1 3 6102 6101 61 => 61 610200 diff --git a/cpp/tests/exhaustive.cpp b/cpp/tests/sweep.cpp similarity index 85% rename from cpp/tests/exhaustive.cpp rename to cpp/tests/sweep.cpp index 1cf75ef..d228dfa 100644 --- a/cpp/tests/exhaustive.cpp +++ b/cpp/tests/sweep.cpp @@ -1,13 +1,15 @@ // Released under Apache 2.0 license as described in the file LICENSE. -// A deterministic exhaustive sweep of small readable instances, comparing the -// literal translation with the summarised implementation. +// A deterministic sweep over a finite class of small readable instances, +// comparing the literal translation with the summarised implementation. // -// The fuzzers sample readable instances; this test enumerates them: every -// sorted database (up to four keys) over a fixed universe of interesting keys, -// every selector key (including ⊥ and ⊤) paired with offsets in [-3, 5], and -// every readable result (the complete traversal, each nonempty truncated -// prefix, or the empty result when the traversal is empty) in both directions. +// It is exhaustive over this bounded family, but not over the input space. +// The fuzzers sample readable instances; this test enumerates a fixed finite +// family: every sorted database (up to four keys) over a fixed universe of +// interesting keys, every selector key (including ⊥ and ⊤) paired with offsets +// in [-3, 5], and every readable result (the complete traversal, each nonempty +// truncated prefix, or the empty result when the traversal is empty) in both +// directions. // // Together with `golden_test`, `diff_corpus`, and `extreme_test`, this covers // every line and branch side of `src/fast.cpp`. @@ -110,6 +112,6 @@ int main() { run(db, keys, cases, bad); } - std::printf("exhaustive: %lld cases, %lld mismatches\n", cases, bad); + std::printf("sweep: %lld cases, %lld mismatches\n", cases, bad); return bad == 0 ? 0 : 1; }