From e32e42d933109125773a05d3bf7e7378ea537c36 Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Tue, 17 Nov 2026 16:08:07 -0500 Subject: [PATCH] Add a C++ translation of the closed form - Literal transliteration of `Statement.closedForm` plus an equivalent summarised implementation - Borrowed input keys (`std::string_view` / `std::span`) and owned result keys (`std::string`) - Golden-vector test, differential/safety fuzzers, and a nanobench comparison The literal and summarised implementations agree on 40,000 Lean golden vectors, 18,971 sorted corpus cases, and the readable fuzzer; the raw fuzzer runs both implementations under ASan/UBSan. `build/` and the generated `oracle/golden.txt` are ignored. --- cpp/.gitignore | 2 + cpp/CMakeLists.txt | 59 + cpp/README.md | 158 + cpp/oracle/gen.lean | 122 + cpp/run-fuzz.sh | 21 + cpp/src/fast.cpp | 1077 +++++++ cpp/src/fast.h | 13 + cpp/src/literal.cpp | 960 ++++++ cpp/src/literal.h | 10 + cpp/src/model.h | 313 ++ cpp/tests/bench.cpp | 149 + cpp/tests/diff_corpus.cpp | 104 + cpp/tests/fuzz_raw.cpp | 41 + cpp/tests/fuzz_readable.cpp | 59 + cpp/tests/golden.cpp | 130 + cpp/tests/readable_gen.h | 180 ++ cpp/third_party/nanobench.h | 5473 +++++++++++++++++++++++++++++++++++ 17 files changed, 8871 insertions(+) create mode 100644 cpp/.gitignore create mode 100644 cpp/CMakeLists.txt create mode 100644 cpp/README.md create mode 100644 cpp/oracle/gen.lean create mode 100755 cpp/run-fuzz.sh create mode 100644 cpp/src/fast.cpp create mode 100644 cpp/src/fast.h create mode 100644 cpp/src/literal.cpp create mode 100644 cpp/src/literal.h create mode 100644 cpp/src/model.h create mode 100644 cpp/tests/bench.cpp create mode 100644 cpp/tests/diff_corpus.cpp create mode 100644 cpp/tests/fuzz_raw.cpp create mode 100644 cpp/tests/fuzz_readable.cpp create mode 100644 cpp/tests/golden.cpp create mode 100644 cpp/tests/readable_gen.h create mode 100644 cpp/third_party/nanobench.h diff --git a/cpp/.gitignore b/cpp/.gitignore new file mode 100644 index 0000000..44c86b9 --- /dev/null +++ b/cpp/.gitignore @@ -0,0 +1,2 @@ +build/ +oracle/golden.txt diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt new file mode 100644 index 0000000..52033c3 --- /dev/null +++ b/cpp/CMakeLists.txt @@ -0,0 +1,59 @@ +cmake_minimum_required(VERSION 3.20) +project(conflict_range_cpp LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE) +endif() + +option(CR_BUILD_FUZZERS "Build libFuzzer differential/safety harnesses" ON) +option(CR_BUILD_BENCH "Build the nanobench comparison" ON) + +add_library(cr_literal STATIC src/literal.cpp) +target_include_directories(cr_literal PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src) + +add_library(cr_fast STATIC src/fast.cpp) +target_include_directories(cr_fast PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src) + +# Golden-vector check: the literal translation against vectors produced by +# `lake env lean --run cpp/oracle/gen.lean`. +add_executable(golden_test tests/golden.cpp) +target_link_libraries(golden_test PRIVATE cr_literal) + +# Literal vs. fast on Lean-generated cases that satisfy the readable sorting +# invariant. +add_executable(diff_corpus tests/diff_corpus.cpp) +target_link_libraries(diff_corpus PRIVATE cr_literal cr_fast) + +if(CR_BUILD_BENCH) + add_executable(cr_bench tests/bench.cpp) + target_include_directories(cr_bench PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/third_party + ${CMAKE_CURRENT_SOURCE_DIR}/tests) + target_link_libraries(cr_bench PRIVATE cr_literal cr_fast) + target_compile_definitions(cr_bench PRIVATE NDEBUG) +endif() + +if(CR_BUILD_FUZZERS AND CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set(CR_FUZZ_FLAGS -fsanitize=fuzzer,address,undefined + -fno-sanitize-recover=all -g -O1) + + # Compile the implementations into each fuzz target directly, so that the + # library code itself is instrumented (linking a plain static library would + # only instrument the harness). + add_executable(fuzz_readable tests/fuzz_readable.cpp src/literal.cpp + src/fast.cpp) + target_include_directories(fuzz_readable PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/tests) + target_compile_options(fuzz_readable PRIVATE ${CR_FUZZ_FLAGS}) + target_link_options(fuzz_readable PRIVATE -fsanitize=fuzzer,address,undefined) + + add_executable(fuzz_raw tests/fuzz_raw.cpp src/literal.cpp src/fast.cpp) + target_include_directories(fuzz_raw PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/tests) + target_compile_options(fuzz_raw PRIVATE ${CR_FUZZ_FLAGS}) + target_link_options(fuzz_raw PRIVATE -fsanitize=fuzzer,address,undefined) +endif() diff --git a/cpp/README.md b/cpp/README.md new file mode 100644 index 0000000..fa9f7a0 --- /dev/null +++ b/cpp/README.md @@ -0,0 +1,158 @@ +# C++ translation of `Statement.closedForm` + +This directory contains a C++20 translation of `Statement.closedForm` from +`Statement.lean`, an equivalent summarised implementation, a libFuzzer +differential harness, and a nanobench comparison. + +The Lean build is untouched. + +## Layout + +| Path | Purpose | +| --- | --- | +| `src/model.h` | Data model: borrowed input keys (`std::string_view`), owned result keys (`std::string`), `WithTop Key` = `std::optional`, `ℕ∞` = `NatInf`. | +| `src/literal.cpp` | Line-by-line transliteration of `Statement.closedForm`, including its repeated scans. | +| `src/fast.cpp` | The same decision tree, but with a summary of the reported keys and lazily cached cardinalities. | +| `oracle/gen.lean` | Deterministic generator of golden vectors by evaluating the real Lean `closedForm`. | +| `tests/golden.cpp` | Checks the literal translation against the Lean golden vectors. | +| `tests/readable_gen.h` | Readable-by-construction instance generator (owned database and selectors, with borrowed views into them). | +| `tests/fuzz_readable.cpp` | libFuzzer differential test: literal vs. fast on readable instances. | +| `tests/fuzz_raw.cpp` | libFuzzer safety test on arbitrary instances under ASan/UBSan. | +| `tests/bench.cpp` | nanobench literal-vs-fast benchmark. | +| `third_party/nanobench.h` | Vendored nanobench (commit `7252da9d96bd0d71f508e9b2813ab1f57fdfaec1`). | + +## Data model + +Inputs borrow and results own. `RangeRead` selectors and the reported keys in +`Result` are `std::string_view`s (the reported keys via a `std::span`), so a +caller can point them straight at existing storage and the implementation reads +them in place. `ConflictRange` returns owned `std::string`s because the decision +procedure can synthesise endpoints (`keyAfter`, clamp bounds) that are not +present in the input. + +## Build + +```sh +cd cpp +cmake -S . -B build -G Ninja -DCMAKE_CXX_COMPILER=clang++-21 +cmake --build build +``` + +## Golden vectors from Lean + +```sh +cd .. +lake env lean --run cpp/oracle/gen.lean > cpp/oracle/golden.txt +./cpp/build/golden_test cpp/oracle/golden.txt +# golden: 40000 cases, 0 mismatches +``` + +The golden test compares **exact endpoints**, because the literal translation +reproduces Lean's interval encoding exactly (including empty intervals). + +`diff_corpus` compares the literal and fast implementations on the generated +cases that satisfy the readable sorting invariant: + +```sh +./cpp/build/diff_corpus cpp/oracle/golden.txt +``` + +## Fuzzing + +```sh +cd cpp +mkdir -p /tmp/cr-corpus +./build/fuzz_readable -max_total_time=300 /tmp/cr-corpus +./build/fuzz_raw -max_total_time=300 /tmp/cr-corpus-raw +``` + +or simply `./cpp/run-fuzz.sh` (optionally `./cpp/run-fuzz.sh build 600`). +The CMake fuzz targets compile the implementation sources directly so that the +decoder code, not just the harness, is covered by ASan/UBSan. + +`fuzz_readable` builds a sorted, duplicate-free database, resolves both +selectors against it, and reports either the complete traversal or a nonempty +prefix. This makes every generated pair genuinely `Readable`. The differential +check treats all encodings of the empty interval as equivalent and otherwise +requires exact endpoints. + +`fuzz_raw` only checks memory safety and undefined behaviour for arbitrary +(possibly unreadable) inputs. + +One representative validation run on this machine: + +| check | cases / runs | result | +| --- | ---: | --- | +| literal vs. Lean golden vectors | 40,000 | 0 mismatches (exact endpoints) | +| fast vs. literal on sorted corpus | 18,971 | 0 mismatches | +| readable differential fuzzing | 2,310,641 | 0 mismatches | +| raw safety fuzzing | 1,936,245 | no ASan/UBSan findings | + +## Benchmark + +```sh +./cpp/build/cr_bench +``` + +Representative results (Clang 21, `-O3 -DNDEBUG`, medians of nine runs pinned +to one core; relative to the literal translation, lower is better for `fast`). +CPU frequency scaling makes the absolute numbers noisy, but the shape is stable: + +| case | literal | fast | fast/literal | +| --- | ---: | ---: | ---: | +| small complete (8 keys) | 41.1 ns | 16.6 ns | 0.40x | +| small truncated forward | 16.9 ns | 23.1 ns | 1.37x | +| crossed truncated forward | 23.9 ns | 24.4 ns | 1.02x | +| crossed truncated reverse | 24.0 ns | 24.4 ns | 1.02x | +| long complete forward (512 keys) | 2093 ns | 31.9 ns | 0.02x | +| long truncated forward | 14.8 ns | 19.8 ns | 1.34x | +| long truncated reverse | 10.4 ns | 14.2 ns | 1.36x | + +The large win is on complete reads: the literal translation rescans the reported +keys many times (`residualT`, `reportedBelowBegin`, `allBelowEnd`, ...), while +the summary answers those questions in `O(log n)` or from a cached count. +Truncated and crossed cases are dominated by the shared decision tree, so the +two implementations are close; the literal path is slightly cheaper there +because it performs no summary setup. + +Against the same implementation with owned input keys (the previous model), the +borrowed-key model changes the costs as follows (nine interleaved runs each, +median): + +| case | literal (borrow vs own) | fast (borrow vs own) | +| --- | ---: | ---: | +| small complete | -64% | -17% | +| small truncated forward | -33% | -19% | +| crossed truncated forward | +4% | +2% | +| crossed truncated reverse | +4% | +2% | +| long complete forward | -69% | -15% | +| long truncated forward | -29% | -6% | +| long truncated reverse | -16% | +16% | + +The complete-read gains come from not copying the reported keys just to +normalise the result: a forward result is now the caller's span, and a reverse +result is a reversed view rather than an owned copy. The fast path also benefits +because it previously normalised complete results the same way. Truncated +reverse (`fast`) is the one small regression: it is two nanoseconds and a few +dozen instructions, and the other paths move in the opposite direction, so it +reads as code-layout/code-generation noise rather than extra algorithmic work. + + +## Why the fast implementation is equivalent + +The fast implementation replays exactly the same decision tree. The only +changes are: + +* `reportedBelowBegin` and `residualT` are summarised once. On readable inputs the + reported keys are sorted in traversal order, so both counts are binary + searches. The summaries depend only on the read and on the multiset of + reported keys, so they survive the reversed-view normalisation. +* Interval cardinalities (`intervalCard`) are cached lazily. +* Complete observations are not copied: a forward result borrows the caller's + span, and a reversed result is viewed backwards in place. (The previous + owned-key model copied and reversed the reported keys here.) +* `allBelowEnd` is `residualT == 0` and `anyAtOrAboveEnd` is `residualT != 0`. + +No branch condition or returned endpoint is otherwise changed. Equivalence is +therefore only claimed for readable inputs; `fuzz_raw` deliberately does not +compare outputs on arbitrary inputs. diff --git a/cpp/oracle/gen.lean b/cpp/oracle/gen.lean new file mode 100644 index 0000000..cd8a9a1 --- /dev/null +++ b/cpp/oracle/gen.lean @@ -0,0 +1,122 @@ +import Statement + +/-! +Deterministic golden-vector generator for the C++ translation of +`Statement.closedForm`. + +Run with: + + lake env lean --run cpp/oracle/gen.lean > cpp/oracle/golden.txt + +Each output line is + + ... => + +where a key is lowercase hex, `-` is the empty key, and `T` is `⊤`. +-/ + +open Statement + +def hexDigit (n : Nat) : Char := + if n < 10 then Char.ofNat (48 + n) else Char.ofNat (87 + n) + +def hexByte (b : Byte) : String := + let n := b.val + String.ofList [hexDigit (n / 16), hexDigit (n % 16)] + +def encKey : WithTop Key → String + | none => "T" + | some k => if k.isEmpty then "-" else String.join (k.map hexByte) + +def encRes (res : Result) : String := + let more := if res.more then "1" else "0" + let ks := res.keys.map (fun k => if k.isEmpty then "-" else String.join (k.map hexByte)) + more ++ " " ++ toString res.keys.length ++ " " ++ String.intercalate " " ks + +def encCase (r : RangeRead) (res : Result) : String := + let out := closedForm r res + encKey r.begin.key ++ " " ++ toString r.begin.offset ++ " " ++ + encKey r.end_.key ++ " " ++ toString r.end_.offset ++ " " ++ + (if r.reverse then "1" else "0") ++ " " ++ encRes res ++ " => " ++ + encKey out.begin ++ " " ++ encKey out.end_ + +def key (xs : List Nat) : Key := xs.map (fun n => Fin.ofNat 256 n) + +/-- Keys chosen to exercise prefixes, zero bytes, ordering, and the empty key. -/ +def keyUniverse : List Key := + [ key [], key [0], key [0, 0], key [0, 1], key [0, 2], key [1], key [1, 0], + key [1, 1], key [2], key [0, 0, 0], key [255], key [0, 1, 0], key [3, 0] ] + +def keyChoices : List (WithTop Key) := + (⊤ : WithTop Key) :: keyUniverse.map fun k => (k : WithTop Key) + +def offsets : List Int := + [-256, -10, -5, -3, -2, -1, 0, 1, 2, 3, 4, 5, 10, 100, 255, 256] + +def nth {α : Type} (l : List α) (i : Nat) (d : α) : α := + l.getD (i % l.length) d + +def step (s : Nat) : Nat := (s * 1103515245 + 12345) % 2147483648 + +/-- A weighted byte: the boundary values are common, the rest is uniform. -/ +def weightedByte (s : Nat) : Byte := + let r := step s % 256 + if r < 24 then 0 + else if r < 40 then 1 + else if r < 56 then 2 + else if r < 72 then 255 + else Fin.ofNat 256 r + +/-- A random key of length `0..8` over all 256 byte values. -/ +def randomKey (s : Nat) : Key := + let len := s % 9 + (List.range len).map fun i => weightedByte (s + i) + +structure Gen where + r : RangeRead + res : Result + +def mkCase (s : Nat) : Gen := + let s1 := step s + let s2 := step s1 + let s3 := step s2 + let s4 := step s3 + let s5 := step s4 + let s6 := step s5 + let s7 := step s6 + let r : RangeRead := + { begin := + { key := nth keyChoices s1 (⊤ : WithTop Key) + offset := nth offsets s2 0 } + end_ := + { key := nth keyChoices s3 (⊤ : WithTop Key) + offset := nth offsets s4 0 } + reverse := s5 % 2 = 1 } + let mode := s6 % 6 + let resKeys : List Key := + match mode with + | 0 => [] + | 1 => [randomKey s7] + | 2 => + let start := s7 % keyUniverse.length + (keyUniverse.drop start).take (1 + (step s7) % 4) + | 3 => + let start := s7 % keyUniverse.length + ((keyUniverse.drop start).take (1 + (step s7) % 4)).reverse + | 4 => + [randomKey s7, randomKey (step s7), randomKey (step (step s7))] + | _ => + let a := randomKey s7 + [a, a, randomKey (step s7)] + let more := s6 % 3 = 0 + { r := r, res := { keys := resKeys, more := more } } + +def main : IO Unit := do + let n := 40000 + let mut s := 1234567 + let mut out : Array String := #[] + for _ in List.range n do + s := step s + let g := mkCase s + out := out.push (encCase g.r g.res) + IO.println (String.intercalate "\n" out.toList) diff --git a/cpp/run-fuzz.sh b/cpp/run-fuzz.sh new file mode 100755 index 0000000..9d7e28a --- /dev/null +++ b/cpp/run-fuzz.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Build (if needed) and run the differential and safety fuzzers. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +build="${1:-$here/build}" +seconds="${2:-300}" + +if [[ ! -x "$build/fuzz_readable" ]]; then + cmake -S "$here" -B "$build" -G Ninja -DCMAKE_CXX_COMPILER=clang++-21 + cmake --build "$build" --target fuzz_readable fuzz_raw +fi + +corpus="${TMPDIR:-/tmp}/cr-fuzz-corpus" +mkdir -p "$corpus/readable" "$corpus/raw" + +echo "== differential fuzzing (readable instances) ==" +"$build/fuzz_readable" -max_total_time="$seconds" -max_len=1024 "$corpus/readable" + +echo "== safety fuzzing (arbitrary instances) ==" +"$build/fuzz_raw" -max_total_time="$seconds" -max_len=1024 "$corpus/raw" diff --git a/cpp/src/fast.cpp b/cpp/src/fast.cpp new file mode 100644 index 0000000..760f24c --- /dev/null +++ b/cpp/src/fast.cpp @@ -0,0 +1,1077 @@ +// A summarised implementation of the same decision procedure as +// `Statement.closedForm`. +// +// `literal.cpp` mirrors the Lean definition line by line and therefore rescans +// `res.keys` (and re-derives small arithmetic facts) on almost every branch. +// This version summarises the reported keys once (two binary searches on the +// sorted traversal order) and caches the handful of interval cardinalities the +// decision tree needs, then replays the same tree. Reversed results are handled +// through a `reversed` view instead of copying the key list. +#include "fast.h" + +#include +#include + +namespace cr::fast { +namespace { + +// --------------------------------------------------------------------------- +// Observational summary +// --------------------------------------------------------------------------- + +// A view of the reported keys. `reversed` means the logically forward view is +// the reverse of the underlying span; this is how the complete/truncated +// normalisation of a reversed read is represented without copying. +struct Obs { + std::span keys; + bool reversed = false; + bool more = false; + // Whether the *underlying* span is ascending. Readable results are sorted + // in traversal order, so `!r.reverse` records this and lets the summary use + // binary searches. A reversed view does not change the span itself. + bool ascending = true; + std::size_t n = 0; + + const KeyView& firstK() const { return reversed ? keys.back() : keys.front(); } + const KeyView& lastK() const { return reversed ? keys.front() : keys.back(); } +}; + +// Facts that only depend on the read and on the *multiset* of reported keys. +// They are therefore invariant under reversing the view. +struct Ctx { + const RangeRead* r = nullptr; + std::size_t n = 0; + std::size_t below = 0; // number of reported keys < begin.key + std::size_t t = 0; // number of reported keys >= end.key + // Lazily computed interval cardinalities. + mutable bool haveBE = false; + mutable bool haveEB = false; + mutable NatInf be; + mutable NatInf eb; +}; + +NatInf cardBE(const Ctx& c) { + if (!c.haveBE) { + c.be = intervalCard(c.r->begin.key, c.r->end_.key); + c.haveBE = true; + } + return c.be; +} + +NatInf cardEB(const Ctx& c) { + if (!c.haveEB) { + c.eb = intervalCard(c.r->end_.key, c.r->begin.key); + c.haveEB = true; + } + return c.eb; +} + +NatInf cardKO(KeyView b, OptKeyView e) { + if (!e) return NatInf::top(); + if (!keyLt(b, *e)) return NatInf::of(0); + if (isPrefix(b, *e) && allZeroFrom(*e, b.size())) + return NatInf::of(e->size() - b.size()); + return NatInf::top(); +} + +NatInf cardOK(OptKeyView b, KeyView e) { + if (!b) return NatInf::of(0); + if (!keyLt(*b, e)) return NatInf::of(0); + if (isPrefix(*b, e) && allZeroFrom(e, b->size())) + return NatInf::of(e.size() - b->size()); + return NatInf::top(); +} + +// Number of reported keys strictly below `x`. `ascending` says whether the +// underlying span is ascending (readable forward) or descending (readable +// reverse). This is a binary search, so the summary is cheap even for long +// results. +std::size_t countLess(std::span ks, OptKeyView x, + bool ascending) { + if (!x) return ks.size(); // every key is below ⊤ + if (x->empty()) return 0; // no key is below the empty key + const std::size_t n = ks.size(); + if (ascending) { + std::size_t lo = 0, hi = n; + while (lo < hi) { + std::size_t mid = lo + (hi - lo) / 2; + if (keyLt(ks[mid], *x)) lo = mid + 1; + else hi = mid; + } + return lo; + } + // Descending: the keys below `x` form a suffix. + std::size_t lo = 0, hi = n; + while (lo < hi) { + std::size_t mid = lo + (hi - lo) / 2; + if (keyLt(ks[mid], *x)) hi = mid; + else lo = mid + 1; + } + return n - lo; +} + +Ctx makeCtx(const RangeRead& r, const Obs& o) { + Ctx c; + c.r = &r; + c.n = o.n; + const std::span ks = o.keys; + c.below = countLess(ks, r.begin.key, o.ascending); + c.t = o.n - countLess(ks, r.end_.key, o.ascending); + return c; +} + +// --------------------------------------------------------------------------- +// Comparisons that avoid allocating a one-element `OptKey` +// --------------------------------------------------------------------------- + +// ↑k < b +bool kLtO(KeyView k, OptKeyView b) { return !b || keyLt(k, *b); } +// ↑k <= b +bool kLeO(KeyView k, OptKeyView b) { return !b || !keyLt(*b, k); } +// a < ↑k +bool oLtK(OptKeyView a, KeyView k) { return a.has_value() && keyLt(*a, k); } +// a <= ↑k +bool oLeK(OptKeyView a, KeyView k) { return a.has_value() && !keyLt(k, *a); } + +OptKey after(KeyView k) { return OptKey{keyAfter(k)}; } + +std::uint64_t bSlack(const Ctx& c) { return beginSlack(*c.r); } +std::uint64_t rE(const Ctx& c) { return residualE(*c.r); } +std::uint64_t rB(const Ctx& c) { return residualB(*c.r); } +std::uint64_t npd(const Ctx& c) { return clampNonposD(*c.r); } + +bool allBelowEnd(const Ctx& c) { return c.t == 0; } +bool anyAtOrAboveEnd(const Ctx& c) { return c.t != 0; } + +const KeyView& leastReportedK(const Ctx& c, const Obs& o) { + return c.r->reverse ? o.lastK() : o.firstK(); +} + +// --------------------------------------------------------------------------- +// Arithmetic helpers, on cached facts +// --------------------------------------------------------------------------- + +template +ConflictRange mk(B&& b, E&& e) { + return ConflictRange{ownKey(std::forward(b)), ownKey(std::forward(e))}; +} + +std::uint64_t clampSlack(const Ctx& c) { + return natSub(bSlack(c) + 1, c.below); +} + +OptKey clampEndOneUpper(const Ctx& c, const Obs& o) { + NatInf slack = NatInf::of(clampSlack(c)); + NatInf cap = cardEB(c); + if (niLe(slack, cap) && (niLt(slack, cap) || !o.firstK().empty())) + return ownKey(c.r->begin.key); + return ownKey(c.r->end_.key); +} + +std::uint64_t clampHighThresh(const Ctx& c) { + return clampSlack(c) + (isBottom(c.r->end_.key) ? 1 : 0); +} + +OptKey clampHighU(const Ctx& c, const Obs& o) { + NatInf cap = cardKO(keyAfter(o.lastK()), c.r->begin.key); + if (niLe(NatInf::of(clampHighThresh(c)), cap)) + return ownKey(c.r->begin.key); + return after(o.lastK()); +} + +std::uint64_t clampLongC(const Ctx& c) { + return natSub(natSub(bSlack(c), c.below), npd(c)); +} + +OptKey clampLongUpper(const Ctx& c, const Obs& o) { + NatInf cap = cardEB(c); + NatInf val = NatInf::of(clampLongC(c)); + if (niLt(cap, val) || (niEq(cap, val) && o.firstK().empty())) + return ownKey(c.r->end_.key); + return ownKey(c.r->begin.key); +} + +bool clampNonposBottom(const Ctx& c) { + NatInf lhs = niAdd(NatInf::of(c.below), + niSub(NatInf::of(npd(c) + 1), cardBE(c))); + return niLe(lhs, NatInf::of(bSlack(c))); +} + +bool clampedNonpos(const Ctx& c, const Obs& o) { + return c.r->begin.offset <= 0 && oLeK(c.r->begin.key, leastReportedK(c, o)); +} + +bool clampsToBottom(const Ctx& c, const Obs& o) { + return c.r->begin.offset <= 0 && oLeK(c.r->begin.key, o.firstK()); +} + +std::uint64_t cELo(Int oB, Int oE) { + return std::max(toNat(static_cast<__int128>(2) - oE), + toNat(static_cast<__int128>(oB) - oE + 1)); +} + +std::uint64_t cEminP(Int oE) { + return toNat(static_cast<__int128>(2) - oE); +} + +std::uint64_t noPinB(NatInf KBE, Int oB, Int oE) { + if (KBE.inf) return 0; + std::uint64_t lo = cELo(oB, oE); + return lo > KBE.n ? lo - KBE.n : 0; +} + +std::uint64_t residualD(std::uint64_t rb, NatInf g) { + if (g.inf) return 0; + return natSub(rb, g.n); +} + +std::uint64_t closedFormDeficit(const Ctx& c) { + return residualD(rB(c), cardBE(c)); +} + +std::uint64_t gapOvershoot(const Ctx& c) { + return natSub(rE(c), static_cast(c.t)); +} + +std::uint64_t crossedOvershootGeneral(const Ctx& c) { + return natSub(natSub(rE(c), static_cast(c.t)), rB(c)); +} + +OptKeyView crossedBegin(const Ctx& c) { + if (niLe(NatInf::of(crossedOvershootGeneral(c)), cardEB(c))) + return c.r->end_.key; + return c.r->begin.key; +} + +// --------------------------------------------------------------------------- +// The decision tree +// --------------------------------------------------------------------------- + +struct Impl { + // ----- crossed complete -------------------------------------------------- + + bool negCrossClampEndOne(const Ctx& c, const Obs& o) const { + return !c.r->reverse && c.r->begin.offset < 0 && !o.more && + c.r->end_.offset == 1 && kLtO(o.firstK(), c.r->begin.key) && + c.below <= bSlack(c); + } + + bool negCrossHeadBottom(const Ctx& c, const Obs& o) const { + return !c.r->reverse && c.r->begin.offset < 0 && !o.more && + c.r->end_.offset <= 0 && optLe(c.r->begin.key, c.r->end_.key) && + o.firstK().empty(); + } + + bool negCrossClampNonposLastGe(const Ctx& c, const Obs& o) const { + return !c.r->reverse && c.r->begin.offset < 0 && !o.more && + c.r->end_.offset <= 0 && kLtO(o.firstK(), c.r->begin.key) && + oLeK(c.r->begin.key, o.lastK()) && c.below <= bSlack(c); + } + + bool negCrossClampNonposBottom(const Ctx& c, const Obs& o) const { + return !c.r->reverse && c.r->begin.offset < 0 && !o.more && + c.r->end_.offset <= 0 && optLe(c.r->begin.key, c.r->end_.key) && + kLtO(o.firstK(), c.r->begin.key) && clampNonposBottom(c); + } + + bool negCrossClampLongNonpos(const Ctx& c, const Obs& o) const { + return !c.r->reverse && c.r->begin.offset < 0 && !o.more && + c.r->end_.offset <= 0 && kLtO(o.firstK(), c.r->begin.key) && + c.below <= bSlack(c) && clampNonposBottom(c) && + optLt(c.r->end_.key, c.r->begin.key); + } + + ConflictRange closedFormNegCrossClampUnified(const Ctx& c, const Obs& o) const { + OptKey end; + if (negCrossClampEndOne(c, o)) + end = clampEndOneUpper(c, o); + else if (negCrossClampLongNonpos(c, o) && + !negCrossClampNonposLastGe(c, o)) + end = clampLongUpper(c, o); + else + end = c.r->end_.key; + return mk(bottomKey(), std::move(end)); + } + + OptKey negCrossCompleteHighM(const Ctx& c, const Obs& o) const { + return optMax(c.r->begin.key, after(o.lastK())); + } + + std::uint64_t negCrossHighD(const Ctx& c) const { + return natSub(rE(c), static_cast(c.n)); + } + + NatInf negCrossHighGapCap(const Ctx& c, const Obs& o) const { + return cardOK(c.r->end_.key, o.firstK()); + } + + bool negCrossHighUsesEnd(const Ctx& c, const Obs& o) const { + NatInf d = NatInf::of(negCrossHighD(c)); + NatInf gap = negCrossHighGapCap(c, o); + return niLt(d, gap) || + (niLe(d, gap) && c.r->begin.key.has_value()); + } + + bool negCrossCompleteHigh(const Ctx& c, const Obs& o) const { + return !c.r->reverse && c.r->begin.offset < 0 && !o.more && + c.r->end_.offset >= 2 && kLtO(o.firstK(), c.r->begin.key) && + oLtK(c.r->end_.key, o.firstK()); + } + + ConflictRange closedFormNegCrossCompleteHigh(const Ctx& c, const Obs& o) const { + if (c.below <= bSlack(c)) { + if (c.n == rE(c)) return mk(bottomKey(), clampHighU(c, o)); + return mk(bottomKey(), topKey()); + } + if (c.n == rE(c)) + return mk(c.r->end_.key, negCrossCompleteHighM(c, o)); + if (negCrossHighUsesEnd(c, o)) return mk(c.r->end_.key, topKey()); + return mk(OptKey{o.firstK()}, topKey()); + } + + ConflictRange closedFormNegCrossOvershoot(const Ctx& c, const Obs& o) const { + if (c.below <= bSlack(c)) return mk(bottomKey(), topKey()); + return mk(OptKey{o.firstK()}, topKey()); + } + + ConflictRange closedFormNegCrossEndOne(const Ctx& c, const Obs& o) const { + return mk(OptKey{o.firstK()}, c.r->end_.key); + } + + ConflictRange closedFormNegCrossEndOneLong(const Ctx& c, const Obs& o) const { + return mk(OptKey{o.firstK()}, c.r->begin.key); + } + + ConflictRange closedFormNegCrossCompleteResidual(const Ctx& c, const Obs& o) const { + // Keep the synthesised end key alive while `rp` borrows it. + OptKey reducedEnd = after(o.lastK()); + RangeRead rp = *c.r; + rp.end_.key = optView(reducedEnd); + rp.end_.offset = rp.end_.offset - static_cast(c.t); + Ctx cp = makeCtx(rp, o); + if (rp.end_.offset >= 2) + return closedFormNegCrossOvershoot(cp, o); + if (cp.below <= bSlack(cp)) + return mk(bottomKey(), clampEndOneUpper(cp, o)); + if (optLe(c.r->begin.key, rp.end_.key)) + return closedFormNegCrossEndOne(cp, o); + return closedFormNegCrossEndOneLong(cp, o); + } + + ConflictRange closedFormNegCrossCompleteHighResidual(const Ctx& c, const Obs& o) const { + if (negCrossCompleteHigh(c, o)) + return closedFormNegCrossCompleteHigh(c, o); + return closedFormNegCrossCompleteResidual(c, o); + } + + ConflictRange closedFormNegCrossNonclampUnified(const Ctx& c, const Obs& o) const { + return mk(OptKey{o.firstK()}, optMax(c.r->end_.key, c.r->begin.key)); + } + + ConflictRange closedFormNegCrossTopUnified(const Ctx& c, const Obs& o) const { + OptKey begin = c.below <= bSlack(c) ? bottomKey() + : OptKey{leastReportedK(c, o)}; + return mk(std::move(begin), topKey()); + } + + // ----- crossed truncated ------------------------------------------------- + + bool negCrossTruncNonposGap(const Ctx& c, const Obs& o) const { + return o.more && !c.r->reverse && c.r->begin.offset < 0 && + kLtO(leastReportedK(c, o), c.r->begin.key) && + c.r->end_.offset <= 0 && c.below <= bSlack(c) && + kLtO(o.lastK(), c.r->begin.key) && + kLtO(o.lastK(), c.r->end_.key) && + !isTop(c.r->end_.key) && + (optLt(c.r->end_.key, c.r->begin.key) || + niLt(cardBE(c), NatInf::of(npd(c) + 1))); + } + + bool negCrossTruncNonposGapA2aBottom(const Ctx& c, const Obs& o) const { + return negCrossTruncNonposGap(c, o) && + optLt(c.r->end_.key, c.r->begin.key) && clampNonposBottom(c); + } + + bool negCrossTruncNonposGapA2aBottomHigh(const Ctx& c, const Obs& o) const { + std::uint64_t extra = o.firstK().empty() ? 1 : 0; + NatInf lhs = NatInf::of(bSlack(c) + 1 + extra); + NatInf rhs = niAdd(NatInf::of(static_cast(c.n)), + cardKO(keyAfter(o.lastK()), c.r->begin.key)); + return negCrossTruncNonposGapA2aBottom(c, o) && niLe(lhs, rhs); + } + + ConflictRange closedFormNegCrossTruncNonposGapA2aBottom(const Ctx& c, + const Obs& o) const { + if (negCrossTruncNonposGapA2aBottomHigh(c, o)) + return mk(bottomKey(), c.r->begin.key); + return mk(bottomKey(), c.r->end_.key); + } + + bool finiteUnderCond(const Ctx& c, const Obs& o) const { + NatInf lhs = niAdd(NatInf::of(natSub(c.below, 1)), + cardKO(keyAfter(o.firstK()), c.r->begin.key)); + return niLe(lhs, NatInf::of(bSlack(c))); + } + + bool negCrossTruncRevResidualFiniteOver(const Ctx& c, const Obs& o) const { + return o.more && c.r->reverse && c.r->begin.offset < 0 && + c.r->end_.offset >= 2 && oLtK(c.r->end_.key, o.lastK()) && + kLtO(o.lastK(), c.r->begin.key) && c.t == rE(c); + } + + ConflictRange closedFormNegCrossTruncRevResidualFiniteUnified(const Ctx& c, + const Obs& o) const { + OptKeyView L = asKey(o.lastK()); + OptKey U = finiteUnderCond(c, o) ? after(o.firstK()) : ownKey(c.r->begin.key); + if (negCrossTruncRevResidualFiniteOver(c, o)) + return mk(c.r->end_.key, U); + return mk(std::move(L), std::move(U)); + } + + bool sentinelLargeTight(const Ctx& c, const Obs& o) const { + NatInf lhs = NatInf::of(rE(c)); + NatInf rhs = niAdd(cardOK(c.r->end_.key, o.lastK()), + NatInf::of(static_cast(c.n))); + __int128 nPlusBegin = static_cast<__int128>(c.n) + c.r->begin.offset; + return niEq(lhs, rhs) && nPlusBegin == 1; + } + + bool negCrossTruncRevResidualTopLarge(const Ctx& c, const Obs& o) const { + if (!(o.more && c.r->reverse && c.r->begin.offset < 0 && + c.r->end_.offset >= 2 && oLtK(c.r->end_.key, o.lastK()) && + c.t < rE(c))) + return false; + NatInf lhs = NatInf::of(rE(c)); + NatInf rhs = niAdd(cardOK(c.r->end_.key, o.lastK()), + NatInf::of(static_cast(c.n))); + return niLe(lhs, rhs); + } + + bool negCrossTruncRevResidualTopLargeSentinel(const Ctx& c, const Obs& o) const { + return negCrossTruncRevResidualTopLarge(c, o) && isTop(c.r->begin.key); + } + + ConflictRange closedFormNegCrossTruncRevResidualTopLargeUnified(const Ctx& c, + const Obs& o) const { + OptKey L = OptKey{o.lastK()}; + if (negCrossTruncRevResidualTopLargeSentinel(c, o) && + sentinelLargeTight(c, o)) + return mk(std::move(L), topKey()); + return mk(c.r->end_.key, topKey()); + } + + // ----- complete / offset one --------------------------------------------- + + ConflictRange closedFormReverseTruncatedZero(const Ctx& c, const Obs& o) const { + if (c.r->end_.offset <= 1) + return mk(asKey(o.lastK()), c.r->end_.key); + OptKeyView L = asKey(o.lastK()); + if (c.t == rE(c)) + return mk(optMin(L, c.r->end_.key), after(o.firstK())); + if (isTop(c.r->end_.key)) return mk(std::move(L), topKey()); + KeyView E = *c.r->end_.key; + std::uint64_t m = natSub(natSub(rE(c), c.t), 1); + if (keyLt(keyAfterN(m, E), o.lastK())) + return mk(asKey(E), topKey()); + return mk(std::move(L), topKey()); + } + + ConflictRange truncatedForwardClosed(const Ctx& c, const Obs& o) const { + NatInf cap = cardEB(c); + NatInf room = NatInf::of(natSub(rE(c), static_cast(c.n))); + if (niLe(cap, room)) + return mk(c.r->begin.key, after(o.lastK())); + return mk(c.r->end_.key, after(o.lastK())); + } + + ConflictRange completeClosedWith(OptKey endKey, const Ctx& c, const Obs& o) const { + (void)o; + OptKeyView begin = c.r->begin.key; + if (optLt(c.r->end_.key, c.r->begin.key) && + niLe(NatInf::of(gapOvershoot(c)), cardEB(c))) + begin = c.r->end_.key; + if (c.r->end_.offset <= 1) + return mk(c.r->begin.key, c.r->end_.key); + if (c.t == rE(c)) return mk(std::move(begin), std::move(endKey)); + return mk(std::move(begin), topKey()); + } + + ConflictRange completeReversedClosed(const Ctx& c, const Obs& o) const { + return completeClosedWith(after(o.firstK()), c, o); + } + ConflictRange completeForwardClosed(const Ctx& c, const Obs& o) const { + return completeClosedWith(after(o.lastK()), c, o); + } + ConflictRange completeClosed(const Ctx& c, const Obs& o) const { + return c.r->reverse ? completeReversedClosed(c, o) + : completeForwardClosed(c, o); + } + + ConflictRange closedFormOffsetOneTruncated(const Ctx& c, const Obs& o) const { + if (c.r->end_.offset <= 0) + return mk(c.r->begin.key, c.r->end_.key); + return mk(c.r->begin.key, after(o.lastK())); + } + + ConflictRange topRange(const Ctx& c) const { + return mk(c.r->begin.key, topKey()); + } + + ConflictRange closedFormOffsetOneWith(OptKey endKey, const Ctx& c, + const Obs& o) const { + (void)o; + if (c.r->end_.offset <= 1) + return mk(c.r->begin.key, c.r->end_.key); + if (c.t == rE(c)) + return mk(c.r->begin.key, std::move(endKey)); + return topRange(c); + } + + ConflictRange closedFormOffsetOneRev(const Ctx& c, const Obs& o) const { + return closedFormOffsetOneWith(after(o.firstK()), c, o); + } + ConflictRange closedFormOffsetOne(const Ctx& c, const Obs& o) const { + return closedFormOffsetOneWith(after(o.lastK()), c, o); + } + ConflictRange closedFormOffsetOneAny(const Ctx& c, const Obs& o) const { + return c.r->reverse ? closedFormOffsetOneRev(c, o) + : closedFormOffsetOne(c, o); + } + + ConflictRange closedFormOffsetOneUniversal(const Ctx& c, const Obs& o) const { + if (o.more) + return c.r->reverse ? closedFormReverseTruncatedZero(c, o) + : closedFormOffsetOneTruncated(c, o); + return closedFormOffsetOneAny(c, o); + } + + ConflictRange closedFormOffsetOneTotal(const Ctx& c, const Obs& o) const { + if (optLt(c.r->end_.key, c.r->begin.key)) { + if (o.more) + return c.r->reverse ? closedFormReverseTruncatedZero(c, o) + : truncatedForwardClosed(c, o); + return completeClosed(c, o); + } + return closedFormOffsetOneUniversal(c, o); + } + + // ----- zero crossed ------------------------------------------------------ + + ConflictRange closedFormZeroCrossEnd(const Ctx& c, const Obs& o) const { + return mk(OptKey{leastReportedK(c, o)}, c.r->end_.key); + } + + ConflictRange completeForwardCrossedGeneral(const Ctx& c, const Obs& o) const { + if (c.r->end_.offset <= 1) + return mk(c.r->begin.key, c.r->end_.key); + if (c.t + rB(c) == rE(c)) + return mk(c.r->end_.key, after(o.lastK())); + return mk(crossedBegin(c), topKey()); + } + + RangeRead zeroBeginAnchor(const Ctx& c, const Obs& o) const { + RangeRead out = *c.r; + out.begin.key = asKey(o.firstK()); + out.begin.offset = 1; + return out; + } + + ConflictRange zeroCrossLongAnchor(const Ctx& c, const Obs& o) const { + RangeRead a = zeroBeginAnchor(c, o); + Ctx ca = makeCtx(a, o); + if (oLtK(c.r->end_.key, o.firstK())) + return completeForwardCrossedGeneral(ca, o); + return closedFormOffsetOne(ca, o); + } + + ConflictRange closedFormZeroCrossLong(const Ctx& c, const Obs& o) const { + if (isTop(c.r->begin.key)) { + KeyView H = o.firstK(); + std::uint64_t e = rE(c); + std::uint64_t t = c.t; + NatInf g = cardOK(c.r->end_.key, H); + bool useEnd = oLtK(c.r->end_.key, H) && + niLt(niSub(NatInf::of(e), NatInf::of(t)), g); + return mk(useEnd ? c.r->end_.key : OptKey{H}, topKey()); + } + ConflictRange a = zeroCrossLongAnchor(c, o); + return mk(std::move(a.begin), optMax(a.end_, c.r->begin.key)); + } + + ConflictRange closedFormZeroCrossUnderEnd(const Ctx& c, const Obs& o) const { + RangeRead a = zeroBeginAnchor(c, o); + Ctx ca = makeCtx(a, o); + return closedFormOffsetOneTotal(ca, o); + } + + ConflictRange closedFormZeroCrossComplete(const Ctx& c, const Obs& o) const { + if (c.r->end_.offset <= 0) return closedFormZeroCrossEnd(c, o); + if (c.r->end_.offset == 1) { + if (optLt(c.r->end_.key, c.r->begin.key)) + return mk(OptKey{leastReportedK(c, o)}, c.r->begin.key); + return closedFormZeroCrossEnd(c, o); + } + if (isTop(c.r->begin.key) || optLt(c.r->end_.key, c.r->begin.key)) + return closedFormZeroCrossLong(c, o); + return closedFormZeroCrossUnderEnd(c, o); + } + + ConflictRange closedFormZeroCrossCompleteRev(const Ctx& c, const Obs& o) const { + RangeRead r2 = *c.r; + r2.reverse = false; + Obs o2 = o; + o2.reversed = true; + o2.more = false; + Ctx c2 = makeCtx(r2, o2); + return closedFormZeroCrossComplete(c2, o2); + } + + bool zeroCrossUnderEnd(const Ctx& c, const Obs& o) const { + return c.r->begin.offset == 0 && !o.more && !c.r->reverse && + c.r->end_.offset >= 2 && + kLtO(leastReportedK(c, o), c.r->begin.key) && + optLe(c.r->begin.key, c.r->end_.key); + } + + ConflictRange closedFormZeroCrossResidual(const Ctx& c, const Obs& o) const { + if (zeroCrossUnderEnd(c, o)) return closedFormZeroCrossUnderEnd(c, o); + return closedFormZeroCrossLong(c, o); + } + + ConflictRange closedFormZeroCrossTruncRevCrossed(const Ctx& c, const Obs& o) const { + Obs complete = o; + complete.more = false; + return closedFormZeroCrossCompleteRev(c, complete); + } + + // ----- empty result ------------------------------------------------------ + + bool AlwaysEmpty(const RangeRead& r) const { + NatInf gap = intervalCard(r.begin.key, r.end_.key); + std::uint64_t b = noPinB(gap, r.begin.offset, r.end_.offset); + std::uint64_t e = cELo(r.begin.offset, r.end_.offset); + std::uint64_t cE = cEminP(r.end_.offset); + NatInf underBegin = intervalCard(bottomKey(), r.begin.key); + NatInf underEnd = intervalCard(bottomKey(), r.end_.key); + + bool inner; + if (isTop(r.begin.key)) { + if (isTop(r.end_.key)) + inner = r.begin.offset <= 0 && r.begin.offset < r.end_.offset; + else + inner = r.begin.offset <= 0 && r.begin.offset < r.end_.offset && + niLe(NatInf::of(cE), underEnd); + } else if (isTop(r.end_.key)) { + inner = true; + } else { + bool firstFour = + niLe(NatInf::of(b), underBegin) && + niLe(NatInf::of(e), underEnd) && + niLe(NatInf::of(natSub(b, e)), + intervalCard(r.end_.key, r.begin.key)) && + niLe(NatInf::of(natSub(e, b)), + intervalCard(r.begin.key, r.end_.key)); + __int128 ePlusEnd = static_cast<__int128>(e) + r.end_.offset - 1; + __int128 bPlusBegin = static_cast<__int128>(b) + r.begin.offset - 1; + inner = firstFour && ePlusEnd > 0 && bPlusBegin < ePlusEnd; + } + return !inner; + } + + ConflictRange emptyRange(const RangeRead& r) const { + return mk(r.begin.key, r.begin.key); + } + + ConflictRange crossedRangeGeneral(const RangeRead& r) const { + NatInf diff = niSub(NatInf::of(residualE(r)), NatInf::of(residualB(r))); + if (niLe(diff, intervalCard(r.end_.key, r.begin.key))) + return mk(r.end_.key, topKey()); + return mk(r.begin.key, topKey()); + } + + ConflictRange emptyValue(const RangeRead& r) const { + if (AlwaysEmpty(r)) return emptyRange(r); + if (r.begin.offset <= 0) { + if (r.end_.offset >= 2) return mk(bottomKey(), topKey()); + if (niLt(intervalCard(r.end_.key, r.begin.key), + NatInf::of(toNat(static_cast<__int128>(r.end_.offset) - + r.begin.offset)))) + return mk(bottomKey(), r.end_.key); + return mk(bottomKey(), optMax(r.begin.key, r.end_.key)); + } + if (r.end_.offset <= 1) return mk(r.begin.key, r.end_.key); + if (optLe(r.begin.key, r.end_.key)) return topRange(r); + return crossedRangeGeneral(r); + } + + ConflictRange topRange(const RangeRead& r) const { + return mk(r.begin.key, topKey()); + } + + // ----- complete and forward truncated results ---------------------------- + + OptKeyView closedFormBegin(const Ctx& c, const Obs& o) const { + return clampsToBottom(c, o) ? bottomKeyView() : c.r->begin.key; + } + + bool closedFormFinite(const Ctx& c) const { + return closedFormDeficit(c) + static_cast(c.t) == rE(c); + } + + ConflictRange closedFormCompleteForward(const Ctx& c, const Obs& o) const { + OptKeyView begin = closedFormBegin(c, o); + if (c.r->end_.offset <= 1) + return mk(std::move(begin), c.r->end_.key); + if (clampsToBottom(c, o) || optLe(c.r->begin.key, c.r->end_.key)) { + if (c.t == rE(c)) return mk(std::move(begin), after(o.lastK())); + if (allBelowEnd(c)) return mk(std::move(begin), topKey()); + if (closedFormFinite(c)) return mk(std::move(begin), after(o.lastK())); + return mk(std::move(begin), topKey()); + } + if (closedFormFinite(c)) + return mk(c.r->end_.key, after(o.lastK())); + return mk(crossedBegin(c), topKey()); + } + + ConflictRange closedFormComplete(const Ctx& c, const Obs& o) const { + if (!c.r->reverse) return closedFormCompleteForward(c, o); + RangeRead r2 = *c.r; + r2.reverse = false; + Obs o2 = o; + o2.reversed = true; + o2.more = false; + Ctx c2 = makeCtx(r2, o2); + return closedFormCompleteForward(c2, o2); + } + + NatInf closedFormForwardCapacity(const Ctx& c, const Obs& o) const { + if (clampsToBottom(c, o)) return NatInf::of(0); + return cardEB(c); + } + + ConflictRange closedFormForwardTruncated(const Ctx& c, const Obs& o) const { + OptKeyView begin = closedFormBegin(c, o); + if (c.r->end_.offset <= 0) + return mk(std::move(begin), c.r->end_.key); + std::uint64_t room = + natSub(natSub(rE(c), static_cast(c.n)), rB(c)); + if (niLe(closedFormForwardCapacity(c, o), NatInf::of(room))) + return mk(std::move(begin), after(o.lastK())); + return mk(c.r->end_.key, after(o.lastK())); + } + + // ----- reversed truncation ---------------------------------------------- + + std::uint64_t closedFormReverseCount(const Ctx& c) const { + if (optLt(c.r->end_.key, c.r->begin.key)) + return static_cast(c.n); + return c.t; + } + + bool closedFormReverseFinite(const Ctx& c) const { + return closedFormDeficit(c) + closedFormReverseCount(c) == rE(c); + } + + bool closedFormCrossedRoom(const Ctx& c, const Obs& o) const { + NatInf lhs = + NatInf::of(natSub(rE(c), static_cast(c.n))); + return niLe(lhs, cardOK(c.r->end_.key, o.lastK())); + } + + ConflictRange closedFormReverseTruncatedSkip(const Ctx& c, const Obs& o) const { + if (c.r->end_.offset <= 1) + return mk(c.r->begin.key, c.r->end_.key); + if (allBelowEnd(c)) return topRange(c); + if (closedFormReverseFinite(c)) { + OptKeyView begin = optLt(c.r->end_.key, c.r->begin.key) + ? c.r->end_.key + : c.r->begin.key; + return mk(std::move(begin), after(o.firstK())); + } + OptKeyView begin = + (optLt(c.r->end_.key, c.r->begin.key) && closedFormCrossedRoom(c, o)) + ? c.r->end_.key + : c.r->begin.key; + return mk(std::move(begin), topKey()); + } + + ConflictRange closedFormReverseTruncated(const Ctx& c, const Obs& o) const { + if (rB(c) == 0) return closedFormReverseTruncatedZero(c, o); + return closedFormReverseTruncatedSkip(c, o); + } + + ConflictRange closedFormPositive(const Ctx& c, const Obs& o) const { + if (!o.more) return closedFormComplete(c, o); + if (!c.r->reverse) return closedFormForwardTruncated(c, o); + return closedFormReverseTruncated(c, o); + } + + // ----- crossed dispatch -------------------------------------------------- + + bool overshoot(const Ctx& c) const { + return c.r->end_.offset >= 2 && allBelowEnd(c); + } + + bool zeroCrossOvershootComplete(const Ctx& c, const Obs& o) const { + return c.r->begin.offset == 0 && !o.more && !c.r->reverse && + overshoot(c) && kLtO(leastReportedK(c, o), c.r->begin.key); + } + + bool negCrossOvershoot(const Ctx& c, const Obs& o) const { + return !c.r->reverse && c.r->begin.offset < 0 && !o.more && overshoot(c) && + kLtO(o.firstK(), c.r->begin.key); + } + + bool crossedCompleteOvershoot(const Ctx& c, const Obs& o) const { + return zeroCrossOvershootComplete(c, o) || negCrossOvershoot(c, o); + } + + bool crossedCompleteEndLeOne(const Ctx& c, const Obs& o) const { + if (!(!c.r->reverse && !o.more && c.r->begin.offset <= 0 && + kLtO(o.firstK(), c.r->begin.key))) + return false; + if (c.r->end_.offset == 1 && bSlack(c) < c.below) + return true; + return c.r->end_.offset <= 0 && + !(c.below <= bSlack(c) && clampNonposBottom(c)); + } + + bool zeroCrossLongTop(const Ctx& c, const Obs& o) const { + return c.r->begin.offset == 0 && !o.more && !c.r->reverse && + c.r->end_.offset >= 2 && isTop(c.r->begin.key); + } + + bool zeroCrossLongFinite(const Ctx& c, const Obs& o) const { + return c.r->begin.offset == 0 && !c.r->reverse && + c.r->end_.offset >= 2 && !isTop(c.r->begin.key) && + optLt(c.r->end_.key, c.r->begin.key) && + kLtO(leastReportedK(c, o), c.r->begin.key); + } + + bool crossedCompleteBeginZero(const Ctx& c, const Obs& o) const { + return zeroCrossUnderEnd(c, o) || zeroCrossLongTop(c, o) || + zeroCrossLongFinite(c, o); + } + + bool crossedCompleteClamp(const Ctx& c, const Obs& o) const { + return negCrossClampEndOne(c, o) || negCrossHeadBottom(c, o) || + negCrossClampNonposBottom(c, o) || + negCrossClampLongNonpos(c, o); + } + + ConflictRange closedFormCrossedComplete(const Ctx& c, const Obs& o) const { + if (crossedCompleteOvershoot(c, o)) + return closedFormNegCrossTopUnified(c, o); + if (crossedCompleteEndLeOne(c, o)) + return closedFormNegCrossNonclampUnified(c, o); + if (crossedCompleteBeginZero(c, o)) + return closedFormZeroCrossResidual(c, o); + if (crossedCompleteClamp(c, o)) + return closedFormNegCrossClampUnified(c, o); + return closedFormNegCrossCompleteHighResidual(c, o); + } + + // ----- crossed truncated dispatch --------------------------------------- + + bool zeroCrossTruncRevCrossed(const Ctx& c, const Obs& o) const { + return c.r->begin.offset == 0 && c.r->reverse && o.more && + kLtO(leastReportedK(c, o), c.r->begin.key); + } + + bool negCrossTruncRevUnder(const Ctx& c, const Obs& o) const { + return o.more && c.r->reverse && c.r->begin.offset < 0 && + kLtO(o.lastK(), c.r->begin.key) && + optLe(c.r->begin.key, c.r->end_.key) && c.r->end_.offset <= 1; + } + + bool negCrossTruncRevCrossedLow(const Ctx& c, const Obs& o) const { + return o.more && c.r->reverse && c.r->begin.offset < 0 && + kLtO(o.lastK(), c.r->begin.key) && + optLt(c.r->end_.key, c.r->begin.key) && + c.r->end_.offset <= 1 && + niLe(niAdd(niAdd(cardEB(c), + NatInf::of(static_cast(c.n))), + NatInf::of(toNat(static_cast<__int128>(1) - + c.r->end_.offset))), + NatInf::of(bSlack(c) + 1)); + } + + bool revTruncLeOneLow(const Ctx& c, const Obs& o) const { + return negCrossTruncRevUnder(c, o) || + negCrossTruncRevCrossedLow(c, o); + } + + bool negCrossTruncRevBelowEnd(const Ctx& c, const Obs& o) const { + return o.more && c.r->reverse && c.r->begin.offset < 0 && + kLtO(o.lastK(), c.r->begin.key) && overshoot(c); + } + + bool negCrossTruncRevResidualRoom(const Ctx& c, const Obs& o) const { + return o.more && c.r->reverse && c.r->begin.offset < 0 && + c.r->end_.offset >= 2 && anyAtOrAboveEnd(c) && + niLe(niAdd(cardOK(c.r->end_.key, o.firstK()), NatInf::of(2)), + NatInf::of(rE(c))); + } + + bool negCrossTruncRevResidualTopCrossedSmallGap(const Ctx& c, + const Obs& o) const { + if (!(o.more && c.r->reverse && c.r->begin.offset < 0 && + c.r->end_.offset >= 2 && oLtK(c.r->end_.key, o.lastK()))) + return false; + NatInf lhs = niAdd(cardOK(c.r->end_.key, o.lastK()), + NatInf::of(static_cast(c.n))); + return niLt(lhs, NatInf::of(rE(c))); + } + + bool negCrossTruncRevResidualTopCrossed(const Ctx& c, const Obs& o) const { + return o.more && c.r->reverse && c.r->begin.offset < 0 && + c.r->end_.offset >= 2 && kLeO(o.lastK(), c.r->end_.key) && + c.t < rE(c); + } + + bool revTruncGeTwoTop(const Ctx& c, const Obs& o) const { + return negCrossTruncRevBelowEnd(c, o) || + negCrossTruncRevResidualRoom(c, o) || + negCrossTruncRevResidualTopCrossedSmallGap(c, o) || + negCrossTruncRevResidualTopCrossed(c, o); + } + + bool negCrossTruncRevResidualTopLargeCrossed(const Ctx& c, const Obs& o) const { + return negCrossTruncRevResidualTopLarge(c, o) && + kLtO(o.lastK(), c.r->begin.key) && !isTop(c.r->begin.key); + } + + bool revTruncGeTwoLarge(const Ctx& c, const Obs& o) const { + return negCrossTruncRevResidualTopLargeCrossed(c, o) || + negCrossTruncRevResidualTopLargeSentinel(c, o); + } + + bool negCrossTruncTopTop(const Ctx& c, const Obs& o) const { + return o.more && !c.r->reverse && c.r->begin.offset < 0 && + isTop(c.r->begin.key) && isTop(c.r->end_.key) && + c.r->end_.offset <= 0; + } + + bool negCrossTruncTopTopEq(const Ctx& c, const Obs& o) const { + if (!negCrossTruncTopTop(c, o)) return false; + std::uint64_t extra = + toNat(static_cast<__int128>(1) - c.r->end_.offset); + return static_cast(c.n) + extra == bSlack(c) + 1; + } + + std::uint64_t closedFormClampSlack(const Ctx& c, const Obs& o) const { + return natSub(bSlack(c) + 1, static_cast(c.n)) + + (o.firstK().empty() ? 1 : 0); + } + + bool closedFormClampGapSmall(const Ctx& c, const Obs& o) const { + return niLe(cardKO(o.lastK(), c.r->begin.key), + NatInf::of(closedFormClampSlack(c, o))); + } + + bool closedFormClampGapExactSaturated(const Ctx& c, const Obs& o) const { + if (!(!o.firstK().empty() && isBottom(c.r->end_.key) && + rE(c) == static_cast(c.n))) + return false; + return niEq(cardKO(keyAfter(o.lastK()), c.r->begin.key), + NatInf::of(natSub(bSlack(c) + 1, + static_cast(c.n)))); + } + + bool closedFormClampGapLow(const Ctx& c, const Obs& o) const { + return closedFormClampGapSmall(c, o) || + closedFormClampGapExactSaturated(c, o); + } + + OptKey closedFormClampUpper(const Ctx& c, const Obs& o) const { + if (closedFormClampGapLow(c, o)) return after(o.lastK()); + return ownKey(c.r->begin.key); + } + + bool closedFormForwardSaturated(const Ctx& c, const Obs& o) const { + if (!(rE(c) > 0 && oLtK(c.r->end_.key, o.firstK()))) return false; + NatInf rhs = niAdd(cardOK(c.r->end_.key, o.firstK()), + NatInf::of(static_cast(c.n))); + return niLt(NatInf::of(rE(c)), rhs); + } + + ConflictRange closedFormForwardTruncatedNonpos(const Ctx& c, + const Obs& o) const { + if (c.below <= bSlack(c)) { + if (c.r->end_.offset <= 0) { + if (optLt(c.r->end_.key, c.r->begin.key)) { + if (clampNonposBottom(c)) { + if (kLtO(o.lastK(), c.r->end_.key)) + return closedFormNegCrossTruncNonposGapA2aBottom(c, o); + return mk(bottomKey(), clampLongUpper(c, o)); + } + return mk(OptKey{o.firstK()}, c.r->begin.key); + } + if (negCrossTruncTopTopEq(c, o)) + return mk(OptKey{o.firstK()}, topKey()); + if (clampNonposBottom(c)) + return mk(bottomKey(), c.r->end_.key); + return mk(OptKey{o.firstK()}, c.r->end_.key); + } + return mk(bottomKey(), closedFormClampUpper(c, o)); + } + if (c.r->end_.offset <= 0) + return mk(asKey(o.firstK()), + optMax(c.r->begin.key, c.r->end_.key)); + OptKeyView begin = closedFormForwardSaturated(c, o) + ? c.r->end_.key + : asKey(o.firstK()); + return mk(std::move(begin), optMax(c.r->begin.key, after(o.lastK()))); + } + + ConflictRange closedFormCrossed(const Ctx& c, const Obs& o) const { + if (!o.more) { + if (!c.r->reverse) return closedFormCrossedComplete(c, o); + RangeRead r2 = *c.r; + r2.reverse = false; + Obs o2 = o; + o2.reversed = true; + o2.more = false; + Ctx c2 = makeCtx(r2, o2); + return closedFormCrossedComplete(c2, o2); + } + if (c.r->reverse) { + if (zeroCrossTruncRevCrossed(c, o)) + return closedFormZeroCrossTruncRevCrossed(c, o); + if (revTruncLeOneLow(c, o)) + return mk(OptKey{o.lastK()}, c.r->end_.key); + if (c.r->end_.offset <= 1) + return mk(OptKey{o.lastK()}, c.r->begin.key); + if (revTruncGeTwoTop(c, o)) + return mk(OptKey{o.lastK()}, topKey()); + if (revTruncGeTwoLarge(c, o)) + return closedFormNegCrossTruncRevResidualTopLargeUnified(c, o); + return closedFormNegCrossTruncRevResidualFiniteUnified(c, o); + } + return closedFormForwardTruncatedNonpos(c, o); + } + + // ----- entry point ------------------------------------------------------ + + ConflictRange closedForm(const RangeRead& r, const Result& res) const { + if (res.keys.empty()) return emptyValue(r); + Obs o; + o.keys = res.keys; + o.reversed = false; + o.more = res.more; + o.ascending = !r.reverse; + o.n = res.keys.size(); + Ctx c = makeCtx(r, o); + if (r.begin.offset >= 1 || clampedNonpos(c, o)) + return closedFormPositive(c, o); + return closedFormCrossed(c, o); + } +}; + +} // namespace + +ConflictRange closedForm(const RangeRead& r, const Result& res) { + return Impl{}.closedForm(r, res); +} + +} // namespace cr::fast diff --git a/cpp/src/fast.h b/cpp/src/fast.h new file mode 100644 index 0000000..9b1e0c9 --- /dev/null +++ b/cpp/src/fast.h @@ -0,0 +1,13 @@ +#pragma once + +#include "model.h" + +namespace cr::fast { + +// A summarised implementation of the same decision procedure as +// `Statement.closedForm`. It is equivalent on readable inputs (and in fact +// reproduces the same endpoints), but computes the result statistics once and +// avoids the literal translation's repeated scans and allocations. +ConflictRange closedForm(const RangeRead& r, const Result& res); + +} // namespace cr::fast diff --git a/cpp/src/literal.cpp b/cpp/src/literal.cpp new file mode 100644 index 0000000..f6d1aec --- /dev/null +++ b/cpp/src/literal.cpp @@ -0,0 +1,960 @@ +// A deliberately literal transliteration of `Statement.closedForm` from +// Statement.lean. Every helper mirrors the Lean definition of the same name, +// including repeated scans of `res.keys`; correctness of the translation, not +// speed, is the goal of this file. +#include "literal.h" + +#include +#include + +namespace cr::literal { +namespace { + +using cr::afterFirst; +using cr::afterLast; +using cr::asKey; +using cr::beginSlack; +using cr::bottomKey; +using cr::clampNonposD; +using cr::first; +using cr::firstKey; +using cr::intervalCard; +using cr::isBottom; +using cr::isTop; +using cr::keyAfter; +using cr::keyAfterN; +using cr::keyLt; +using cr::last; +using cr::lastKey; +using cr::leastReported; +using cr::leastReportedOpt; +using cr::natSub; +using cr::niAdd; +using cr::niEq; +using cr::niLe; +using cr::niLt; +using cr::niSub; +using cr::optGe; +using cr::optGt; +using cr::optLe; +using cr::optLt; +using cr::optMax; +using cr::optMin; +using cr::reportedBelowBegin; +using cr::residualB; +using cr::residualE; +using cr::residualT; +using cr::toNat; +using cr::topKey; + +template +ConflictRange mkRange(B&& b, E&& e) { + return ConflictRange{ownKey(std::forward(b)), ownKey(std::forward(e))}; +} + +// --------------------------------------------------------------------------- +// Arithmetic helpers +// --------------------------------------------------------------------------- + +NatInf clampCap(const RangeRead& r) { return intervalCard(r.end_.key, r.begin.key); } + +std::uint64_t clampSlack(const RangeRead& r, const Result& res) { + return natSub(beginSlack(r) + 1, reportedBelowBegin(r, res)); +} + +OptKey clampEndOneUpper(const RangeRead& r, const Result& res) { + NatInf slack = NatInf::of(clampSlack(r, res)); + NatInf cap = clampCap(r); + if (niLe(slack, cap) && (niLt(slack, cap) || first(res) != bottomKeyView())) + return ownKey(r.begin.key); + return ownKey(r.end_.key); +} + +NatInf clampHighCap(const RangeRead& r, const Result& res) { + return intervalCard(afterLast(res), r.begin.key); +} + +std::uint64_t clampHighThresh(const RangeRead& r, const Result& res) { + return clampSlack(r, res) + (isBottom(r.end_.key) ? 1 : 0); +} + +OptKey clampHighU(const RangeRead& r, const Result& res) { + if (niLe(NatInf::of(clampHighThresh(r, res)), clampHighCap(r, res))) + return ownKey(r.begin.key); + return afterLast(res); +} + +std::uint64_t clampLongC(const RangeRead& r, const Result& res) { + return natSub(natSub(beginSlack(r), reportedBelowBegin(r, res)), clampNonposD(r)); +} + +OptKey clampLongUpper(const RangeRead& r, const Result& res) { + NatInf cap = clampCap(r); + NatInf c = NatInf::of(clampLongC(r, res)); + if (niLt(cap, c) || (niEq(cap, c) && first(res) == bottomKeyView())) + return ownKey(r.end_.key); + return ownKey(r.begin.key); +} + +bool clampNonposBottom(const RangeRead& r, const Result& res) { + NatInf lhs = niAdd(NatInf::of(reportedBelowBegin(r, res)), + niSub(NatInf::of(clampNonposD(r) + 1), + intervalCard(r.begin.key, r.end_.key))); + return niLe(lhs, NatInf::of(beginSlack(r))); +} + +bool clampedNonpos(const RangeRead& r, const Result& res) { + return r.begin.offset <= 0 && + optLe(r.begin.key, leastReportedOpt(r, res)); +} + +bool clampsToBottom(const RangeRead& r, const Result& res) { + return r.begin.offset <= 0 && optLe(r.begin.key, first(res)); +} + +std::uint64_t cELo(Int oB, Int oE) { + return std::max(toNat(static_cast<__int128>(2) - oE), + toNat(static_cast<__int128>(oB) - oE + 1)); +} + +std::uint64_t cEminP(Int oE) { + return toNat(static_cast<__int128>(2) - oE); +} + +std::uint64_t noPinB(NatInf KBE, Int oB, Int oE) { + if (KBE.inf) return 0; + std::uint64_t lo = cELo(oB, oE); + return lo > KBE.n ? lo - KBE.n : 0; +} + +std::uint64_t residualD(const RangeRead& r, NatInf g) { + if (g.inf) return 0; + return natSub(residualB(r), g.n); +} + +std::uint64_t closedFormDeficit(const RangeRead& r) { + return residualD(r, intervalCard(r.begin.key, r.end_.key)); +} + +bool allBelowEnd(const RangeRead& r, const Result& res) { + for (KeyView k : res.keys) + if (!optLt(asKey(k), r.end_.key)) return false; + return true; +} + +bool anyAtOrAboveEnd(const RangeRead& r, const Result& res) { + for (KeyView k : res.keys) + if (optLe(r.end_.key, asKey(k))) return true; + return false; +} + +std::uint64_t gapOvershoot(const RangeRead& r, const Result& res) { + return natSub(residualE(r), residualT(r, res)); +} + +std::uint64_t crossedOvershootGeneral(const RangeRead& r, const Result& res) { + return natSub(natSub(residualE(r), residualT(r, res)), residualB(r)); +} + +OptKeyView crossedBegin(const RangeRead& r, const Result& res) { + if (niLe(NatInf::of(crossedOvershootGeneral(r, res)), + intervalCard(r.end_.key, r.begin.key))) + return r.end_.key; + return r.begin.key; +} + +// --------------------------------------------------------------------------- +// Crossed complete cases +// --------------------------------------------------------------------------- + +bool negCrossClampEndOne(const RangeRead& r, const Result& res) { + return !r.reverse && r.begin.offset < 0 && !res.more && + r.end_.offset == 1 && optLt(first(res), r.begin.key) && + reportedBelowBegin(r, res) <= beginSlack(r); +} + +bool negCrossHeadBottom(const RangeRead& r, const Result& res) { + return !r.reverse && r.begin.offset < 0 && !res.more && + r.end_.offset <= 0 && optLe(r.begin.key, r.end_.key) && + first(res) == bottomKeyView(); +} + +bool negCrossClampNonposLastGe(const RangeRead& r, const Result& res) { + return !r.reverse && r.begin.offset < 0 && !res.more && + r.end_.offset <= 0 && optLt(first(res), r.begin.key) && + optLe(r.begin.key, last(res)) && + reportedBelowBegin(r, res) <= beginSlack(r); +} + +bool negCrossClampNonposBottom(const RangeRead& r, const Result& res) { + return !r.reverse && r.begin.offset < 0 && !res.more && + r.end_.offset <= 0 && optLe(r.begin.key, r.end_.key) && + optLt(first(res), r.begin.key) && clampNonposBottom(r, res); +} + +bool negCrossClampLongNonpos(const RangeRead& r, const Result& res) { + return !r.reverse && r.begin.offset < 0 && !res.more && + r.end_.offset <= 0 && optLt(first(res), r.begin.key) && + reportedBelowBegin(r, res) <= beginSlack(r) && + clampNonposBottom(r, res) && optLt(r.end_.key, r.begin.key); +} + +ConflictRange closedFormNegCrossClampUnified(const RangeRead& r, const Result& res) { + OptKey end; + if (negCrossClampEndOne(r, res)) + end = clampEndOneUpper(r, res); + else if (negCrossClampLongNonpos(r, res) && + !negCrossClampNonposLastGe(r, res)) + end = clampLongUpper(r, res); + else + end = ownKey(r.end_.key); + return mkRange(bottomKey(), std::move(end)); +} + +OptKey negCrossCompleteHighM(const RangeRead& r, const Result& res) { + return optMax(r.begin.key, afterLast(res)); +} + +std::uint64_t negCrossHighD(const RangeRead& r, const Result& res) { + return natSub(residualE(r), res.keys.size()); +} + +NatInf negCrossHighGapCap(const RangeRead& r, const Result& res) { + return intervalCard(r.end_.key, first(res)); +} + +bool negCrossHighUsesEnd(const RangeRead& r, const Result& res) { + NatInf d = NatInf::of(negCrossHighD(r, res)); + NatInf gap = negCrossHighGapCap(r, res); + return niLt(d, gap) || (niLe(d, gap) && optLt(r.begin.key, topKey())); +} + +bool negCrossCompleteHigh(const RangeRead& r, const Result& res) { + return !r.reverse && r.begin.offset < 0 && !res.more && + r.end_.offset >= 2 && optLt(first(res), r.begin.key) && + optLt(r.end_.key, first(res)); +} + +ConflictRange closedFormNegCrossCompleteHigh(const RangeRead& r, const Result& res) { + if (reportedBelowBegin(r, res) <= beginSlack(r)) { + if (res.keys.size() == residualE(r)) return mkRange(bottomKey(), clampHighU(r, res)); + return mkRange(bottomKey(), topKey()); + } + if (res.keys.size() == residualE(r)) + return mkRange(r.end_.key, negCrossCompleteHighM(r, res)); + if (negCrossHighUsesEnd(r, res)) return mkRange(r.end_.key, topKey()); + return mkRange(first(res), topKey()); +} + +ConflictRange topOfLast(KeyView k) { return mkRange(asKey(k), topKey()); } + +ConflictRange closedFormNegCrossOvershoot(const RangeRead& r, const Result& res) { + if (reportedBelowBegin(r, res) <= beginSlack(r)) + return mkRange(bottomKey(), topKey()); + return topOfLast(firstKey(res)); +} + +ConflictRange closedFormNegCrossEndOne(const RangeRead& r, const Result& res) { + return mkRange(first(res), r.end_.key); +} + +ConflictRange closedFormNegCrossEndOneLong(const RangeRead& r, const Result& res) { + return mkRange(first(res), r.begin.key); +} + +ConflictRange closedFormNegCrossCompleteResidual(const RangeRead& r, const Result& res) { + OptKey reducedEnd = afterLast(res); + RangeRead rp = r; + rp.end_.key = optView(reducedEnd); + rp.end_.offset = r.end_.offset - static_cast(residualT(r, res)); + if (rp.end_.offset >= 2) + return closedFormNegCrossOvershoot(rp, res); + if (reportedBelowBegin(rp, res) <= beginSlack(rp)) + return mkRange(bottomKey(), clampEndOneUpper(rp, res)); + if (optLe(r.begin.key, rp.end_.key)) + return closedFormNegCrossEndOne(rp, res); + return closedFormNegCrossEndOneLong(rp, res); +} + +ConflictRange closedFormNegCrossCompleteHighResidual(const RangeRead& r, const Result& res) { + if (negCrossCompleteHigh(r, res)) + return closedFormNegCrossCompleteHigh(r, res); + return closedFormNegCrossCompleteResidual(r, res); +} + +ConflictRange closedFormNegCrossNonclampUnified(const RangeRead& r, const Result& res) { + return mkRange(first(res), optMax(r.end_.key, r.begin.key)); +} + +ConflictRange closedFormNegCrossTopUnified(const RangeRead& r, const Result& res) { + OptKeyView begin = reportedBelowBegin(r, res) <= beginSlack(r) + ? bottomKeyView() + : leastReportedOpt(r, res); + return mkRange(std::move(begin), topKey()); +} + +// --------------------------------------------------------------------------- +// Crossed truncated cases +// --------------------------------------------------------------------------- + +bool negCrossTruncNonposGap(const RangeRead& r, const Result& res) { + return res.more && !r.reverse && r.begin.offset < 0 && + optLt(leastReportedOpt(r, res), r.begin.key) && + r.end_.offset <= 0 && + reportedBelowBegin(r, res) <= beginSlack(r) && + optLt(last(res), r.begin.key) && + optLt(last(res), r.end_.key) && + !isTop(r.end_.key) && + (optLt(r.end_.key, r.begin.key) || + niLt(intervalCard(r.begin.key, r.end_.key), + NatInf::of(clampNonposD(r) + 1))); +} + +bool negCrossTruncNonposGapA2aBottom(const RangeRead& r, const Result& res) { + return negCrossTruncNonposGap(r, res) && optLt(r.end_.key, r.begin.key) && + clampNonposBottom(r, res); +} + +bool negCrossTruncNonposGapA2aBottomHigh(const RangeRead& r, const Result& res) { + std::uint64_t extra = isBottom(first(res)) ? 1 : 0; + NatInf lhs = NatInf::of(beginSlack(r) + 1 + extra); + NatInf rhs = niAdd(NatInf::of(res.keys.size()), + intervalCard(afterLast(res), r.begin.key)); + return negCrossTruncNonposGapA2aBottom(r, res) && niLe(lhs, rhs); +} + +ConflictRange closedFormNegCrossTruncNonposGapA2aBottom(const RangeRead& r, + const Result& res) { + if (negCrossTruncNonposGapA2aBottomHigh(r, res)) + return mkRange(bottomKey(), r.begin.key); + return mkRange(bottomKey(), r.end_.key); +} + +bool finiteUnderCond(const RangeRead& r, const Result& res) { + NatInf lhs = niAdd(NatInf::of(natSub(reportedBelowBegin(r, res), 1)), + intervalCard(afterFirst(res), r.begin.key)); + return niLe(lhs, NatInf::of(beginSlack(r))); +} + +bool negCrossTruncRevResidualFiniteOver(const RangeRead& r, const Result& res) { + return res.more && r.reverse && r.begin.offset < 0 && r.end_.offset >= 2 && + optLt(r.end_.key, last(res)) && optLt(last(res), r.begin.key) && + residualT(r, res) == residualE(r); +} + +ConflictRange closedFormNegCrossTruncRevResidualFiniteUnified(const RangeRead& r, + const Result& res) { + OptKeyView L = last(res); + OptKey U = finiteUnderCond(r, res) ? afterFirst(res) : ownKey(r.begin.key); + if (negCrossTruncRevResidualFiniteOver(r, res)) + return mkRange(r.end_.key, U); + return mkRange(std::move(L), std::move(U)); +} + +bool sentinelLargeTight(const RangeRead& r, const Result& res) { + NatInf lhs = NatInf::of(residualE(r)); + NatInf rhs = niAdd(intervalCard(r.end_.key, last(res)), + NatInf::of(res.keys.size())); + __int128 nPlusBegin = static_cast<__int128>(res.keys.size()) + r.begin.offset; + return niEq(lhs, rhs) && nPlusBegin == 1; +} + +bool negCrossTruncRevResidualTopLarge(const RangeRead& r, const Result& res) { + if (!(res.more && r.reverse && r.begin.offset < 0 && r.end_.offset >= 2 && + optLt(r.end_.key, last(res)) && residualT(r, res) < residualE(r))) + return false; + NatInf lhs = NatInf::of(residualE(r)); + NatInf rhs = niAdd(intervalCard(r.end_.key, last(res)), + NatInf::of(res.keys.size())); + return niLe(lhs, rhs); +} + +bool negCrossTruncRevResidualTopLargeSentinel(const RangeRead& r, const Result& res) { + return negCrossTruncRevResidualTopLarge(r, res) && isTop(r.begin.key); +} + +ConflictRange closedFormNegCrossTruncRevResidualTopLargeUnified(const RangeRead& r, + const Result& res) { + OptKeyView L = last(res); + if (negCrossTruncRevResidualTopLargeSentinel(r, res) && + sentinelLargeTight(r, res)) + return mkRange(std::move(L), topKey()); + return mkRange(r.end_.key, topKey()); +} + +ConflictRange truncatedRangeRev(const RangeRead& r, KeyView lastReported) { + return mkRange(asKey(lastReported), r.end_.key); +} + + +// --------------------------------------------------------------------------- +// Complete and offset-one cases +// --------------------------------------------------------------------------- + +ConflictRange closedFormReverseTruncatedZero(const RangeRead& r, const Result& res) { + if (r.end_.offset <= 1) return truncatedRangeRev(r, lastKey(res)); + OptKeyView L = last(res); + if (residualT(r, res) == residualE(r)) { + return mkRange(optMin(L, r.end_.key), afterFirst(res)); + } + if (isTop(r.end_.key)) return mkRange(std::move(L), topKey()); + KeyView E = *r.end_.key; + std::uint64_t n = natSub(natSub(residualE(r), residualT(r, res)), 1); + if (keyLt(keyAfterN(n, E), lastKey(res))) + return mkRange(asKey(E), topKey()); + return mkRange(std::move(L), topKey()); +} + +ConflictRange truncatedRange(const RangeRead& r, KeyView lastReported) { + return mkRange(r.begin.key, asKey(keyAfter(lastReported))); +} + +ConflictRange truncatedForwardClosed(const RangeRead& r, const Result& res) { + NatInf cap = intervalCard(r.end_.key, r.begin.key); + NatInf room = NatInf::of(natSub(residualE(r), res.keys.size())); + if (niLe(cap, room)) return truncatedRange(r, lastKey(res)); + return mkRange(r.end_.key, afterLast(res)); +} + +ConflictRange completeClosedWith(const OptKey& endKey, const RangeRead& r, + const Result& res) { + OptKeyView begin = r.begin.key; + if (optLt(r.end_.key, r.begin.key) && + niLe(NatInf::of(gapOvershoot(r, res)), + intervalCard(r.end_.key, r.begin.key))) + begin = r.end_.key; + if (r.end_.offset <= 1) + return mkRange(r.begin.key, r.end_.key); + if (residualT(r, res) == residualE(r)) + return mkRange(std::move(begin), endKey); + return mkRange(std::move(begin), topKey()); +} + +ConflictRange completeReversedClosed(const RangeRead& r, const Result& res) { + return completeClosedWith(afterFirst(res), r, res); +} +ConflictRange completeForwardClosed(const RangeRead& r, const Result& res) { + return completeClosedWith(afterLast(res), r, res); +} +ConflictRange completeClosed(const RangeRead& r, const Result& res) { + return r.reverse ? completeReversedClosed(r, res) : completeForwardClosed(r, res); +} + +ConflictRange closedFormOffsetOneTruncated(const RangeRead& r, const Result& res) { + if (r.end_.offset <= 0) return mkRange(r.begin.key, r.end_.key); + return truncatedRange(r, lastKey(res)); +} + +ConflictRange topRange(const RangeRead& r) { return mkRange(r.begin.key, topKey()); } + +ConflictRange closedFormOffsetOneWith(KeyView endKey, const RangeRead& r, + const Result& res) { + if (r.end_.offset <= 1) return mkRange(r.begin.key, r.end_.key); + if (residualT(r, res) == residualE(r)) return truncatedRange(r, endKey); + return topRange(r); +} + +ConflictRange closedFormOffsetOneRev(const RangeRead& r, const Result& res) { + return closedFormOffsetOneWith(firstKey(res), r, res); +} +ConflictRange closedFormOffsetOne(const RangeRead& r, const Result& res) { + return closedFormOffsetOneWith(lastKey(res), r, res); +} +ConflictRange closedFormOffsetOneAny(const RangeRead& r, const Result& res) { + return r.reverse ? closedFormOffsetOneRev(r, res) : closedFormOffsetOne(r, res); +} + +ConflictRange closedFormOffsetOneUniversal(const RangeRead& r, const Result& res) { + if (res.more) + return r.reverse ? closedFormReverseTruncatedZero(r, res) + : closedFormOffsetOneTruncated(r, res); + return closedFormOffsetOneAny(r, res); +} + +ConflictRange closedFormOffsetOneTotal(const RangeRead& r, const Result& res) { + if (optLt(r.end_.key, r.begin.key)) { + if (res.more) + return r.reverse ? closedFormReverseTruncatedZero(r, res) + : truncatedForwardClosed(r, res); + return completeClosed(r, res); + } + return closedFormOffsetOneUniversal(r, res); +} + +ConflictRange closedFormZeroCrossEndOneLong(const RangeRead& r, const Result& res) { + return mkRange(leastReportedOpt(r, res), r.begin.key); +} + +// --------------------------------------------------------------------------- +// Zero-crossed cases +// --------------------------------------------------------------------------- + +ConflictRange closedFormZeroCrossEnd(const RangeRead& r, const Result& res) { + return mkRange(leastReportedOpt(r, res), r.end_.key); +} + +ConflictRange completeForwardCrossedGeneral(const RangeRead& r, const Result& res) { + if (r.end_.offset <= 1) return mkRange(r.begin.key, r.end_.key); + if (residualT(r, res) + residualB(r) == residualE(r)) + return mkRange(r.end_.key, afterLast(res)); + return mkRange(crossedBegin(r, res), topKey()); +} + +RangeRead zeroBeginAnchor(const RangeRead& r, const Result& res) { + RangeRead out = r; + out.begin.key = first(res); + out.begin.offset = 1; + return out; +} + +ConflictRange zeroCrossLongAnchor(const RangeRead& r, const Result& res) { + if (optLt(r.end_.key, first(res))) + return completeForwardCrossedGeneral(zeroBeginAnchor(r, res), res); + return closedFormOffsetOne(zeroBeginAnchor(r, res), res); +} + +ConflictRange closedFormZeroCrossLong(const RangeRead& r, const Result& res) { + if (isTop(r.begin.key)) { + OptKeyView H = first(res); + std::uint64_t e = residualE(r); + std::uint64_t t = residualT(r, res); + NatInf g = intervalCard(r.end_.key, H); + bool useEnd = optLt(r.end_.key, H) && + niLt(niSub(NatInf::of(e), NatInf::of(t)), g); + return mkRange(useEnd ? r.end_.key : H, topKey()); + } + ConflictRange a = zeroCrossLongAnchor(r, res); + return mkRange(a.begin, optMax(a.end_, r.begin.key)); +} + +ConflictRange closedFormZeroCrossUnderEnd(const RangeRead& r, const Result& res) { + return closedFormOffsetOneTotal(zeroBeginAnchor(r, res), res); +} + +ConflictRange closedFormZeroCrossComplete(const RangeRead& r, const Result& res) { + if (r.end_.offset <= 0) return closedFormZeroCrossEnd(r, res); + if (r.end_.offset == 1) { + if (optLt(r.end_.key, r.begin.key)) + return closedFormZeroCrossEndOneLong(r, res); + return closedFormZeroCrossEnd(r, res); + } + if (isTop(r.begin.key) || optLt(r.end_.key, r.begin.key)) + return closedFormZeroCrossLong(r, res); + return closedFormZeroCrossUnderEnd(r, res); +} + +ConflictRange closedFormZeroCrossCompleteRev(const RangeRead& r, const Result& res) { + std::vector rev = reversedKeys(res); + return closedFormZeroCrossComplete(forwardRead(r), Result{rev, false}); +} + +bool zeroCrossUnderEnd(const RangeRead& r, const Result& res) { + return r.begin.offset == 0 && !res.more && !r.reverse && + r.end_.offset >= 2 && optLt(leastReportedOpt(r, res), r.begin.key) && + optLe(r.begin.key, r.end_.key); +} + +ConflictRange closedFormZeroCrossResidual(const RangeRead& r, const Result& res) { + if (zeroCrossUnderEnd(r, res)) return closedFormZeroCrossUnderEnd(r, res); + return closedFormZeroCrossLong(r, res); +} + +ConflictRange closedFormZeroCrossTruncRevCrossed(const RangeRead& r, const Result& res) { + Result complete = res; + complete.more = false; + return closedFormZeroCrossCompleteRev(r, complete); +} + +// --------------------------------------------------------------------------- +// Empty result +// --------------------------------------------------------------------------- + +bool AlwaysEmpty(const RangeRead& r) { + NatInf gap = intervalCard(r.begin.key, r.end_.key); + std::uint64_t b = noPinB(gap, r.begin.offset, r.end_.offset); + std::uint64_t e = cELo(r.begin.offset, r.end_.offset); + std::uint64_t cE = cEminP(r.end_.offset); + NatInf underBegin = intervalCard(bottomKey(), r.begin.key); + NatInf underEnd = intervalCard(bottomKey(), r.end_.key); + + bool inner; + if (isTop(r.begin.key)) { + if (isTop(r.end_.key)) + inner = r.begin.offset <= 0 && r.begin.offset < r.end_.offset; + else + inner = r.begin.offset <= 0 && r.begin.offset < r.end_.offset && + niLe(NatInf::of(cE), underEnd); + } else if (isTop(r.end_.key)) { + inner = true; + } else { + bool firstFour = + niLe(NatInf::of(b), underBegin) && + niLe(NatInf::of(e), underEnd) && + niLe(NatInf::of(natSub(b, e)), intervalCard(r.end_.key, r.begin.key)) && + niLe(NatInf::of(natSub(e, b)), intervalCard(r.begin.key, r.end_.key)); + __int128 ePlusEnd = static_cast<__int128>(e) + r.end_.offset - 1; + __int128 bPlusBegin = static_cast<__int128>(b) + r.begin.offset - 1; + inner = firstFour && ePlusEnd > 0 && bPlusBegin < ePlusEnd; + } + return !inner; +} + +ConflictRange emptyRange(const RangeRead& r) { + return mkRange(r.begin.key, r.begin.key); +} + +ConflictRange crossedRangeGeneral(const RangeRead& r) { + NatInf diff = niSub(NatInf::of(residualE(r)), NatInf::of(residualB(r))); + if (niLe(diff, intervalCard(r.end_.key, r.begin.key))) + return mkRange(r.end_.key, topKey()); + return mkRange(r.begin.key, topKey()); +} + +ConflictRange emptyValue(const RangeRead& r) { + if (AlwaysEmpty(r)) return emptyRange(r); + if (r.begin.offset <= 0) { + if (r.end_.offset >= 2) return mkRange(bottomKey(), topKey()); + if (niLt(intervalCard(r.end_.key, r.begin.key), + NatInf::of(toNat(static_cast<__int128>(r.end_.offset) - + r.begin.offset)))) + return mkRange(bottomKey(), r.end_.key); + return mkRange(bottomKey(), optMax(r.begin.key, r.end_.key)); + } + if (r.end_.offset <= 1) return mkRange(r.begin.key, r.end_.key); + if (optLe(r.begin.key, r.end_.key)) return topRange(r); + return crossedRangeGeneral(r); +} + +// --------------------------------------------------------------------------- +// Complete and forward-truncated results +// --------------------------------------------------------------------------- + +OptKeyView closedFormBegin(const RangeRead& r, const Result& res) { + return clampsToBottom(r, res) ? bottomKeyView() : r.begin.key; +} + +bool closedFormFinite(const RangeRead& r, const Result& res) { + return closedFormDeficit(r) + residualT(r, res) == residualE(r); +} + +ConflictRange closedFormCompleteForward(const RangeRead& r, const Result& res) { + OptKeyView begin = closedFormBegin(r, res); + if (r.end_.offset <= 1) return mkRange(begin, r.end_.key); + if (clampsToBottom(r, res) || optLe(r.begin.key, r.end_.key)) { + if (residualT(r, res) == residualE(r)) + return mkRange(begin, afterLast(res)); + if (allBelowEnd(r, res)) return mkRange(begin, topKey()); + if (closedFormFinite(r, res)) return mkRange(begin, afterLast(res)); + return mkRange(begin, topKey()); + } + if (closedFormFinite(r, res)) return mkRange(r.end_.key, afterLast(res)); + return mkRange(crossedBegin(r, res), topKey()); +} + +ConflictRange closedFormComplete(const RangeRead& r, const Result& res) { + if (!r.reverse) return closedFormCompleteForward(forwardRead(r), res); + std::vector rev = reversedKeys(res); + return closedFormCompleteForward(forwardRead(r), Result{rev, false}); +} + +NatInf closedFormForwardCapacity(const RangeRead& r, const Result& res) { + if (clampsToBottom(r, res)) return NatInf::of(0); + return intervalCard(r.end_.key, r.begin.key); +} + +ConflictRange closedFormForwardTruncated(const RangeRead& r, const Result& res) { + OptKeyView begin = closedFormBegin(r, res); + if (r.end_.offset <= 0) return mkRange(begin, r.end_.key); + std::uint64_t room = natSub(natSub(residualE(r), res.keys.size()), residualB(r)); + if (niLe(closedFormForwardCapacity(r, res), NatInf::of(room))) + return mkRange(begin, afterLast(res)); + return mkRange(r.end_.key, afterLast(res)); +} + +// --------------------------------------------------------------------------- +// Reversed truncation +// --------------------------------------------------------------------------- + +std::uint64_t closedFormReverseCount(const RangeRead& r, const Result& res) { + if (optLt(r.end_.key, r.begin.key)) + return static_cast(res.keys.size()); + return residualT(r, res); +} + +bool closedFormReverseFinite(const RangeRead& r, const Result& res) { + return closedFormDeficit(r) + closedFormReverseCount(r, res) == residualE(r); +} + +bool closedFormCrossedRoom(const RangeRead& r, const Result& res) { + NatInf lhs = NatInf::of(natSub(residualE(r), res.keys.size())); + return niLe(lhs, intervalCard(r.end_.key, last(res))); +} + +ConflictRange closedFormReverseTruncatedSkip(const RangeRead& r, const Result& res) { + if (r.end_.offset <= 1) return mkRange(r.begin.key, r.end_.key); + if (allBelowEnd(r, res)) return topRange(r); + if (closedFormReverseFinite(r, res)) { + OptKeyView begin = optLt(r.end_.key, r.begin.key) ? r.end_.key : r.begin.key; + return mkRange(begin, afterFirst(res)); + } + OptKeyView begin = (optLt(r.end_.key, r.begin.key) && closedFormCrossedRoom(r, res)) + ? r.end_.key + : r.begin.key; + return mkRange(begin, topKey()); +} + +ConflictRange closedFormReverseTruncated(const RangeRead& r, const Result& res) { + if (residualB(r) == 0) return closedFormReverseTruncatedZero(r, res); + return closedFormReverseTruncatedSkip(r, res); +} + +ConflictRange closedFormPositive(const RangeRead& r, const Result& res) { + if (!res.more) return closedFormComplete(r, res); + if (!r.reverse) return closedFormForwardTruncated(r, res); + return closedFormReverseTruncated(r, res); +} + +// --------------------------------------------------------------------------- +// Crossed table +// --------------------------------------------------------------------------- + +bool overshoot(const RangeRead& r, const Result& res) { + return r.end_.offset >= 2 && allBelowEnd(r, res); +} + +bool zeroCrossOvershootComplete(const RangeRead& r, const Result& res) { + return r.begin.offset == 0 && !res.more && !r.reverse && overshoot(r, res) && + optLt(leastReportedOpt(r, res), r.begin.key); +} + +bool negCrossOvershoot(const RangeRead& r, const Result& res) { + return !r.reverse && r.begin.offset < 0 && !res.more && overshoot(r, res) && + optLt(first(res), r.begin.key); +} + +bool crossedCompleteOvershoot(const RangeRead& r, const Result& res) { + return zeroCrossOvershootComplete(r, res) || negCrossOvershoot(r, res); +} + +bool crossedCompleteEndLeOne(const RangeRead& r, const Result& res) { + if (!(!r.reverse && !res.more && r.begin.offset <= 0 && + optLt(first(res), r.begin.key))) + return false; + if (r.end_.offset == 1 && beginSlack(r) < reportedBelowBegin(r, res)) + return true; + return r.end_.offset <= 0 && + !(reportedBelowBegin(r, res) <= beginSlack(r) && + clampNonposBottom(r, res)); +} + +bool zeroCrossLongTop(const RangeRead& r, const Result& res) { + return r.begin.offset == 0 && !res.more && !r.reverse && + r.end_.offset >= 2 && isTop(r.begin.key); +} + +bool zeroCrossLongFinite(const RangeRead& r, const Result& res) { + return r.begin.offset == 0 && !res.more && !r.reverse && + r.end_.offset >= 2 && !isTop(r.begin.key) && + optLt(r.end_.key, r.begin.key) && + optLt(leastReportedOpt(r, res), r.begin.key); +} + +bool crossedCompleteBeginZero(const RangeRead& r, const Result& res) { + return zeroCrossUnderEnd(r, res) || zeroCrossLongTop(r, res) || + zeroCrossLongFinite(r, res); +} + +bool crossedCompleteClamp(const RangeRead& r, const Result& res) { + return negCrossClampEndOne(r, res) || negCrossHeadBottom(r, res) || + negCrossClampNonposBottom(r, res) || + negCrossClampLongNonpos(r, res); +} + +ConflictRange closedFormCrossedComplete(const RangeRead& r, const Result& res) { + if (crossedCompleteOvershoot(r, res)) + return closedFormNegCrossTopUnified(r, res); + if (crossedCompleteEndLeOne(r, res)) + return closedFormNegCrossNonclampUnified(r, res); + if (crossedCompleteBeginZero(r, res)) + return closedFormZeroCrossResidual(r, res); + if (crossedCompleteClamp(r, res)) + return closedFormNegCrossClampUnified(r, res); + return closedFormNegCrossCompleteHighResidual(r, res); +} + +bool zeroCrossTruncRevCrossed(const RangeRead& r, const Result& res) { + return r.begin.offset == 0 && r.reverse && res.more && + optLt(leastReportedOpt(r, res), r.begin.key); +} + +bool negCrossTruncRevUnder(const RangeRead& r, const Result& res) { + return res.more && r.reverse && r.begin.offset < 0 && + optLt(last(res), r.begin.key) && optLe(r.begin.key, r.end_.key) && + r.end_.offset <= 1; +} + +bool negCrossTruncRevCrossedLow(const RangeRead& r, const Result& res) { + return res.more && r.reverse && r.begin.offset < 0 && + optLt(last(res), r.begin.key) && optLt(r.end_.key, r.begin.key) && + r.end_.offset <= 1 && + niLe(niAdd(niAdd(intervalCard(r.end_.key, r.begin.key), + NatInf::of(res.keys.size())), + NatInf::of(toNat(static_cast<__int128>(1) - r.end_.offset))), + NatInf::of(beginSlack(r) + 1)); +} + +bool revTruncLeOneLow(const RangeRead& r, const Result& res) { + return negCrossTruncRevUnder(r, res) || + negCrossTruncRevCrossedLow(r, res); +} + +bool negCrossTruncRevBelowEnd(const RangeRead& r, const Result& res) { + return res.more && r.reverse && r.begin.offset < 0 && + optLt(last(res), r.begin.key) && overshoot(r, res); +} + +bool negCrossTruncRevResidualRoom(const RangeRead& r, const Result& res) { + return res.more && r.reverse && r.begin.offset < 0 && + r.end_.offset >= 2 && anyAtOrAboveEnd(r, res) && + niLe(niAdd(intervalCard(r.end_.key, first(res)), NatInf::of(2)), + NatInf::of(residualE(r))); +} + +bool negCrossTruncRevResidualTopCrossedSmallGap(const RangeRead& r, + const Result& res) { + if (!(res.more && r.reverse && r.begin.offset < 0 && + r.end_.offset >= 2 && optLt(r.end_.key, last(res)))) + return false; + NatInf lhs = niAdd(intervalCard(r.end_.key, last(res)), + NatInf::of(res.keys.size())); + return niLt(lhs, NatInf::of(residualE(r))); +} + +bool negCrossTruncRevResidualTopCrossed(const RangeRead& r, const Result& res) { + return res.more && r.reverse && r.begin.offset < 0 && + r.end_.offset >= 2 && optLe(last(res), r.end_.key) && + residualT(r, res) < residualE(r); +} + +bool revTruncGeTwoTop(const RangeRead& r, const Result& res) { + return negCrossTruncRevBelowEnd(r, res) || + negCrossTruncRevResidualRoom(r, res) || + negCrossTruncRevResidualTopCrossedSmallGap(r, res) || + negCrossTruncRevResidualTopCrossed(r, res); +} + +bool negCrossTruncRevResidualTopLargeCrossed(const RangeRead& r, + const Result& res) { + return negCrossTruncRevResidualTopLarge(r, res) && + optLt(last(res), r.begin.key) && !isTop(r.begin.key); +} + +bool revTruncGeTwoLarge(const RangeRead& r, const Result& res) { + return negCrossTruncRevResidualTopLargeCrossed(r, res) || + negCrossTruncRevResidualTopLargeSentinel(r, res); +} + +bool negCrossTruncTopTop(const RangeRead& r, const Result& res) { + return res.more && !r.reverse && r.begin.offset < 0 && + isTop(r.begin.key) && isTop(r.end_.key) && r.end_.offset <= 0; +} + +bool negCrossTruncTopTopEq(const RangeRead& r, const Result& res) { + if (!negCrossTruncTopTop(r, res)) return false; + std::uint64_t extra = toNat(static_cast<__int128>(1) - r.end_.offset); + return res.keys.size() + extra == beginSlack(r) + 1; +} + +std::uint64_t closedFormClampSlack(const RangeRead& r, const Result& res) { + return natSub(beginSlack(r) + 1, res.keys.size()) + + (isBottom(first(res)) ? 1 : 0); +} + +bool closedFormClampGapSmall(const RangeRead& r, const Result& res) { + return niLe(intervalCard(asKey(lastKey(res)), r.begin.key), + NatInf::of(closedFormClampSlack(r, res))); +} + +bool closedFormClampGapExactSaturated(const RangeRead& r, const Result& res) { + if (!(first(res) != bottomKeyView() && isBottom(r.end_.key) && + residualE(r) == res.keys.size())) + return false; + return niEq(intervalCard(afterLast(res), r.begin.key), + NatInf::of(natSub(beginSlack(r) + 1, res.keys.size()))); +} + +bool closedFormClampGapLow(const RangeRead& r, const Result& res) { + return closedFormClampGapSmall(r, res) || + closedFormClampGapExactSaturated(r, res); +} + +OptKey closedFormClampUpper(const RangeRead& r, const Result& res) { + if (closedFormClampGapLow(r, res)) return afterLast(res); + return ownKey(r.begin.key); +} + +bool closedFormForwardSaturated(const RangeRead& r, const Result& res) { + if (!(residualE(r) > 0 && optLt(r.end_.key, first(res)))) return false; + NatInf rhs = niAdd(intervalCard(r.end_.key, first(res)), + NatInf::of(res.keys.size())); + return niLt(NatInf::of(residualE(r)), rhs); +} + +ConflictRange closedFormForwardTruncatedNonpos(const RangeRead& r, + const Result& res) { + if (reportedBelowBegin(r, res) <= beginSlack(r)) { + if (r.end_.offset <= 0) { + if (optLt(r.end_.key, r.begin.key)) { + if (clampNonposBottom(r, res)) { + if (optLt(last(res), r.end_.key)) + return closedFormNegCrossTruncNonposGapA2aBottom(r, res); + return mkRange(bottomKey(), clampLongUpper(r, res)); + } + return mkRange(first(res), r.begin.key); + } + if (negCrossTruncTopTopEq(r, res)) + return mkRange(first(res), topKey()); + if (clampNonposBottom(r, res)) + return mkRange(bottomKey(), r.end_.key); + return mkRange(first(res), r.end_.key); + } + return mkRange(bottomKey(), closedFormClampUpper(r, res)); + } + if (r.end_.offset <= 0) + return mkRange(first(res), optMax(r.begin.key, r.end_.key)); + OptKeyView begin = closedFormForwardSaturated(r, res) ? r.end_.key : first(res); + return mkRange(begin, optMax(r.begin.key, afterLast(res))); +} + +ConflictRange closedFormCrossed(const RangeRead& r, const Result& res) { + if (!res.more) { + if (!r.reverse) return closedFormCrossedComplete(r, res); + std::vector rev = reversedKeys(res); + return closedFormCrossedComplete(forwardRead(r), Result{rev, false}); + } + if (r.reverse) { + if (zeroCrossTruncRevCrossed(r, res)) + return closedFormZeroCrossTruncRevCrossed(r, res); + if (revTruncLeOneLow(r, res)) + return truncatedRangeRev(r, lastKey(res)); + if (r.end_.offset <= 1) + return mkRange(last(res), r.begin.key); + if (revTruncGeTwoTop(r, res)) + return topOfLast(lastKey(res)); + if (revTruncGeTwoLarge(r, res)) + return closedFormNegCrossTruncRevResidualTopLargeUnified(r, res); + return closedFormNegCrossTruncRevResidualFiniteUnified(r, res); + } + return closedFormForwardTruncatedNonpos(r, res); +} + +} // namespace + +ConflictRange closedForm(const RangeRead& r, const Result& res) { + if (res.keys.empty()) return emptyValue(r); + if (r.begin.offset >= 1 || clampedNonpos(r, res)) + return closedFormPositive(r, res); + return closedFormCrossed(r, res); +} + +} // namespace cr::literal diff --git a/cpp/src/literal.h b/cpp/src/literal.h new file mode 100644 index 0000000..2be91c3 --- /dev/null +++ b/cpp/src/literal.h @@ -0,0 +1,10 @@ +#pragma once + +#include "model.h" + +namespace cr::literal { + +// A literal transliteration of `Statement.closedForm` from Statement.lean. +ConflictRange closedForm(const RangeRead& r, const Result& res); + +} // namespace cr::literal diff --git a/cpp/src/model.h b/cpp/src/model.h new file mode 100644 index 0000000..1dfa0c6 --- /dev/null +++ b/cpp/src/model.h @@ -0,0 +1,313 @@ +// Shared data model for the C++ translation of `Statement.closedForm`. +// +// Keys are byte strings ordered lexicographically. The model separates +// ownership on the two sides of the API: +// +// * input keys (the selectors in `RangeRead` and the reported keys in +// `Result`) are `std::string_view`s, so callers can point them straight +// at their own storage without copying; +// * result keys (`ConflictRange`) are owned `std::string`s, because the +// decision procedure can synthesise new keys (`keyAfter`, `afterFirst`, +// clamp bounds) that outlive the inputs. +// +// `OptKey`/`OptKeyView` represent Lean's `WithTop Key`: `std::nullopt` is `⊤`, +// and an empty key is the bottom key. `NatInf` represents Lean's `ℕ∞` +// (`ENat`): `inf` is `⊤`, otherwise `n` is the finite value. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cr { + +using Byte = std::uint8_t; +// Owned byte string. `std::string`'s small-string optimisation keeps the +// short keys used by this model allocation-free, and its comparisons go +// through `char_traits`, which orders bytes as `unsigned char`. +using Key = std::string; +// Borrowed byte string, used for every input key. +using KeyView = std::string_view; +using OptKey = std::optional; +using OptKeyView = std::optional; +using Int = std::int64_t; + +struct Selector { + OptKeyView key; + Int offset = 0; +}; + +struct RangeRead { + Selector begin; + Selector end_; + bool reverse = false; +}; + +struct Result { + // A non-owning view of the reported keys. The caller keeps the underlying + // keys alive for the duration of the call. + std::span keys; + bool more = false; +}; + +struct ConflictRange { + OptKey begin; + OptKey end_; +}; + +inline bool operator==(const ConflictRange& a, const ConflictRange& b) { + return a.begin == b.begin && a.end_ == b.end_; +} +inline bool operator!=(const ConflictRange& a, const ConflictRange& b) { + return !(a == b); +} + +// --------------------------------------------------------------------------- +// Borrowed/owned conversions +// --------------------------------------------------------------------------- + +inline KeyView keyView(const Key& k) { return KeyView(k); } + +inline OptKeyView optView(const OptKey& k) { + return k ? OptKeyView{KeyView(*k)} : std::nullopt; +} + +inline OptKey ownKey(OptKeyView k) { + if (!k) return std::nullopt; + // The bottom key is the empty string; avoid the string_view constructor. + if (k->empty()) return OptKey{Key{}}; + return OptKey{Key(*k)}; +} +inline OptKey ownKey(const OptKey& k) { return k; } +inline OptKey ownKey(OptKey&& k) { return std::move(k); } + +// --------------------------------------------------------------------------- +// Keys and `WithTop Key` +// --------------------------------------------------------------------------- + +inline bool keyLt(KeyView a, KeyView b) { return a.compare(b) < 0; } + +inline bool keyEq(KeyView a, KeyView b) { return a == b; } + +template +inline bool optLt(const std::optional& a, const std::optional& b) { + if (!a) return false; // ⊤ < x is false + if (!b) return true; // ↑k < ⊤ + return keyLt(*a, *b); +} + +template +inline bool optLe(const std::optional& a, const std::optional& b) { + return !optLt(b, a); +} +template +inline bool optGt(const std::optional& a, const std::optional& b) { + return optLt(b, a); +} +template +inline bool optGe(const std::optional& a, const std::optional& b) { + return !optLt(a, b); +} + +template +inline OptKey ownAny(const std::optional& k) { + return k ? OptKey{Key(*k)} : std::nullopt; +} + +template +inline OptKey optMin(const std::optional& a, const std::optional& b) { + return optLt(a, b) ? ownAny(a) : ownAny(b); +} +template +inline OptKey optMax(const std::optional& a, const std::optional& b) { + return optLt(a, b) ? ownAny(b) : ownAny(a); +} + +inline OptKey topKey() { return std::nullopt; } +inline OptKey bottomKey() { return OptKey{Key{}}; } +// A view of a static empty string, so empty views always have a valid pointer. +inline KeyView emptyKeyView() { + static const Key empty; + return KeyView(empty); +} +inline OptKeyView topKeyView() { return std::nullopt; } +inline OptKeyView bottomKeyView() { return OptKeyView{emptyKeyView()}; } + +inline bool isTop(OptKeyView k) { return !k.has_value(); } +inline bool isBottom(OptKeyView k) { return k.has_value() && k->empty(); } + +inline OptKey asKey(Key k) { return OptKey{std::move(k)}; } +inline OptKeyView asKey(KeyView k) { return OptKeyView{k}; } + +inline Key keyAfter(KeyView k) { + Key out(k); + out.push_back(0); + return out; +} + +inline Key keyAfterN(std::uint64_t n, KeyView k) { + Key out(k); + out.reserve(out.size() + static_cast(n)); + out.insert(out.end(), static_cast(n), 0); + return out; +} + +inline bool isPrefix(KeyView prefix, KeyView whole) { + return prefix.size() <= whole.size() && + std::equal(prefix.begin(), prefix.end(), whole.begin()); +} + +inline bool allZeroFrom(KeyView k, std::size_t from) { + for (std::size_t i = from; i < k.size(); ++i) + if (k[i] != 0) return false; + return true; +} + +// --------------------------------------------------------------------------- +// `ℕ∞` (ENat) +// --------------------------------------------------------------------------- + +struct NatInf { + bool inf = false; + std::uint64_t n = 0; + + static NatInf top() { return NatInf{true, 0}; } + static NatInf of(std::uint64_t v) { return NatInf{false, v}; } +}; + +inline bool niLe(NatInf a, NatInf b) { + if (a.inf) return b.inf; + if (b.inf) return true; + return a.n <= b.n; +} +inline bool niLt(NatInf a, NatInf b) { + if (a.inf) return false; + if (b.inf) return true; + return a.n < b.n; +} +inline bool niEq(NatInf a, NatInf b) { + if (a.inf != b.inf) return false; + return a.inf || a.n == b.n; +} +inline NatInf niAdd(NatInf a, NatInf b) { + if (a.inf || b.inf) return NatInf::top(); + return NatInf::of(a.n + b.n); +} + +// Lean's `ENat` subtraction: `a - ⊤ = 0`, `⊤ - b = ⊤` (b finite), and +// finite values subtract with truncation at zero. +inline NatInf niSub(NatInf a, NatInf b) { + if (b.inf) return NatInf::of(0); + if (a.inf) return NatInf::top(); + return NatInf::of(a.n >= b.n ? a.n - b.n : 0); +} + +// `Int.toNat`. +inline std::uint64_t toNat(__int128 x) { + return x <= 0 ? 0 : static_cast(x); +} +inline std::uint64_t toNat(Int x) { return toNat(static_cast<__int128>(x)); } + +// Natural-number truncated subtraction. +inline std::uint64_t natSub(std::uint64_t a, std::uint64_t b) { + return a >= b ? a - b : 0; +} + +// --------------------------------------------------------------------------- +// Result projections +// --------------------------------------------------------------------------- + +inline KeyView firstKey(const Result& res) { + return res.keys.empty() ? emptyKeyView() : res.keys.front(); +} +inline KeyView lastKey(const Result& res) { + return res.keys.empty() ? emptyKeyView() : res.keys.back(); +} +inline OptKeyView first(const Result& res) { return asKey(firstKey(res)); } +inline OptKeyView last(const Result& res) { return asKey(lastKey(res)); } +inline OptKey afterFirst(const Result& res) { + return asKey(keyAfter(firstKey(res))); +} +inline OptKey afterLast(const Result& res) { + return asKey(keyAfter(lastKey(res))); +} + +inline KeyView leastReported(const RangeRead& r, const Result& res) { + return r.reverse ? lastKey(res) : firstKey(res); +} +inline OptKeyView leastReportedOpt(const RangeRead& r, const Result& res) { + return asKey(leastReported(r, res)); +} + +inline RangeRead forwardRead(const RangeRead& r) { + RangeRead out = r; + out.reverse = false; + return out; +} + +// A reversed view of the reported keys. The caller owns the returned vector, +// whose elements borrow from `res.keys`. +inline std::vector reversedKeys(const Result& res) { + return {res.keys.rbegin(), res.keys.rend()}; +} + +// --------------------------------------------------------------------------- +// Cardinality of `[b, e)` +// --------------------------------------------------------------------------- + +inline NatInf intervalCard(OptKeyView b, OptKeyView e) { + if (!b) return NatInf::of(0); // b = ⊤ + if (!e) return NatInf::top(); // e = ⊤ + const KeyView bb = *b; + const KeyView ee = *e; + if (!keyLt(bb, ee)) return NatInf::of(0); // e <= b + if (isPrefix(bb, ee) && allZeroFrom(ee, bb.size())) + return NatInf::of(static_cast(ee.size() - bb.size())); + return NatInf::top(); +} + +// --------------------------------------------------------------------------- +// Result-based counters, in the exact form used by the statement +// --------------------------------------------------------------------------- + +inline std::uint64_t residualT(const RangeRead& r, const Result& res) { + if (isTop(r.end_.key)) return 0; + std::uint64_t count = 0; + for (KeyView k : res.keys) + if (!keyLt(k, *r.end_.key)) ++count; // r.end_.key <= k + return count; +} + +inline std::uint64_t reportedBelowBegin(const RangeRead& r, const Result& res) { + if (isTop(r.begin.key)) return static_cast(res.keys.size()); + std::uint64_t count = 0; + for (KeyView k : res.keys) + if (keyLt(k, *r.begin.key)) ++count; + return count; +} + +inline std::uint64_t residualB(const RangeRead& r) { + return toNat(static_cast<__int128>(r.begin.offset) - 1); +} +inline std::uint64_t residualE(const RangeRead& r) { + return toNat(static_cast<__int128>(r.end_.offset) - 1); +} +inline std::uint64_t beginSlack(const RangeRead& r) { + return toNat(-static_cast<__int128>(r.begin.offset)); +} +inline std::uint64_t clampNonposD(const RangeRead& r) { + return toNat(-static_cast<__int128>(r.end_.offset)); +} + +inline std::uint64_t clampCap_(const RangeRead& r) { + NatInf c = intervalCard(r.end_.key, r.begin.key); + return niEq(c, NatInf::top()) ? 0 : c.n; // caller must check finiteness +} + +} // namespace cr diff --git a/cpp/tests/bench.cpp b/cpp/tests/bench.cpp new file mode 100644 index 0000000..94b6ed8 --- /dev/null +++ b/cpp/tests/bench.cpp @@ -0,0 +1,149 @@ +// nanobench comparison of the literal translation and the summarised +// implementation on representative readable instances. +#define ANKERL_NANOBENCH_IMPLEMENT +#include "nanobench.h" + +#include +#include +#include +#include +#include + +#include "../src/fast.h" +#include "../src/literal.h" +#include "readable_gen.h" + +namespace { + +using namespace cr; + +struct Case { + gen::Instance inst; + RangeRead r; + Result res; + const char* name = ""; + + // Build the borrowed views once the case sits at its final address. + void prepare() { + inst.finish(); + r = inst.read(); + res = inst.result(); + } +}; + +std::vector makeDb(std::size_t n, std::size_t len) { + std::vector db; + db.reserve(n); + for (std::size_t i = 0; i < n; ++i) { + Key k; + k.reserve(len); + for (std::size_t j = 0; j < len; ++j) + k.push_back(static_cast((i >> (8 * j)) & 0xff)); + // Ensure strict ordering is preserved for the multi-byte encoding. + if (!db.empty() && !keyLt(db.back(), k)) k.push_back(static_cast(i)); + db.push_back(std::move(k)); + } + std::sort(db.begin(), db.end()); + db.erase(std::unique(db.begin(), db.end()), db.end()); + return db; +} + +Case makeCase(const std::vector& db, OptKey beginKey, Int beginOff, + OptKey endKey, Int endOff, bool reverse, bool truncate, + const char* name) { + gen::Instance inst; + inst.beginKey = std::move(beginKey); + inst.beginOffset = beginOff; + inst.endKey = std::move(endKey); + inst.endOffset = endOff; + inst.reverse = reverse; + + OptKey lo = gen::resolve(db, Selector{optView(inst.beginKey), beginOff}); + OptKey hi = gen::resolve(db, Selector{optView(inst.endKey), endOff}); + std::vector complete = gen::inRange(db, lo, hi); + if (reverse) std::reverse(complete.begin(), complete.end()); + + if (truncate && complete.size() > 1) { + inst.keys.assign( + complete.begin(), + complete.begin() + static_cast(complete.size() / 2)); + inst.more = true; + } else { + inst.keys = std::move(complete); + inst.more = false; + } + Case c; + c.inst = std::move(inst); + c.name = name; + return c; +} + +void check(const Case& c) { + ConflictRange a = literal::closedForm(c.r, c.res); + ConflictRange b = fast::closedForm(c.r, c.res); + if (!gen::equivalent(a, b)) { + std::fprintf(stderr, "benchmark case %s disagrees between implementations\n", + c.name); + std::exit(1); + } +} + +} // namespace + +int main() { + using ankerl::nanobench::Bench; + + const std::vector smallDb = makeDb(8, 1); + const std::vector mediumDb = makeDb(48, 2); + const std::vector longDb = makeDb(512, 3); + + const OptKey bot = bottomKey(); + const OptKey top = std::nullopt; + + // A finite key above every key in `smallDb` and `longDb`, so the complete + // reads below take the scanning branch (`residualT`, `allBelowEnd`, ...) + // instead of short-circuiting on `⊤`. + Key aboveKey; + aboveKey.push_back(static_cast(0xff)); + aboveKey.push_back(static_cast(0xff)); + const OptKey above = OptKey{aboveKey}; + + std::vector cases; + // A short complete read. An end offset of 2 (rather than 1) skips the + // `end.offset <= 1` shortcut, so the literal translation actually rescans the + // reported keys and the summary has something to summarise. + cases.push_back(makeCase(smallDb, bot, 1, above, 2, false, false, + "small complete")); + // A short truncated read (cross-free). + cases.push_back(makeCase(smallDb, smallDb[1], 1, top, 1, false, true, + "small truncated fwd")); + // A crossed read: the end selector resolves before the begin selector. + cases.push_back(makeCase(mediumDb, mediumDb[40], 1, mediumDb[3], 1, false, + true, "crossed truncated fwd")); + cases.push_back(makeCase(mediumDb, mediumDb[40], 1, mediumDb[3], 1, true, + true, "crossed truncated rev")); + // Long complete / truncated reads. + cases.push_back( + makeCase(longDb, bot, 1, above, 2, false, false, "long complete fwd")); + cases.push_back(makeCase(longDb, bot, 1, top, 1, false, true, + "long truncated fwd")); + cases.push_back(makeCase(longDb, bot, 1, top, 1, true, true, + "long truncated rev")); + + // The case vector is final now, so the borrowed views can point at it. + for (Case& c : cases) c.prepare(); + + for (const Case& c : cases) { + check(c); + Bench bench; + bench.title(c.name).relative(true).warmup(20).minEpochIterations(200); + bench.run("literal", [&] { + ankerl::nanobench::doNotOptimizeAway( + literal::closedForm(c.r, c.res)); + }); + bench.run("fast", [&] { + ankerl::nanobench::doNotOptimizeAway(fast::closedForm(c.r, c.res)); + }); + } + return 0; +} diff --git a/cpp/tests/diff_corpus.cpp b/cpp/tests/diff_corpus.cpp new file mode 100644 index 0000000..46cfaa9 --- /dev/null +++ b/cpp/tests/diff_corpus.cpp @@ -0,0 +1,104 @@ +// Compare the literal translation with the summarised implementation on the +// Lean-generated corpus (readable or not). +#include +#include +#include +#include +#include + +#include "../src/fast.h" +#include "../src/literal.h" + +namespace { +using namespace cr; + +bool parseKey(const std::string& tok, OptKey& out) { + if (tok == "T") { out = std::nullopt; return true; } + if (tok == "-") { out = Key{}; return true; } + if (tok.size() % 2 != 0) return false; + auto hex = [](char c) -> int { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; + }; + Key k; + for (std::size_t i = 0; i < tok.size(); i += 2) { + int hi = hex(tok[i]), lo = hex(tok[i + 1]); + if (hi < 0 || lo < 0) return false; + k.push_back(static_cast(hi * 16 + lo)); + } + out = std::move(k); + return true; +} + +std::string encKey(const OptKey& k) { + if (!k) return "T"; + if (k->empty()) return "-"; + static const char* d = "0123456789abcdef"; + std::string s; + for (Byte b : *k) { s.push_back(d[b >> 4]); s.push_back(d[b & 15]); } + return s; +} +} // namespace + +int main(int argc, char** argv) { + if (argc < 2) return 2; + std::ifstream in(argv[1]); + std::string line; + std::size_t cases = 0, bad = 0; + while (std::getline(in, line)) { + if (line.empty() || line[0] == '#') continue; + std::istringstream ss(line); + std::string bk, ek, arrow, ob, oe; + long long bo = 0, eo = 0; + int rev = 0, more = 0; + std::size_t n = 0; + OptKey beginKey, endKey; + if (!(ss >> bk >> bo >> ek >> eo >> rev >> more >> n)) continue; + if (!parseKey(bk, beginKey) || !parseKey(ek, endKey)) continue; + // Owned reported keys plus the views borrowed by both implementations. + std::vector keys; + keys.reserve(n); + bool ok = true; + for (std::size_t i = 0; i < n; ++i) { + std::string kt; + if (!(ss >> kt)) { ok = false; break; } + OptKey p; + if (!parseKey(kt, p) || !p) { ok = false; break; } + keys.push_back(std::move(*p)); + } + if (!ok || !(ss >> arrow >> ob >> oe) || arrow != "=>") continue; + // `fast` assumes the readable sorting invariant, so only compare cases that + // satisfy it (this is a superset of the readable cases). + const bool reverse = rev != 0; + bool sorted = true; + for (std::size_t i = 1; i < keys.size(); ++i) { + if (reverse ? !keyLt(keys[i], keys[i - 1]) + : !keyLt(keys[i - 1], keys[i])) { + sorted = false; + break; + } + } + if (!sorted) continue; + std::vector keyViews; + keyViews.reserve(keys.size()); + for (const Key& k : keys) keyViews.push_back(k); + RangeRead r{Selector{optView(beginKey), static_cast(bo)}, + Selector{optView(endKey), static_cast(eo)}, reverse}; + Result res{keyViews, more != 0}; + ++cases; + ConflictRange a = literal::closedForm(r, res); + ConflictRange b = fast::closedForm(r, res); + if (a.begin != b.begin || a.end_ != b.end_) { + if (++bad <= 20) { + std::fprintf(stderr, "MISMATCH: lit [%s,%s) fast [%s,%s)\n %s\n", + encKey(a.begin).c_str(), encKey(a.end_).c_str(), + encKey(b.begin).c_str(), encKey(b.end_).c_str(), + line.c_str()); + } + } + } + std::printf("diff: %zu cases, %zu mismatches\n", cases, bad); + return bad == 0 ? 0 : 1; +} diff --git a/cpp/tests/fuzz_raw.cpp b/cpp/tests/fuzz_raw.cpp new file mode 100644 index 0000000..4c04789 --- /dev/null +++ b/cpp/tests/fuzz_raw.cpp @@ -0,0 +1,41 @@ +// libFuzzer safety harness: arbitrary (not necessarily readable) instances. +// Equivalence is only promised on readable inputs, so this harness merely runs +// both implementations under ASan/UBSan to catch memory-safety or UB issues in +// the defensive branches. +#include +#include +#include + +#include "../src/fast.h" +#include "../src/literal.h" +#include "readable_gen.h" + +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, + std::size_t size) { + cr::gen::Bytes bytes{data, size, 0}; + for (int i = 0; i < 8; ++i) { + const std::size_t maxLen = 1 + bytes.u8() % 16; + // Owned selector keys and reported keys; the borrows below point here. + cr::OptKey beginKey = cr::gen::randomOptKey(bytes, maxLen); + cr::OptKey endKey = cr::gen::randomOptKey(bytes, maxLen); + cr::RangeRead r; + r.begin = cr::Selector{cr::optView(beginKey), + cr::gen::randomOffset(bytes)}; + r.end_ = cr::Selector{cr::optView(endKey), cr::gen::randomOffset(bytes)}; + r.reverse = (bytes.u8() & 1) != 0; + + std::vector keys; + const std::size_t n = bytes.u8() % 16; + keys.reserve(n); + for (std::size_t j = 0; j < n; ++j) + keys.push_back(cr::gen::randomKey(bytes, maxLen)); + std::vector views; + views.reserve(keys.size()); + for (const cr::Key& k : keys) views.push_back(k); + cr::Result res{views, (bytes.u8() & 1) != 0}; + + (void)cr::literal::closedForm(r, res); + (void)cr::fast::closedForm(r, res); + } + return 0; +} diff --git a/cpp/tests/fuzz_readable.cpp b/cpp/tests/fuzz_readable.cpp new file mode 100644 index 0000000..411d2d2 --- /dev/null +++ b/cpp/tests/fuzz_readable.cpp @@ -0,0 +1,59 @@ +// libFuzzer differential harness: the literal translation versus the +// summarised implementation, on readable-by-construction instances. +#include +#include +#include +#include +#include + +#include "../src/fast.h" +#include "../src/literal.h" +#include "readable_gen.h" + +namespace { + +std::string enc(cr::OptKeyView k) { + if (!k) return "T"; + if (k->empty()) return "-"; + static const char* d = "0123456789abcdef"; + std::string s; + for (char c : *k) { + auto b = static_cast(c); + s.push_back(d[b >> 4]); + s.push_back(d[b & 15]); + } + return s; +} + +std::string enc(const cr::OptKey& k) { return enc(cr::optView(k)); } + +} // namespace + +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, + std::size_t size) { + cr::gen::Bytes bytes{data, size, 0}; + for (int i = 0; i < 8; ++i) { + cr::gen::Instance inst = cr::gen::readable(bytes); + cr::RangeRead r = inst.read(); + cr::Result res = inst.result(); + cr::ConflictRange lit = cr::literal::closedForm(r, res); + cr::ConflictRange fast = cr::fast::closedForm(r, res); + if (!cr::gen::equivalent(lit, fast)) { + std::fprintf(stderr, + "MISMATCH literal [%s,%s) fast [%s,%s)\n" + " beginKey=%s beginOffset=%lld endKey=%s endOffset=%lld " + "reverse=%d more=%d n=%zu\n", + enc(lit.begin).c_str(), enc(lit.end_).c_str(), + enc(fast.begin).c_str(), enc(fast.end_).c_str(), + enc(r.begin.key).c_str(), + static_cast(r.begin.offset), + enc(r.end_.key).c_str(), + static_cast(r.end_.offset), (int)r.reverse, + (int)res.more, res.keys.size()); + for (cr::KeyView k : res.keys) + std::fprintf(stderr, " key %s\n", enc(k).c_str()); + std::abort(); + } + } + return 0; +} diff --git a/cpp/tests/golden.cpp b/cpp/tests/golden.cpp new file mode 100644 index 0000000..5506d89 --- /dev/null +++ b/cpp/tests/golden.cpp @@ -0,0 +1,130 @@ +// Validates the C++ literal translation against golden vectors produced by +// `cpp/oracle/gen.lean` (which calls the real `Statement.closedForm`). +#include +#include +#include +#include +#include + +#include "../src/literal.h" + +namespace { + +using namespace cr; + +bool parseKey(const std::string& tok, OptKey& out) { + if (tok == "T") { + out = std::nullopt; + return true; + } + if (tok == "-") { + out = Key{}; + return true; + } + if (tok.size() % 2 != 0) return false; + Key k; + k.reserve(tok.size() / 2); + auto hex = [](char c) -> int { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; + }; + for (std::size_t i = 0; i < tok.size(); i += 2) { + int hi = hex(tok[i]); + int lo = hex(tok[i + 1]); + if (hi < 0 || lo < 0) return false; + k.push_back(static_cast(hi * 16 + lo)); + } + out = std::move(k); + return true; +} + +std::string encKey(const OptKey& k) { + if (!k) return "T"; + if (k->empty()) return "-"; + static const char* digits = "0123456789abcdef"; + std::string out; + out.reserve(k->size() * 2); + for (Byte b : *k) { + out.push_back(digits[b >> 4]); + out.push_back(digits[b & 0xf]); + } + return out; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc < 2) { + std::fprintf(stderr, "usage: %s golden.txt\n", argv[0]); + return 2; + } + std::ifstream in(argv[1]); + if (!in) { + std::fprintf(stderr, "cannot open %s\n", argv[1]); + return 2; + } + std::string line; + std::size_t lineNo = 0; + std::size_t cases = 0; + std::size_t bad = 0; + while (std::getline(in, line)) { + ++lineNo; + if (line.empty() || line[0] == '#') continue; + std::istringstream ss(line); + std::string bk, ek, arrow, ob, oe; + long long bo = 0, eo = 0; + int rev = 0, more = 0; + std::size_t n = 0; + OptKey beginKey, endKey; + if (!(ss >> bk >> bo >> ek >> eo >> rev >> more >> n) || + !parseKey(bk, beginKey) || !parseKey(ek, endKey)) { + std::fprintf(stderr, "line %zu: bad header: %s\n", lineNo, line.c_str()); + ++bad; + continue; + } + // Owned reported keys plus the views the literal translation borrows. + std::vector keys; + keys.reserve(n); + bool parseOk = true; + for (std::size_t i = 0; i < n; ++i) { + std::string kt; + if (!(ss >> kt)) { parseOk = false; break; } + OptKey parsed; + if (!parseKey(kt, parsed) || !parsed.has_value()) { parseOk = false; break; } + keys.push_back(std::move(*parsed)); + } + if (!parseOk || !(ss >> arrow >> ob >> oe) || arrow != "=>") { + std::fprintf(stderr, "line %zu: bad body: %s\n", lineNo, line.c_str()); + ++bad; + continue; + } + std::vector keyViews; + keyViews.reserve(keys.size()); + for (const Key& k : keys) keyViews.push_back(k); + RangeRead r{Selector{optView(beginKey), static_cast(bo)}, + Selector{optView(endKey), static_cast(eo)}, rev != 0}; + Result res{keyViews, more != 0}; + OptKey wantBegin, wantEnd; + if (!parseKey(ob, wantBegin) || !parseKey(oe, wantEnd)) { + std::fprintf(stderr, "line %zu: bad expected output\n", lineNo); + ++bad; + continue; + } + ConflictRange got = literal::closedForm(r, res); + ++cases; + if (got.begin != wantBegin || got.end_ != wantEnd) { + ++bad; + if (bad <= 20) { + std::fprintf(stderr, + "MISMATCH line %zu: got [%s,%s) want [%s,%s)\n %s\n", + lineNo, encKey(got.begin).c_str(), + encKey(got.end_).c_str(), encKey(wantBegin).c_str(), + encKey(wantEnd).c_str(), line.c_str()); + } + } + } + std::printf("golden: %zu cases, %zu mismatches\n", cases, bad); + return bad == 0 ? 0 : 1; +} diff --git a/cpp/tests/readable_gen.h b/cpp/tests/readable_gen.h new file mode 100644 index 0000000..67e7f80 --- /dev/null +++ b/cpp/tests/readable_gen.h @@ -0,0 +1,180 @@ +// Deterministic, readable-by-construction instance generation shared by the +// differential fuzzer and the benchmarks. +// +// A generated pair `(r, res)` always satisfies `Readable r res`: we build a +// sorted, duplicate-free database, resolve the two selectors against it, and +// report either the complete traversal or a nonempty prefix of it. +#pragma once + +#include +#include +#include +#include +#include + +#include "../src/model.h" + +namespace cr::gen { + +struct Bytes { + const std::uint8_t* data = nullptr; + std::size_t size = 0; + std::size_t pos = 0; + + std::uint8_t u8() { + if (size == 0) return 0; + std::uint8_t v = data[pos % size]; + ++pos; + return v; + } + std::uint16_t u16() { + std::uint16_t lo = u8(); + std::uint16_t hi = u8(); + return static_cast(lo | (hi << 8)); + } +}; + +inline Byte weightedByte(Bytes& b) { + // Weight the common boundary bytes; otherwise take the raw byte. + std::uint8_t r = b.u8(); + if (r < 24) return 0; + if (r < 40) return 1; + if (r < 56) return 2; + if (r < 72) return 255; + return b.u8(); +} + +inline Key randomKey(Bytes& b, std::size_t maxLen) { + std::size_t len = b.u8() % (maxLen + 1); + Key k; + k.reserve(len); + for (std::size_t i = 0; i < len; ++i) k.push_back(weightedByte(b)); + return k; +} + +inline OptKey randomOptKey(Bytes& b, std::size_t maxLen) { + // One in eight selectors is anchored at `⊤`. + if ((b.u8() & 7) == 0) return std::nullopt; + return OptKey{randomKey(b, maxLen)}; +} + +inline Int randomOffset(Bytes& b) { + // Bias towards the offsets that decide the interesting boundary cases, but + // still allow the full [-256, 255] window. + std::uint8_t pick = b.u8(); + if (pick < 192) { + static const Int small[] = {-3, -2, -1, 0, 1, 2, 3, 4, 5}; + return small[pick % 9]; + } + return static_cast(b.u16() % 512) - 256; +} + +inline OptKey resolve(const std::vector& db, const Selector& s) { + Int anchor; + if (!s.key) { + anchor = static_cast(db.size()) + s.offset - 1; + } else if (s.key->empty()) { + anchor = s.offset - 1; + } else { + std::size_t below = 0; + for (const Key& k : db) + if (keyLt(k, *s.key)) ++below; + anchor = static_cast(below) + s.offset - 1; + } + if (anchor < 0) return OptKey{Key{}}; + if (static_cast(anchor) >= db.size()) return std::nullopt; + return OptKey{db[static_cast(anchor)]}; +} + +inline std::vector inRange(const std::vector& db, const OptKey& lo, + const OptKey& hi) { + std::vector out; + for (const Key& k : db) { + bool geLo = lo.has_value() && !keyLt(k, *lo); + bool ltHi = !hi || keyLt(k, *hi); + if (geLo && ltHi) out.push_back(k); + } + return out; +} + +// Owning storage for a generated instance. `RangeRead` and `Result` borrow +// keys, so the instance keeps the keys alive and rebuilds views on demand. +struct Instance { + // Reported keys in traversal order; backing storage for `views`. + std::vector keys; + // Views into `keys`, built by `finish()`. + std::vector views; + // Owned selector keys; `read()` borrows them. + OptKey beginKey; + OptKey endKey; + Int beginOffset = 0; + Int endOffset = 0; + bool reverse = false; + bool more = false; + + // Must be called once `keys` is final; safe again after a move. + void finish() { + views.clear(); + views.reserve(keys.size()); + for (const Key& k : keys) views.push_back(k); + } + + RangeRead read() const { + RangeRead r; + r.begin = Selector{optView(beginKey), beginOffset}; + r.end_ = Selector{optView(endKey), endOffset}; + r.reverse = reverse; + return r; + } + + Result result() const { return Result{views, more}; } +}; + +inline Instance readable(Bytes& b) { + const std::size_t maxLen = 1 + b.u8() % 16; // key length 0..16 + std::vector db; + const std::size_t count = b.u8() % 20; // 0..19 stored keys + db.reserve(count); + for (std::size_t i = 0; i < count; ++i) db.push_back(randomKey(b, maxLen)); + std::sort(db.begin(), db.end()); + db.erase(std::unique(db.begin(), db.end()), db.end()); + + Instance inst; + inst.beginKey = randomOptKey(b, maxLen); + inst.beginOffset = randomOffset(b); + inst.endKey = randomOptKey(b, maxLen); + inst.endOffset = randomOffset(b); + inst.reverse = (b.u8() & 1) != 0; + + OptKey lo = resolve(db, Selector{optView(inst.beginKey), inst.beginOffset}); + OptKey hi = resolve(db, Selector{optView(inst.endKey), inst.endOffset}); + std::vector complete = inRange(db, lo, hi); + if (inst.reverse) std::reverse(complete.begin(), complete.end()); + + if (!complete.empty() && (b.u8() & 1) != 0) { + std::size_t m = 1 + b.u8() % complete.size(); + inst.keys.assign(complete.begin(), + complete.begin() + static_cast(m)); + inst.more = true; + } else { + inst.keys = std::move(complete); + inst.more = false; + } + inst.finish(); + return inst; +} + +// Range equivalence used by the differential check: any encoding of the empty +// interval is equivalent to any other; nonempty intervals must match exactly. +inline bool isEmptyRange(const ConflictRange& c) { + return !optLt(c.begin, c.end_); +} + +inline bool equivalent(const ConflictRange& a, const ConflictRange& b) { + const bool ea = isEmptyRange(a); + const bool eb = isEmptyRange(b); + if (ea || eb) return ea && eb; + return a.begin == b.begin && a.end_ == b.end_; +} + +} // namespace cr::gen diff --git a/cpp/third_party/nanobench.h b/cpp/third_party/nanobench.h new file mode 100644 index 0000000..e29d4e5 --- /dev/null +++ b/cpp/third_party/nanobench.h @@ -0,0 +1,5473 @@ +// __ _ _______ __ _ _____ ______ _______ __ _ _______ _ _ +// | \ | |_____| | \ | | | |_____] |______ | \ | | |_____| +// | \_| | | | \_| |_____| |_____] |______ | \_| |_____ | | +// +// Microbenchmark framework for C++11/14/17/20 +// https://github.com/martinus/nanobench +// +// Licensed under the MIT License . +// SPDX-License-Identifier: MIT +// Copyright (c) 2019-2023 Martin Leitner-Ankerl +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef ANKERL_NANOBENCH_H_INCLUDED +#define ANKERL_NANOBENCH_H_INCLUDED + +// see https://semver.org/ +#define ANKERL_NANOBENCH_VERSION_MAJOR 4 // incompatible API changes +#define ANKERL_NANOBENCH_VERSION_MINOR 6 // backwards-compatible changes +#define ANKERL_NANOBENCH_VERSION_PATCH 0 // backwards-compatible bug fixes + +/////////////////////////////////////////////////////////////////////////////////////////////////// +// public facing api - as minimal as possible +/////////////////////////////////////////////////////////////////////////////////////////////////// + +#include // high_resolution_clock +#include // log & exp for the paired A/B statistics +#include // memcpy +#include // for std::ostream* custom output target in Config +#include // all names +#include // holds context information of results +#include // holds all results + +#define ANKERL_NANOBENCH(x) ANKERL_NANOBENCH_PRIVATE_##x() + +// MSVC reports __cplusplus as 199711L whatever /std: says, unless /Zc:__cplusplus is passed - and it +// is not passed by default. _MSVC_LANG carries the real value. Getting this wrong does not fail to +// compile, it silently takes the pre-C++17 branch on MSVC, which is how [[nodiscard]] was missing +// there. +#if defined(_MSVC_LANG) +# define ANKERL_NANOBENCH_PRIVATE_CXX() _MSVC_LANG +#else +# define ANKERL_NANOBENCH_PRIVATE_CXX() __cplusplus +#endif +#define ANKERL_NANOBENCH_PRIVATE_CXX98() 199711L +#define ANKERL_NANOBENCH_PRIVATE_CXX11() 201103L +#define ANKERL_NANOBENCH_PRIVATE_CXX14() 201402L +#define ANKERL_NANOBENCH_PRIVATE_CXX17() 201703L + +#if ANKERL_NANOBENCH(CXX) >= ANKERL_NANOBENCH(CXX17) +# define ANKERL_NANOBENCH_PRIVATE_NODISCARD() [[nodiscard]] +# define ANKERL_NANOBENCH_PRIVATE_HAS_STRING_VIEW() 1 +#else +# define ANKERL_NANOBENCH_PRIVATE_NODISCARD() +# define ANKERL_NANOBENCH_PRIVATE_HAS_STRING_VIEW() 0 +#endif + +#if ANKERL_NANOBENCH(HAS_STRING_VIEW) +# include // the name() and run() overloads +#endif + +#if defined(__clang__) +# define ANKERL_NANOBENCH_PRIVATE_IGNORE_PADDED_PUSH() \ + _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wpadded\"") +# define ANKERL_NANOBENCH_PRIVATE_IGNORE_PADDED_POP() _Pragma("clang diagnostic pop") +#else +# define ANKERL_NANOBENCH_PRIVATE_IGNORE_PADDED_PUSH() +# define ANKERL_NANOBENCH_PRIVATE_IGNORE_PADDED_POP() +#endif + +#if defined(__GNUC__) +# define ANKERL_NANOBENCH_PRIVATE_IGNORE_EFFCPP_PUSH() _Pragma("GCC diagnostic push") _Pragma("GCC diagnostic ignored \"-Weffc++\"") +# define ANKERL_NANOBENCH_PRIVATE_IGNORE_EFFCPP_POP() _Pragma("GCC diagnostic pop") +#else +# define ANKERL_NANOBENCH_PRIVATE_IGNORE_EFFCPP_PUSH() +# define ANKERL_NANOBENCH_PRIVATE_IGNORE_EFFCPP_POP() +#endif + +#if defined(ANKERL_NANOBENCH_LOG_ENABLED) +# include +# define ANKERL_NANOBENCH_LOG(x) \ + do { \ + std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << x << std::endl; \ + } while (0) +#else +# define ANKERL_NANOBENCH_LOG(x) \ + do { \ + } while (0) +#endif + +#define ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS() 0 +#if defined(__linux__) && !defined(ANKERL_NANOBENCH_DISABLE_PERF_COUNTERS) +# include +# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0) +// PERF_COUNT_HW_REF_CPU_CYCLES only available since kernel 3.3 +// PERF_FLAG_FD_CLOEXEC since kernel 3.14 +# undef ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS +# define ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS() 1 +# endif +#endif + +#if defined(__clang__) +# define ANKERL_NANOBENCH_NO_SANITIZE(...) __attribute__((no_sanitize(__VA_ARGS__))) +#else +# define ANKERL_NANOBENCH_NO_SANITIZE(...) +#endif + +#if defined(_MSC_VER) +# define ANKERL_NANOBENCH_PRIVATE_NOINLINE() __declspec(noinline) +#else +# define ANKERL_NANOBENCH_PRIVATE_NOINLINE() __attribute__((noinline)) +#endif + +// Whether doNotOptimizeAway can use GCC-style inline assembly, which is both faster and a stronger barrier than the +// function call it falls back to. Testing for _MSC_VER is not enough to rule it out: clang-cl and clang in MSVC +// compatibility mode define _MSC_VER but not __GNUC__, even though they do understand the assembly. +// See https://github.com/martinus/nanobench/issues/111 +#if defined(__clang__) || defined(__GNUC__) +# define ANKERL_NANOBENCH_PRIVATE_ASM_DONT_OPTIMIZE_AWAY() 1 +#else +# define ANKERL_NANOBENCH_PRIVATE_ASM_DONT_OPTIMIZE_AWAY() 0 +#endif + +// workaround missing "is_trivially_copyable" in g++ < 5.0 +// See https://stackoverflow.com/a/31798726/48181 +#if defined(__GNUC__) && __GNUC__ < 5 +# define ANKERL_NANOBENCH_IS_TRIVIALLY_COPYABLE(...) __has_trivial_copy(__VA_ARGS__) +#else +# define ANKERL_NANOBENCH_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value +#endif + +// noexcept may be missing for std::string. +// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58265 +#define ANKERL_NANOBENCH_PRIVATE_NOEXCEPT_STRING_MOVE() std::is_nothrow_move_assignable::value + +// nanobench throws only for programming errors in a render template - a tag that does not exist, a +// section in the wrong place. With exceptions turned off there is nothing sensible left to do but +// stop, so the argument is never evaluated and the message building disappears entirely. +// +// std::abort() rather than a per-compiler intrinsic on purpose: the branch that would pick one is +// the branch no CI leg can reach, and an untested call into CRT internals is exactly the kind of +// thing that turns out not to compile the first time someone actually needs it. +#if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND) +# define ANKERL_NANOBENCH_THROW(...) throw __VA_ARGS__ +#else +# define ANKERL_NANOBENCH_THROW(...) std::abort() +#endif + +// declarations /////////////////////////////////////////////////////////////////////////////////// + +namespace ankerl { +namespace nanobench { + +using Clock = std::conditional::type; +class Bench; +struct Config; +class Result; +class Rng; +class BigO; +class CompareResult; + +namespace detail { +template +class SetupRunner; + +// One of compare()'s alternatives, with its type forgotten so that they can live in a vector. +// +// The alternatives all have different types, so holding them needs either type erasure or a template +// recursion that walks the pack at every step. This is the erasure, and it costs nothing that +// matters: `run` points at an instantiation that knows the concrete type, so the measuring loop is +// inside a function where the operation still inlines exactly as it would otherwise. What is added +// is one indirect call per *epoch*, against an epoch of a millisecond. +// +// std::function would not do, because it would put that indirection in the inner loop instead. +struct ErasedOp { + void (*run)(void* op, uint64_t numIters); // NOLINT(misc-non-private-member-variables-in-classes) + void* op; // NOLINT(misc-non-private-member-variables-in-classes) +}; + +template +void runErasedOp(void* op, uint64_t numIters) { + auto& concrete = *static_cast(op); + for (uint64_t i = 0; i < numIters; ++i) { + concrete(); + } +} + +template +ErasedOp eraseOp(Op&& op) { + using Bare = typename std::remove_reference::type; + // The operation is only ever called, never written to, so casting a const one back to non-const + // to fit it through void* is safe - a const lambda's operator() is const or it could not be + // called at all. + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + return ErasedOp{&runErasedOp, const_cast(static_cast(&op))}; +} +} // namespace detail + +/** + * @brief Renders output from a mustache-like template and benchmark results. + * + * The templating facility here is heavily inspired by [mustache - logic-less templates](https://mustache.github.io/). + * It adds a few more features that are necessary to get all of the captured data out of nanobench. Please read the + * excellent [mustache manual](https://mustache.github.io/mustache.5.html) to see what this is all about. + * + * nanobench output has two nested layers, *result* and *measurement*. Here is a hierarchy of the allowed tags: + * + * * `{{#result}}` Marks the begin of the result layer. Whatever comes after this will be instantiated as often as + * a benchmark result is available. Within it, you can use these tags: + * + * * `{{title}}` See Bench::title. + * + * * `{{name}}` Benchmark name, usually directly provided with Bench::run, but can also be set with Bench::name. + * + * * `{{unit}}` Unit, e.g. `byte`. Defaults to `op`, see Bench::unit. + * + * * `{{batch}}` Batch size, see Bench::batch. + * + * * `{{complexityN}}` Value used for asymptotic complexity calculation. See Bench::complexityN. + * + * * `{{epochs}}` Number of epochs, see Bench::epochs. + * + * * `{{clockResolution}}` Accuracy of the clock, i.e. what's the smallest time possible to measure with the clock. + * For modern systems, this can be around 20 ns. This value is automatically determined by nanobench at the first + * benchmark that is run, and used as a static variable throughout the application's runtime. + * + * * `{{clockResolutionMultiple}}` Configuration multiplier for `clockResolution`. See Bench::clockResolutionMultiple. + * This is the target runtime for each measurement (epoch). That means the more accurate your clock is, the faster + * will be the benchmark. Basing the measurement's runtime on the clock resolution is the main reason why nanobench is so fast. + * + * * `{{maxEpochTime}}` Configuration for a maximum time each measurement (epoch) is allowed to take. Note that at least + * a single iteration will be performed, even when that takes longer than maxEpochTime. See Bench::maxEpochTime. + * + * * `{{minEpochTime}}` Minimum epoch time, defaults to 1ms. See Bench::minEpochTime. + * + * * `{{minEpochIterations}}` See Bench::minEpochIterations. + * + * * `{{epochIterations}}` See Bench::epochIterations. + * + * * `{{warmup}}` Number of iterations used before measuring starts. See Bench::warmup. + * + * * `{{relative}}` True or false, depending on the setting you have used. See Bench::relative. + * + * * `{{context(variableName)}}` See Bench::context. + * + * Apart from these tags, it is also possible to use some mathematical operations on the measurement data. The operations + * are of the form `{{command(name)}}`. Currently `name` can be one of `elapsed`, `iterations`. If performance counters + * are available (currently only on current Linux systems), you also have `pagefaults`, `cpucycles`, + * `contextswitches`, `instructions`, `branchinstructions`, and `branchmisses`. All the measures (except `iterations`) are + * provided for a single iteration (so `elapsed` is the time a single iteration took). The following tags are available: + * + * `elapsed` is in **seconds**, which is rarely the unit a report wants. Since the template language has no arithmetic + * to scale it afterwards, the time measure also comes in `elapsedms`, `elapsedus` and `elapsedns` — the same value in + * milliseconds, microseconds and nanoseconds. So `{{minimum(elapsedms)}}` is the fastest iteration in milliseconds. + * Note this is unrelated to Bench::timeUnit(), which only sets the unit of the `ns/op` column in the table. + * `{{medianAbsolutePercentError(...)}}` is a relative error, so it is the same number whichever of them you ask for. + * + * * `{{median()}}` Calculate median of a measurement data set, e.g. `{{median(elapsed)}}`. + * + * * `{{average()}}` Average (mean) calculation. + * + * * `{{medianAbsolutePercentError()}}` Calculates MdAPE, the Median Absolute Percentage Error. The MdAPE is an excellent + * metric for the variation of measurements. It is more robust to outliers than the + * [Mean absolute percentage error (M-APE)](https://en.wikipedia.org/wiki/Mean_absolute_percentage_error). + * @f[ + * \mathrm{MdAPE}(e) = \mathrm{med}\{| \frac{e_i - \mathrm{med}\{e\}}{e_i}| \} + * @f] + * E.g. for *elapsed*: First, @f$ \mathrm{med}\{e\} @f$ calculates the median by sorting and then taking the middle element + * of all *elapsed* measurements. This is used to calculate the absolute percentage + * error to this median for each measurement, as in @f$ | \frac{e_i - \mathrm{med}\{e\}}{e_i}| @f$. All these results + * are sorted, and the middle value is chosen as the median absolute percent error. + * + * This measurement is a bit hard to interpret, but it is very robust against outliers. E.g. a value of 5% means that half of the + * measurements deviate less than 5% from the median, and the other deviate more than 5% from the median. + * + * * `{{sum()}}` Sum of all the measurements. E.g. `{{sum(iterations)}}` will give you the total number of iterations +* measured in this benchmark. + * + * * `{{minimum()}}` Minimum of all measurements. + * + * * `{{maximum()}}` Maximum of all measurements. + * + * * `{{sumProduct(, )}}` Calculates the sum of the products of corresponding measures: + * @f[ + * \mathrm{sumProduct}(a,b) = \sum_{i=1}^{n}a_i\cdot b_i + * @f] + * E.g. to calculate total runtime of the benchmark, you multiply iterations with elapsed time for each measurement, and + * sum these results up: + * `{{sumProduct(iterations, elapsed)}}`. + * + * * `{{#measurement}}` To access individual measurement results, open the begin tag for measurements. + * + * * `{{elapsed}}` Average elapsed wall clock time per iteration, in seconds. + * + * * `{{iterations}}` Number of iterations in the measurement. The number of iterations will fluctuate due + * to some applied randomness, to enhance accuracy. + * + * * `{{pagefaults}}` Average number of pagefaults per iteration. + * + * * `{{cpucycles}}` Average number of CPU cycles processed per iteration. + * + * * `{{contextswitches}}` Average number of context switches per iteration. + * + * * `{{instructions}}` Average number of retired instructions per iteration. + * + * * `{{branchinstructions}}` Average number of branches executed per iteration. + * + * * `{{branchmisses}}` Average number of branches that were missed per iteration. + * + * * `{{/measurement}}` Ends the measurement tag. + * + * * `{{/result}}` Marks the end of the result layer. This is the end marker for the template part that will be instantiated + * for each benchmark result. + * + * + * For the layer tags *result* and *measurement* you additionally can use these special markers: + * + * * ``{{#-first}}`` - Begin marker of a template that will be instantiated *only for the first* entry in the layer. Use is only + * allowed between the begin and end marker of the layer. So between ``{{#result}}`` and ``{{/result}}``, or between + * ``{{#measurement}}`` and ``{{/measurement}}``. Finish the template with ``{{/-first}}``. + * + * * ``{{^-first}}`` - Begin marker of a template that will be instantiated *for each except the first* entry in the layer. This, + * this is basically the inversion of ``{{#-first}}``. Use is only allowed between the begin and end marker of the layer. + * So between ``{{#result}}`` and ``{{/result}}``, or between ``{{#measurement}}`` and ``{{/measurement}}``. + * + * * ``{{/-first}}`` - End marker for either ``{{#-first}}`` or ``{{^-first}}``. + * + * * ``{{#-last}}`` - Begin marker of a template that will be instantiated *only for the last* entry in the layer. Use is only + * allowed between the begin and end marker of the layer. So between ``{{#result}}`` and ``{{/result}}``, or between + * ``{{#measurement}}`` and ``{{/measurement}}``. Finish the template with ``{{/-last}}``. + * + * * ``{{^-last}}`` - Begin marker of a template that will be instantiated *for each except the last* entry in the layer. This, + * this is basically the inversion of ``{{#-last}}``. Use is only allowed between the begin and end marker of the layer. + * So between ``{{#result}}`` and ``{{/result}}``, or between ``{{#measurement}}`` and ``{{/measurement}}``. + * + * * ``{{/-last}}`` - End marker for either ``{{#-last}}`` or ``{{^-last}}``. + * + @verbatim embed:rst + + For an overview of all the possible data you can get out of nanobench, please see the tutorial at :ref:`tutorial-template-json`. + + The templates that ship with nanobench are: + + * :cpp:func:`templates::csv() ` + * :cpp:func:`templates::json() ` + * :cpp:func:`templates::htmlBoxplot() ` + * :cpp:func:`templates::pyperf() ` + + @endverbatim + * + * @param mustacheTemplate The template. + * @param bench Benchmark, containing all the results. + * @param out Output for the generated output. + */ +void render(char const* mustacheTemplate, Bench const& bench, std::ostream& out); +void render(std::string const& mustacheTemplate, Bench const& bench, std::ostream& out); + +/** + * Same as render(char const* mustacheTemplate, Bench const& bench, std::ostream& out), but for when + * you only have results available. + * + * @param mustacheTemplate The template. + * @param results All the results to be used for rendering. + * @param out Output for the generated output. + */ +void render(char const* mustacheTemplate, std::vector const& results, std::ostream& out); +void render(std::string const& mustacheTemplate, std::vector const& results, std::ostream& out); + +/** + * @brief Same, for the outcome of a Bench::compare(). + * + @verbatim embed:rst + The section is ``{{#alternative}}``, one iteration per alternative with the baseline first, and + inside it the ratio tags ``{{relative}}``, ``{{relativeLow}}``, ``{{relativeHigh}}``, + ``{{tiedRounds}}`` and ``{{significant}}`` are available on top of everything a result tag + already offers, so ``{{median(elapsed)}}`` and ``{{title}}`` work as they do elsewhere. + ``{{rounds}}`` and ``{{comparisons}}`` describe the experiment and can be used anywhere. + + Without this a comparison could only be read by a human out of the printed table: + :cpp:func:`templates::csv() ` and its neighbours take a + :cpp:class:`Bench ` or a vector of + :cpp:class:`Result `, neither of which carries a ratio or its + interval. See :ref:`tutorial-template-compare-csv`. + @endverbatim + */ +void render(char const* mustacheTemplate, CompareResult const& cmp, std::ostream& out); +void render(std::string const& mustacheTemplate, CompareResult const& cmp, std::ostream& out); + +// Contains mustache-like templates +namespace templates { + +/*! + @brief CSV data for the benchmark results. + + Generates a comma-separated values dataset. First line is the header, each following line is a summary of each benchmark run. + + @verbatim embed:rst + See the tutorial at :ref:`tutorial-template-csv` for an example. + @endverbatim + */ +char const* csv() noexcept; + +/*! + @brief CSV data for a paired comparison, one row per alternative. + + Carries the ratio and the bounds of its confidence interval, which is what separates a comparison + from a set of independent runs. Meant for ``render(templates::compareCsv(), cmp, out)``. + + @verbatim embed:rst + See the tutorial at :ref:`tutorial-template-compare-csv` for an example. + @endverbatim + */ +char const* compareCsv() noexcept; + +/*! + @brief HTML output that uses plotly to generate an interactive boxplot chart. See the tutorial for an example output. + + The output uses only the elapsed wall clock time, and displays each epoch as a single dot. + @verbatim embed:rst + See the tutorial at :ref:`tutorial-template-html` for an example. + @endverbatim + + @see also ankerl::nanobench::render() + */ +char const* htmlBoxplot() noexcept; + +/*! + @brief Output in pyperf compatible JSON format, which can be used for more analyzation. + @verbatim embed:rst + See the tutorial at :ref:`tutorial-template-pyperf` for an example how to further analyze the output. + @endverbatim + */ +char const* pyperf() noexcept; + +/*! + @brief Template to generate JSON data. + + The generated JSON data contains *all* data that has been generated. All times are as double values, in seconds. The output can get + quite large. + @verbatim embed:rst + See the tutorial at :ref:`tutorial-template-json` for an example. + @endverbatim + */ +char const* json() noexcept; + +} // namespace templates + +namespace detail { + +template +struct PerfCountSet; + +class IterationLogic; +class PerformanceCounters; + +#if ANKERL_NANOBENCH(PERF_COUNTERS) +class LinuxPerformanceCounters; +#endif + +} // namespace detail +} // namespace nanobench +} // namespace ankerl + +// definitions //////////////////////////////////////////////////////////////////////////////////// + +namespace ankerl { +namespace nanobench { +namespace detail { + +template +struct PerfCountSet { + T pageFaults{}; + T cpuCycles{}; + T contextSwitches{}; + T instructions{}; + T branchInstructions{}; + T branchMisses{}; +}; + +} // namespace detail + +ANKERL_NANOBENCH(IGNORE_PADDED_PUSH) +struct Config { + // actual benchmark config + std::string mBenchmarkTitle = "benchmark"; // NOLINT(misc-non-private-member-variables-in-classes) + std::string mBenchmarkName = "noname"; // NOLINT(misc-non-private-member-variables-in-classes) + std::string mUnit = "op"; // NOLINT(misc-non-private-member-variables-in-classes) + double mBatch = 1.0; // NOLINT(misc-non-private-member-variables-in-classes) + double mComplexityN = -1.0; // NOLINT(misc-non-private-member-variables-in-classes) + size_t mNumEpochs = 11; // NOLINT(misc-non-private-member-variables-in-classes) + size_t mClockResolutionMultiple = static_cast(1000); // NOLINT(misc-non-private-member-variables-in-classes) + std::chrono::nanoseconds mMaxEpochTime = std::chrono::milliseconds(100); // NOLINT(misc-non-private-member-variables-in-classes) + std::chrono::nanoseconds mMinEpochTime = std::chrono::milliseconds(1); // NOLINT(misc-non-private-member-variables-in-classes) + uint64_t mMinEpochIterations{1}; // NOLINT(misc-non-private-member-variables-in-classes) + // If not 0, run *exactly* these number of iterations per epoch. + uint64_t mEpochIterations{0}; // NOLINT(misc-non-private-member-variables-in-classes) + uint64_t mWarmup = 0; // NOLINT(misc-non-private-member-variables-in-classes) + std::ostream* mOut = nullptr; // NOLINT(misc-non-private-member-variables-in-classes) + std::chrono::duration mTimeUnit = std::chrono::nanoseconds{1}; // NOLINT(misc-non-private-member-variables-in-classes) + std::string mTimeUnitName = "ns"; // NOLINT(misc-non-private-member-variables-in-classes) + bool mShowPerformanceCounters = true; // NOLINT(misc-non-private-member-variables-in-classes) + bool mIsRelative = false; // NOLINT(misc-non-private-member-variables-in-classes) + std::unordered_map mContext{}; // NOLINT(misc-non-private-member-variables-in-classes) + // One bit per Column, set when that column is hidden - so the default of 0 shows everything and + // stays the table nanobench has always printed. + uint32_t mHiddenColumns{}; // NOLINT(misc-non-private-member-variables-in-classes) + std::vector mContextColumns{}; // NOLINT(misc-non-private-member-variables-in-classes) + // 0 disables: compare() then runs exactly mNumEpochs rounds, as it always has. + double mTargetIntervalWidth{0.0}; // NOLINT(misc-non-private-member-variables-in-classes) + size_t mMaxEpochs{1000}; // NOLINT(misc-non-private-member-variables-in-classes) + + Config(); + ~Config(); + Config& operator=(Config const& other); + Config& operator=(Config&& other) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)); + Config(Config const& other); + Config(Config&& other) noexcept; +}; +ANKERL_NANOBENCH(IGNORE_PADDED_POP) + +// Result returned after a benchmark has finished. Can be used as a baseline for relative(). +ANKERL_NANOBENCH(IGNORE_PADDED_PUSH) +class Result { +public: + enum class Measure : size_t { + elapsed, + iterations, + pagefaults, + cpucycles, + contextswitches, + instructions, + branchinstructions, + branchmisses, + + /// Number of measures, and what fromString() returns for a name it does not know. Passing it + /// to the accessors below is not an error: it reads as a measure that was never recorded. + _size + }; + + explicit Result(Config benchmarkConfig); + + ~Result(); + Result& operator=(Result const& other); + Result& operator=(Result&& other) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)); + Result(Result const& other); + Result(Result&& other) noexcept; + + // adds new measurement results + // all values are scaled by iters (except iters...) + void add(Clock::duration totalElapsed, uint64_t iters, detail::PerformanceCounters const& pc); + + ANKERL_NANOBENCH(NODISCARD) Config const& config() const noexcept; + + ANKERL_NANOBENCH(NODISCARD) double median(Measure m) const; + ANKERL_NANOBENCH(NODISCARD) double medianAbsolutePercentError(Measure m) const; + ANKERL_NANOBENCH(NODISCARD) double average(Measure m) const; + ANKERL_NANOBENCH(NODISCARD) double sum(Measure m) const noexcept; + ANKERL_NANOBENCH(NODISCARD) double sumProduct(Measure m1, Measure m2) const noexcept; + ANKERL_NANOBENCH(NODISCARD) double minimum(Measure m) const noexcept; + ANKERL_NANOBENCH(NODISCARD) double maximum(Measure m) const noexcept; + ANKERL_NANOBENCH(NODISCARD) std::string const& context(char const* variableName) const; + ANKERL_NANOBENCH(NODISCARD) std::string const& context(std::string const& variableName) const; + + ANKERL_NANOBENCH(NODISCARD) bool has(Measure m) const noexcept; + ANKERL_NANOBENCH(NODISCARD) double get(size_t idx, Measure m) const; + ANKERL_NANOBENCH(NODISCARD) bool empty() const noexcept; + ANKERL_NANOBENCH(NODISCARD) size_t size() const noexcept; + + // Finds string, if not found, returns _size. + static Measure fromString(std::string const& str); + +private: + // The per-epoch values of one measure. Every accessor above goes through this, so how the + // measures are stored - including the extra slot the constructor explains - is written down once + // rather than asserted at a dozen call sites. + ANKERL_NANOBENCH(NODISCARD) std::vector const& measurements(Measure m) const; + + Config mConfig{}; + std::vector> mNameToMeasurements{}; +}; +ANKERL_NANOBENCH(IGNORE_PADDED_POP) + +/** + * @brief The outcome of a paired comparison, see Bench::compare(). + * + @verbatim embed:rst + :cpp:func:`relative() ` compares the median of one benchmark + run entirely before another, which measures the machine's drift as much as the code: nanobench's + own test suite has a case where two *identical* workloads came out 38% apart on a CI runner, while + each reported an ``err%`` of 0.5. Whatever ``err%`` is measuring there, it is not the uncertainty + of the comparison. + + A paired comparison measures the alternatives against each other within the same slice of time, so + anything that affects all of them - a frequency ramp, a noisy neighbour, thermal throttling - + cancels out of the ratios. What is reported is therefore an uncertainty about *the ratio*, which + is the number the caller actually wanted. + + Entry 0 is the baseline that everything else is measured against. + @endverbatim + */ +ANKERL_NANOBENCH(IGNORE_PADDED_PUSH) +class CompareResult { +public: + /// One alternative: its measurements, and how it compares to the baseline. + ANKERL_NANOBENCH(IGNORE_PADDED_PUSH) + struct Entry { + // noexcept because it only moves, and both std::string's and Result's move constructors are: + // gcc's -Wnoexcept fails the build over the vector growth path otherwise. + Entry(std::string entryName, Result entryResult, double entryRelative, double entryRelativeLow, double entryRelativeHigh, + size_t entryTiedRounds) noexcept; + + std::string name; // NOLINT(misc-non-private-member-variables-in-classes) + Result result; // NOLINT(misc-non-private-member-variables-in-classes) + double relative; // NOLINT(misc-non-private-member-variables-in-classes) + double relativeLow; // NOLINT(misc-non-private-member-variables-in-classes) + double relativeHigh; // NOLINT(misc-non-private-member-variables-in-classes) + size_t tiedRounds; // NOLINT(misc-non-private-member-variables-in-classes) + }; + ANKERL_NANOBENCH(IGNORE_PADDED_POP) + + CompareResult(std::vector entries, size_t numRounds); + + /// Number of alternatives compared, including the baseline. + ANKERL_NANOBENCH(NODISCARD) size_t size() const noexcept; + + /// Entry 0 is the baseline. + ANKERL_NANOBENCH(NODISCARD) Entry const& operator[](size_t idx) const; + + /// Number of paired rounds each alternative was measured over. + ANKERL_NANOBENCH(NODISCARD) size_t rounds() const noexcept; + + /** + * @brief How many comparisons the intervals were corrected for: one per alternative besides the + * baseline. + * + @verbatim embed:rst + With ten alternatives there are nine chances to be wrong at 5% each, so the intervals are + widened to keep the *whole table* at 95% rather than each row separately. See + :ref:`ab-comparison` for what that costs - less than it sounds, because the binomial tail is + steep. + @endverbatim + */ + ANKERL_NANOBENCH(NODISCARD) size_t comparisons() const noexcept; + + /** + * @brief True when this alternative's interval excludes the baseline, i.e. the measurement told + * them apart. Always false for the baseline itself. + */ + ANKERL_NANOBENCH(NODISCARD) bool isSignificant(size_t idx) const; + + /// Index of the alternative with the lowest median time. + ANKERL_NANOBENCH(NODISCARD) size_t fastest() const; + +private: + std::vector mEntries{}; + size_t mRounds{}; +}; +ANKERL_NANOBENCH(IGNORE_PADDED_POP) + +/// Writes the comparison as a markdown table, in the same shape as an ordinary benchmark table, with +/// a summary line under it. +std::ostream& operator<<(std::ostream& os, CompareResult const& compareResult); + +/** + * An extremely fast random generator. Currently, this implements *RomuDuoJr*, developed by Mark Overton. Source: + * http://www.romu-random.org/ + * + * RomuDuoJr is extremely fast and provides reasonable good randomness. Not enough for large jobs, but definitely + * good enough for a benchmarking framework. + * + * * Estimated capacity: @f$ 2^{51} @f$ bytes + * * Register pressure: 4 + * * State size: 128 bits + * + * This random generator is a drop-in replacement for the generators supplied by ````. It is not + * cryptographically secure. It's intended purpose is to be very fast so that benchmarks that make use + * of randomness are not distorted too much by the random generator. + * + * Rng also provides a few non-standard helpers, optimized for speed. + */ +class Rng final { +public: + /** + * @brief This RNG provides 64bit randomness. + */ + using result_type = uint64_t; + + static constexpr uint64_t(min)(); + static constexpr uint64_t(max)(); + + /** + * As a safety precaution, we don't allow copying. Copying a PRNG would mean you would have two random generators that produce the + * same sequence, which is generally not what one wants. Instead create a new rng with the default constructor Rng(), which is + * automatically seeded from `std::random_device`. If you really need a copy, use `copy()`. + */ + Rng(Rng const&) = delete; + + /** + * Same as Rng(Rng const&), we don't allow assignment. If you need a new Rng create one with the default constructor Rng(). + */ + Rng& operator=(Rng const&) = delete; + + // moving is ok + Rng(Rng&&) noexcept = default; + Rng& operator=(Rng&&) noexcept = default; + ~Rng() noexcept = default; + + /** + * @brief Creates a new Random generator with random seed. + * + * Instead of a default seed (as the random generators from the STD), this properly seeds the random generator from + * `std::random_device`. It guarantees correct seeding. Note that seeding can be relatively slow, depending on the source of + * randomness used. So it is best to create a Rng once and use it for all your randomness purposes. + */ + Rng(); + + /*! + Creates a new Rng that is seeded with a specific seed. Each Rng created from the same seed will produce the same randomness + sequence. This can be useful for deterministic behavior. + + @verbatim embed:rst + .. note:: + + The random algorithm might change between nanobench releases. Whenever a faster and/or better random + generator becomes available, I will switch the implementation. + @endverbatim + + As per the Romu paper, this seeds the Rng with splitMix64 algorithm and performs 10 initial rounds for further mixing up of the + internal state. + + @param seed The 64bit seed. All values are allowed, even 0. + */ + explicit Rng(uint64_t seed) noexcept; + Rng(uint64_t x, uint64_t y) noexcept; + explicit Rng(std::vector const& data); + + /** + * Creates a copy of the Rng, thus the copy provides exactly the same random sequence as the original. + */ + ANKERL_NANOBENCH(NODISCARD) Rng copy() const noexcept; + + /** + * @brief Produces a 64bit random value. This should be very fast, thus it is marked as inline. In my benchmark, this is ~46 times + * faster than `std::default_random_engine` for producing 64bit random values. It seems that the fastest std contender is + * `std::mt19937_64`. Still, this RNG is 2-3 times as fast. + * + * @return uint64_t The next 64 bit random value. + */ + inline uint64_t operator()() noexcept; + + /** + * Generates a random number between 0 and range (excluding range). + * + * The algorithm only produces 32bit numbers, and is slightly biased. The effect is quite small unless your range is close to the + * maximum value of an integer. It is possible to correct the bias with rejection sampling (see + * [here](https://lemire.me/blog/2016/06/30/fast-random-shuffling/), but this is most likely irrelevant in practices for the + * purposes of this Rng. + * + * See Daniel Lemire's blog post [A fast alternative to the modulo + * reduction](https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/) + * + * @param range Upper exclusive range. E.g a value of 3 will generate random numbers 0, 1, 2. + * @return uint32_t Generated random values in range [0, range(. + */ + inline uint32_t bounded(uint32_t range) noexcept; + + /** + * Provides a random uniform double value between 0 and 1. This uses the method described in [Generating uniform doubles in the + * unit interval](http://prng.di.unimi.it/), and is extremely fast. + * + * @return double Uniformly distributed double value in range [0,1(, excluding 1. + */ + inline double uniform01() noexcept; + + /** + * Shuffles all entries in the given container. Although this has a slight bias due to the implementation of bounded(), this is + * preferable to `std::shuffle` because it is over 5 times faster. See Daniel Lemire's blog post [Fast random + * shuffling](https://lemire.me/blog/2016/06/30/fast-random-shuffling/). + * + * @param container The whole container will be shuffled. + */ + template + void shuffle(Container& container) noexcept; + + /** + * Extracts the full state of the generator, e.g. for serialization. For this RNG this is just 2 values, but to stay API compatible + * with future implementations that potentially use more state, we use a vector. + * + * @return Vector containing the full state: + */ + ANKERL_NANOBENCH(NODISCARD) std::vector state() const; + +private: + static constexpr uint64_t rotl(uint64_t x, unsigned k) noexcept; + + uint64_t mX; + uint64_t mY; +}; + +/** + * @brief A column of the markdown table, for Bench::hideColumn() and Bench::showColumn(). + * + * The names in brackets are the headers the column is printed with; `op` follows Bench::unit(), and the + * time unit follows Bench::timeUnit(). + * + * @see ankerl::nanobench::Bench::hideColumn() + */ +enum class Column : size_t { + relative, ///< `relative` - only shown when Bench::relative() is set. + complexityN, ///< `complexityN` - only shown when Bench::complexityN() is set. + timePerUnit, ///< `ns/op` - time for one unit. + unitPerSecond, ///< `op/s` - units per second. + error, ///< `err%` - median absolute percentage error over the epochs. + instructions, ///< `ins/op` - retired instructions, Linux only. + cycles, ///< `cyc/op` - CPU cycles, Linux only. + ipc, ///< `IPC` - instructions per cycle, Linux only. + branches, ///< `bra/op` - retired branch instructions, Linux only. + branchMisses, ///< `miss%` - percentage of branches mispredicted, Linux only. + total, ///< `total` - wall clock time spent measuring this row. + _size ///< Not a column; the number of them. +}; + +/** + * @brief Main entry point to nanobench's benchmarking facility. + * + * It holds configuration and results from one or more benchmark runs. Usually it is used in a single line, where the object is + * constructed, configured, and then a benchmark is run. E.g. like this: + * + * ankerl::nanobench::Bench().unit("byte").batch(1000).run("random fluctuations", [&] { + * // here be the benchmark code + * }); + * + * In that example Bench() constructs the benchmark, it is then configured with unit() and batch(), and after configuration a + * benchmark is executed with run(). Once run() has finished, it prints the result to `std::cout`. It would also store the results + * in the Bench instance, but in this case the object is immediately destroyed so it's not available any more. + */ +ANKERL_NANOBENCH(IGNORE_PADDED_PUSH) +class Bench { +public: + /** + * @brief Creates a new benchmark for configuration and running of benchmarks. + */ + Bench(); + + Bench(Bench&& other) noexcept; + Bench& operator=(Bench&& other) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)); + Bench(Bench const& other); + Bench& operator=(Bench const& other); + ~Bench() noexcept; + + /*! + @brief Repeatedly calls `op()` based on the configuration, and performs measurements. + + This call is marked with `noinline` to prevent the compiler to optimize beyond different benchmarks. This can have quite a big + effect on benchmark accuracy. + + @verbatim embed:rst + .. note:: + + Each call to your lambda must have a side effect that the compiler can't possibly optimize it away. E.g. add a result to an + externally defined number (like `x` in the above example), and finally call `doNotOptimizeAway` on the variables the compiler + must not remove. You can also use :cpp:func:`ankerl::nanobench::doNotOptimizeAway` directly in the lambda, but be aware that + this has a small overhead. + + @endverbatim + + @tparam Op The code to benchmark. + */ + template + ANKERL_NANOBENCH(NOINLINE) + Bench& run(char const* benchmarkName, Op&& op); + + template + ANKERL_NANOBENCH(NOINLINE) + Bench& run(std::string const& benchmarkName, Op&& op); + +#if ANKERL_NANOBENCH(HAS_STRING_VIEW) + /// Same as run(char const* benchmarkName, Op op), for a std::string_view name. Only available + /// when compiling as C++17 or newer. + template + ANKERL_NANOBENCH(NOINLINE) + Bench& run(std::string_view benchmarkName, Op&& op); +#endif + + /** + * @brief Same as run(char const* benchmarkName, Op op), but instead uses the previously set name. + * @tparam Op The code to benchmark. + */ + template + ANKERL_NANOBENCH(NOINLINE) + Bench& run(Op&& op); + + /** + * @brief Compares alternatives against each other, paired and interleaved. + * + * Takes `name, op` pairs - two or more of them. The first is the baseline everything else is + * measured against, exactly as with relative(), but measured against it *at the same time* + * rather than one after the other. Drift the machine introduces - a frequency ramp, a noisy + * neighbour, thermal throttling - then hits every alternative alike and cancels out of the + * ratios. + * + * @code + * ankerl::nanobench::Bench().epochs(52).compare( + * "std::mt19937", [&] { ... }, // the baseline + * "sfc4", [&] { ... }, + * "romu", [&] { ... }); + * @endcode + * + * Prints an ordinary nanobench table with two extra columns - the ratio to the baseline and a + * confidence interval for it - and a summary line underneath. + * + @verbatim embed:rst + A worked example with output, and the reasoning behind each choice below, is in the tutorial at + :ref:`ab-comparison`. + + What it does, in order: + + #. Calibrates an iteration count once, up front, and uses the **same** count for every + alternative for the whole run. An epoch's fixed overhead is divided by that count, so two + different counts would amortize it differently and bias the ratio - 1.2% on 200us epochs, and + no amount of pairing removes it. The shared count is the slowest alternative's, raised where + that would leave the fastest one's epoch too short for the clock to resolve, and capped so + the slowest one still fits in maxEpochTime(). + #. Runs one epoch of each per round, interleaved, so drift is common to the round. + #. Orders each block of N rounds as a randomly chosen cyclic Latin square, so every alternative + occupies every position exactly once per block and their mean positions are equal - which is + what cancels a drift that is linear over the block. For two alternatives this is exactly + ABBA/BAAB. + #. Reduces each round to ``ln(t_baseline) - ln(t_alternative)``, dropping rounds where either + side measured zero. + #. Reports the median of those as the ratio, with a sign-test interval around it, widened to + keep the whole table at 95% rather than each row separately. + + .. note:: + + Use more rounds than the default 11. An epoch is about a millisecond, so ``epochs(52)`` costs + a tenth of a second per alternative and buys an interval narrow enough to act on. The count is + rounded up to a whole number of blocks, and raised if it is too small to support an interval + at all. + + .. warning:: + + Interleaving means each alternative runs with the others' cache and branch predictor state. + That is usually the more honest measurement for "which should I ship", but it is a different + measurement from running one alone - use :cpp:func:`run() ` + for that. + + @endverbatim + * + * @tparam Args Alternating names and operations. + * @param args `name, op` pairs; at least two. + * @return The ratios and their intervals; also written to output() unless that is nullptr. + */ + template + ANKERL_NANOBENCH(NOINLINE) + CompareResult compare(Args&&... args); + + /** + * @brief Title of the benchmark, will be shown in the table header. Changing the title will start a new markdown table. + * + * @param benchmarkTitle The title of the benchmark. + */ + Bench& title(char const* benchmarkTitle); + Bench& title(std::string const& benchmarkTitle); + + /** + * @brief Gets the title of the benchmark + */ + ANKERL_NANOBENCH(NODISCARD) std::string const& title() const noexcept; + + /// Name of the benchmark, will be shown in the table row. + Bench& name(char const* benchmarkName); + Bench& name(std::string const& benchmarkName); +#if ANKERL_NANOBENCH(HAS_STRING_VIEW) + /// Only available when compiling as C++17 or newer. @see name() + Bench& name(std::string_view benchmarkName); +#endif + ANKERL_NANOBENCH(NODISCARD) std::string const& name() const noexcept; + + /** + * @brief Set context information. + * + * The information can be accessed using custom render templates via `{{context(variableName)}}`. + * Trying to render a variable that hasn't been set before raises an exception. + * Not included in (default) markdown table. + * + * @see clearContext, render + * + * @param variableName The name of the context variable. + * @param variableValue The value of the context variable. + */ + Bench& context(char const* variableName, char const* variableValue); + Bench& context(std::string const& variableName, std::string const& variableValue); + + /** + * @brief Reset context information. + * + * This may improve efficiency when using many context entries, + * or improve robustness by removing spurious context entries. + * + * @see context + */ + Bench& clearContext(); + + /** + * @brief Sets the batch size. + * + * E.g. number of processed byte, or some other metric for the size of the processed data in each iteration. If you benchmark + * hashing of a 1000 byte long string and want byte/sec as a result, you can specify 1000 as the batch size. + * + * @tparam T Any input type is internally cast to `double`. + * @param b batch size + */ + template + Bench& batch(T b) noexcept; + ANKERL_NANOBENCH(NODISCARD) double batch() const noexcept; + + /** + * @brief Sets the operation unit. + * + * Defaults to "op". Could be e.g. "byte" for string processing. This is used for the table header, e.g. to show `ns/byte`. Use + * singular (*byte*, not *bytes*). A change clears the currently collected results. + * + * @param unit The unit name. + */ + Bench& unit(char const* unit); + Bench& unit(std::string const& unit); + ANKERL_NANOBENCH(NODISCARD) std::string const& unit() const noexcept; + + /** + * @brief Sets the time unit to be used for the default output. + * + * Nanobench defaults to using ns (nanoseconds) as output in the markdown. For some benchmarks this is too coarse, so it is + * possible to configure this. E.g. use `timeUnit(1ms, "ms")` to show `ms/op` instead of `ns/op`. + * + * @param tu Time unit to display the results in, default is 1ns. + * @param tuName Name for the time unit, default is "ns" + */ + Bench& timeUnit(std::chrono::duration const& tu, std::string const& tuName); + ANKERL_NANOBENCH(NODISCARD) std::string const& timeUnitName() const noexcept; + ANKERL_NANOBENCH(NODISCARD) std::chrono::duration const& timeUnit() const noexcept; + + /** + * @brief Set the output stream where the resulting markdown table will be printed to. + * + * The default is `&std::cout`. You can disable all output by setting `nullptr`. + * + * @param outstream Pointer to output stream, can be `nullptr`. + */ + Bench& output(std::ostream* outstream) noexcept; + ANKERL_NANOBENCH(NODISCARD) std::ostream* output() const noexcept; + + /** + * Modern processors have a very accurate clock, being able to measure as low as 20 nanoseconds. This is the main trick nanobech to + * be so fast: we find out how accurate the clock is, then run the benchmark only so often that the clock's accuracy is good enough + * for accurate measurements. + * + * The default is to run one epoch for 1000 times the clock resolution. So for 20ns resolution and 11 epochs, this gives a total + * runtime of + * + * @f[ + * 20ns * 1000 * 11 \approx 0.2ms + * @f] + * + * To be precise, nanobench adds a 0-20% random noise to each evaluation. This is to prevent any aliasing effects, and further + * improves accuracy. + * + * Total runtime will be higher though: Some initial time is needed to find out the target number of iterations for each epoch, and + * there is some overhead involved to start & stop timers and calculate resulting statistics and writing the output. + * + * @param multiple Target number of times of clock resolution. Usually 1000 is a good compromise between runtime and accuracy. + */ + Bench& clockResolutionMultiple(size_t multiple) noexcept; + ANKERL_NANOBENCH(NODISCARD) size_t clockResolutionMultiple() const noexcept; + + /** + * @brief Controls number of epochs, the number of measurements to perform. + * + * The reported result will be the median of evaluation of each epoch. The higher you choose this, the more + * deterministic the result be and outliers will be more easily removed. Also the `err%` will be more accurate the higher this + * number is. Note that the `err%` will not necessarily decrease when number of epochs is increased. But it will be a more accurate + * representation of the benchmarked code's runtime stability. + * + * Choose the value wisely. In practice, 11 has been shown to be a reasonable choice between runtime performance and accuracy. + * This setting goes hand in hand with minEpochIterations() (or minEpochTime()). If you are more interested in *median* runtime, + * you might want to increase epochs(). If you are more interested in *mean* runtime, you might want to increase + * minEpochIterations() instead. + * + * @param numEpochs Number of epochs. + */ + Bench& epochs(size_t numEpochs) noexcept; + ANKERL_NANOBENCH(NODISCARD) size_t epochs() const noexcept; + + /** + * @brief Asks Bench::compare() to keep measuring until the ratio is known this precisely. + * + @verbatim embed:rst + The width is in log space, which for the small widths worth asking for is the width of the + interval as a fraction of the ratio: ``0.05`` means the ratio is pinned to about ±2.5%. Zero, + the default, disables it and :cpp:func:`epochs() ` alone + decides the count, as before. + + How many rounds a target takes cannot be known in advance -- it depends on how noisy the + machine is -- and the default 11 is often far too few: it leaves the interval on a ratio + about 25% wide, which cannot tell a 10% difference from nothing. Guessing the number by hand + means guessing again on the next machine. + + So compare() runs **two stages**. The first is ``epochs()`` rounds and is used only to see how + noisy the measurement is; the second runs as many rounds as that implies, and **the reported + result comes from the second stage alone**. Splitting them is what keeps the statistics + honest: a rule that watched one growing sample and stopped as soon as it looked tight enough + would stop preferentially on the samples that happen to look tight, and the interval would + then cover less often than it claims. Here the stopping decision is made from data that is + then discarded, so the second stage's interval is exactly as good as its confidence says. The + price of that is the pilot, a fixed ``epochs()`` rounds. + + Simulated over 4000 trials against skewed noise, asking for a width of 0.2 at a nominal 95%: + two stages cover **96.0%** of the time, and the same rule watching one growing sample covers + **92.9%** while using half the rounds. The first is the sign test's usual slight + over-coverage from discreteness; the second is the interval quietly meaning less than it + says. + + A machine too noisy to reach the target within + :cpp:func:`maxEpochs() ` stops there and reports the + width it did reach, rather than running forever or pretending. + @endverbatim + * + * @param width Interval width to aim for, in log space. 0 disables. + */ + Bench& targetIntervalWidth(double width) noexcept; + ANKERL_NANOBENCH(NODISCARD) double targetIntervalWidth() const noexcept; + + /** + * @brief Upper limit on the rounds Bench::compare() runs when targetIntervalWidth() is set. + * Default 1000. + */ + Bench& maxEpochs(size_t numEpochs) noexcept; + ANKERL_NANOBENCH(NODISCARD) size_t maxEpochs() const noexcept; + + /** + * @brief Upper limit for the runtime of each epoch. + * + * As a safety precaution if the clock is not very accurate, we can set an upper limit for the maximum evaluation time per + * epoch. Default is 100ms. At least a single evaluation of the benchmark is performed. + * + * @see minEpochTime, minEpochIterations + * + * @param t Maximum target runtime for a single epoch. + */ + Bench& maxEpochTime(std::chrono::nanoseconds t) noexcept; + ANKERL_NANOBENCH(NODISCARD) std::chrono::nanoseconds maxEpochTime() const noexcept; + + /** + * @brief Minimum time each epoch should take. + * + * Default is 1ms, so we are mostly relying on clockResolutionMultiple(). In most cases this is exactly what you want. If you see + * that the evaluation is unreliable with a high `err%`, you can increase either minEpochTime() or minEpochIterations(). + * + * @see maxEpochTime, minEpochIterations + * + * @param t Minimum time each epoch should take. + */ + Bench& minEpochTime(std::chrono::nanoseconds t) noexcept; + ANKERL_NANOBENCH(NODISCARD) std::chrono::nanoseconds minEpochTime() const noexcept; + + /** + * @brief Sets the minimum number of iterations each epoch should take. + * + * Default is 1, and we rely on clockResolutionMultiple(). If the `err%` is high and you want a more smooth result, you might want + * to increase the minimum number of iterations, or increase the minEpochTime(). + * + * @see minEpochTime, maxEpochTime, minEpochIterations + * + * @param numIters Minimum number of iterations per epoch. + */ + Bench& minEpochIterations(uint64_t numIters) noexcept; + ANKERL_NANOBENCH(NODISCARD) uint64_t minEpochIterations() const noexcept; + + /** + * Sets exactly the number of iterations for each epoch. Ignores all other epoch limits. This forces nanobench to use exactly + * the given number of iterations for each epoch, not more and not less. Default is 0 (disabled). + * + * @param numIters Exact number of iterations to use. Set to 0 to disable. + */ + Bench& epochIterations(uint64_t numIters) noexcept; + ANKERL_NANOBENCH(NODISCARD) uint64_t epochIterations() const noexcept; + + /** + * @brief Sets a number of iterations that are initially performed without any measurements. + * + * Some benchmarks need a few evaluations to warm up caches / database / whatever access. Normally this should not be needed, since + * we show the median result so initial outliers will be filtered away automatically. If the warmup effect is large though, you + * might want to set it. Default is 0. + * + * @param numWarmupIters Number of warmup iterations. + */ + Bench& warmup(uint64_t numWarmupIters) noexcept; + ANKERL_NANOBENCH(NODISCARD) uint64_t warmup() const noexcept; + + /** + * @brief Marks the next run as the baseline. + * + * Call `relative(true)` to mark the run as the baseline. Successive runs will be compared to this run. Just like all the + * other columns, the comparison is per unit, so it stays meaningful when the runs use a different Bench::batch: + * + * @f[ + * 100\% * \frac{baseline / baselineBatch}{runtime / batch} + * @f] + * + * * 100% means it is exactly as fast as the baseline + * * >100% means it is faster than the baseline. E.g. 200% means the current run is twice as fast as the baseline. + * * <100% means it is slower than the baseline. E.g. 50% means it is twice as slow as the baseline. + * + * See the tutorial section "Comparing Results" for example usage. + * + * @param isRelativeEnabled True to enable processing + */ + Bench& relative(bool isRelativeEnabled) noexcept; + ANKERL_NANOBENCH(NODISCARD) bool relative() const noexcept; + + /** + * @brief Enables/disables performance counters. + * + * On Linux nanobench has a powerful feature to use performance counters. This enables counting of retired instructions, count + * number of branches, missed branches, etc. On default this is enabled, but you can disable it if you don't need that feature. + * + * @param showPerformanceCounters True to enable, false to disable. + */ + Bench& performanceCounters(bool showPerformanceCounters) noexcept; + ANKERL_NANOBENCH(NODISCARD) bool performanceCounters() const noexcept; + + /** + * @brief Removes a column from the table. + * + * The full table is over 150 characters wide, which wraps in most terminals. Hide what you are not + * reading: + * + * bench.hideColumn(Column::instructions) + * .hideColumn(Column::cycles) + * .hideColumn(Column::ipc); + * + * Hiding a column only changes what is printed - the measurement still happens, and results() and + * the render templates are unaffected. To stop *measuring* the performance counters as well, use + * performanceCounters(false), which hides all five of their columns at once. + * + * @param column The column to hide. + */ + Bench& hideColumn(Column column) noexcept; + + /// Shows a column hidden with hideColumn() again. @see hideColumn() + Bench& showColumn(Column column) noexcept; + + /// True when @p column is not hidden. A column can still be absent from the table for other + /// reasons - `relative` without relative(), or the counter columns off Linux. @see hideColumn() + ANKERL_NANOBENCH(NODISCARD) bool isColumnVisible(Column column) const noexcept; + + /** + * @brief Adds a column showing the value of a context variable. + * + * Context variables set with context() are otherwise only reachable from a render template, which + * makes parameterised benchmarks awkward to read on the console. This puts one in the table: + * + * bench.context("threads", std::to_string(n)) + * .contextColumn("threads") + * .run(...); + * + * The column is added once per name, in the order given, before the benchmark name. A row whose + * context does not have the variable is left blank. + * + * @param variableName Name of the context variable to show. @see context() + */ + Bench& contextColumn(std::string const& variableName); + + /// Removes all columns added with contextColumn(). @see contextColumn() + Bench& clearContextColumns() noexcept; + + /** + * @brief Retrieves all benchmark results collected by the bench object so far. + * + * Each call to run() generates a Result that is stored within the Bench instance. This is mostly for advanced users who want to + * see all the nitty gritty details. + * + * @return All results collected so far. + */ + ANKERL_NANOBENCH(NODISCARD) std::vector const& results() const noexcept; + + /*! + @verbatim embed:rst + + Convenience shortcut to :cpp:func:`ankerl::nanobench::doNotOptimizeAway`. + + @endverbatim + */ + template + Bench& doNotOptimizeAway(Arg&& arg); + + /*! + @verbatim embed:rst + + Sets N for asymptotic complexity calculation, so it becomes possible to calculate `Big O + `_ from multiple benchmark evaluations. + + Use :cpp:func:`ankerl::nanobench::Bench::complexityBigO` when the evaluation has finished. See the tutorial + :ref:`asymptotic-complexity` for details. + + @endverbatim + + @tparam T Any type is cast to `double`. + @param n Length of N for the next benchmark run, so it is possible to calculate `bigO`. + */ + template + Bench& complexityN(T n) noexcept; + ANKERL_NANOBENCH(NODISCARD) double complexityN() const noexcept; + + /*! + Calculates [Big O](https://en.wikipedia.org/wiki/Big_O_notation>) of the results with all preconfigured complexity functions. + Currently these complexity functions are fitted into the benchmark results: + + @f$ \mathcal{O}(1) @f$, + @f$ \mathcal{O}(n) @f$, + @f$ \mathcal{O}(\log{}n) @f$, + @f$ \mathcal{O}(n\log{}n) @f$, + @f$ \mathcal{O}(n^2) @f$, + @f$ \mathcal{O}(n^3) @f$. + + If we e.g. evaluate the complexity of `std::sort`, this is the result of `std::cout << bench.complexityBigO()`: + + ``` + | coefficient | err% | complexity + |--------------:|-------:|------------ + | 5.08935e-09 | 2.6% | O(n log n) + | 6.10608e-08 | 8.0% | O(n) + | 1.29307e-11 | 47.2% | O(n^2) + | 2.48677e-15 | 69.6% | O(n^3) + | 9.88133e-06 | 132.3% | O(log n) + | 5.98793e-05 | 162.5% | O(1) + ``` + + So in this case @f$ \mathcal{O}(n\log{}n) @f$ provides the best approximation. + + @verbatim embed:rst + See the tutorial :ref:`asymptotic-complexity` for details. + @endverbatim + @return Evaluation results, which can be printed or otherwise inspected. + */ + std::vector complexityBigO() const; + + /** + * @brief Calculates bigO for a custom function. + * + * E.g. to calculate the mean squared error for @f$ \mathcal{O}(\log{}\log{}n) @f$, which is not part of the default set of + * complexityBigO(), you can do this: + * + * ``` + * auto logLogN = bench.complexityBigO("O(log log n)", [](double n) { + * return std::log2(std::log2(n)); + * }); + * ``` + * + * The resulting mean squared error can be printed with `std::cout << logLogN`. E.g. it prints something like this: + * + * ```text + * 2.46985e-05 * O(log log n), rms=1.48121 + * ``` + * + * @tparam Op Type of mapping operation. + * @param name Name for the function, e.g. "O(log log n)" + * @param op Op's operator() maps a `double` with the desired complexity function, e.g. `log2(log2(n))`. + * @return BigO Error calculation, which is streamable to std::cout. + */ + template + BigO complexityBigO(char const* name, Op op) const; + + template + BigO complexityBigO(std::string const& name, Op op) const; + + /*! + @verbatim embed:rst + + Convenience shortcut to :cpp:func:`ankerl::nanobench::render`. + + @endverbatim + */ + Bench& render(char const* templateContent, std::ostream& os); + Bench& render(std::string const& templateContent, std::ostream& os); + + Bench& config(Config const& benchmarkConfig); + ANKERL_NANOBENCH(NODISCARD) Config const& config() const noexcept; + + /*! + @brief Runs `setupOp()` once before each epoch, without measuring it. + + Use this to restore whatever state your benchmark consumes, when that restoration would otherwise + pollute the measurement: + + @code + bench.setup([&] { data = pristine; }) + .run("consume data", [&] { consume(data); }); + @endcode + + @verbatim embed:rst + .. important:: + + The setup runs **once per epoch, not once per iteration**. An epoch calls ``op()`` many times + (see :cpp:func:`epochIterations() `), and the setup + does not run again in between. So this helps when your operation can be repeated as-is and only + the *starting* state has to be established - and it does **not** help when every single call + mutates the data such that the next call would measure something different. + + For that second case, set :cpp:func:`epochIterations(1) ` + so an epoch is a single call, which makes setup effectively per-iteration - at the cost of much + noisier results, since one call is then timed against the clock's resolution. The alternative, + and usually the better measurement, is to time the setup separately and subtract it. + + Timers are deliberately not started and stopped around each iteration: for anything fast that + costs more than the thing being measured, and the performance counters would have to be + restarted too. + + .. note:: + + The returned object keeps a reference to this ``Bench``, so don't let it outlive it - call + ``run()`` on the same expression, as above. + + @endverbatim + + @tparam SetupOp The untimed code to run before each epoch. + @param setupOp The setup to run. + */ + template + detail::SetupRunner setup(SetupOp setupOp); + +private: + // Collects the `name, op` pack into two flat vectors, once. Everything after this point is + // ordinary index-based code in the implementation block rather than more template recursion. + static void compareCollect(std::vector& /*names*/, std::vector& /*ops*/) {} + + template + static void compareCollect(std::vector& names, std::vector& ops, Name&& name, Op&& op, + Rest&&... rest) { + names.emplace_back(std::forward(name)); + ops.push_back(detail::eraseOp(op)); + compareCollect(names, ops, std::forward(rest)...); + } + + // The measuring half, which no longer needs to know the operations' types. + ANKERL_NANOBENCH(NODISCARD) + CompareResult compareImpl(std::vector const& names, std::vector const& ops); + + // Number of iterations of `op` that makes one epoch last at least `target`. + ANKERL_NANOBENCH(NODISCARD) + uint64_t compareCalibrate(detail::ErasedOp const& op, Clock::duration target) const; + + // The one iteration count the whole experiment runs at. Fixed for every alternative and every + // round: a count that drifted between them would be a second thing changing while the comparison + // is being made, and an epoch's fixed overhead is divided by it, so two different counts amortize + // that overhead differently and bias the ratio. + ANKERL_NANOBENCH(NODISCARD) + uint64_t compareIterations(std::vector const& ops) const; + + // Runs `numIters` iterations of `op` as one epoch, appending the measurement to `result`. + static void compareEpoch(detail::ErasedOp const& op, uint64_t numIters, Result& result); + + template + Bench& runImpl(SetupOp& setupOp, Op&& op); + + template + friend class detail::SetupRunner; + + Config mConfig{}; + std::vector mResults{}; +}; +ANKERL_NANOBENCH(IGNORE_PADDED_POP) + +/** + * @brief Makes sure none of the given arguments are optimized away by the compiler. + * + * @tparam Arg Type of the argument that shouldn't be optimized away. + * @param arg The input that we mark as being used, even though we don't do anything with it. + */ +template +void doNotOptimizeAway(Arg&& arg); + +namespace detail { + +#if !ANKERL_NANOBENCH(ASM_DONT_OPTIMIZE_AWAY) +void doNotOptimizeAwaySink(void const*); + +template +void doNotOptimizeAway(T const& val); + +#else + +// These assembly magic is directly from what Google Benchmark is doing. I have previously used what facebook's folly was doing, but +// this seemed to have compilation problems in some cases. Google Benchmark seemed to be the most well tested anyways. +// see https://github.com/google/benchmark/blob/v1.7.1/include/benchmark/benchmark.h#L443-L446 +template +void doNotOptimizeAway(T const& val) { + // NOLINTNEXTLINE(hicpp-no-assembler) + asm volatile("" : : "r,m"(val) : "memory"); +} + +template +void doNotOptimizeAway(T& val) { +# if defined(__clang__) + // NOLINTNEXTLINE(hicpp-no-assembler) + asm volatile("" : "+r,m"(val) : : "memory"); +# else + // NOLINTNEXTLINE(hicpp-no-assembler) + asm volatile("" : "+m,r"(val) : : "memory"); +# endif +} +#endif + +// internally used, but visible because run() is templated. +// Not movable/copy-able, so we simply use a pointer instead of unique_ptr. This saves us from +// having to include , and the template instantiation overhead of unique_ptr which is unfortunately quite significant. +ANKERL_NANOBENCH(IGNORE_EFFCPP_PUSH) +class IterationLogic { +public: + explicit IterationLogic(Bench const& bench); + IterationLogic(IterationLogic&&) = delete; + IterationLogic& operator=(IterationLogic&&) = delete; + IterationLogic(IterationLogic const&) = delete; + IterationLogic& operator=(IterationLogic const&) = delete; + ~IterationLogic(); + + ANKERL_NANOBENCH(NODISCARD) uint64_t numIters() const noexcept; + void add(std::chrono::nanoseconds elapsed, PerformanceCounters const& pc) noexcept; + void moveResultTo(std::vector& results) noexcept; + +private: + struct Impl; + Impl* mPimpl; +}; +ANKERL_NANOBENCH(IGNORE_EFFCPP_POP) + +ANKERL_NANOBENCH(IGNORE_PADDED_PUSH) +class PerformanceCounters { +public: + PerformanceCounters(PerformanceCounters const&) = delete; + PerformanceCounters(PerformanceCounters&&) = delete; + PerformanceCounters& operator=(PerformanceCounters const&) = delete; + PerformanceCounters& operator=(PerformanceCounters&&) = delete; + + PerformanceCounters(); + ~PerformanceCounters(); + + void beginMeasure(); + void endMeasure(); + void updateResults(uint64_t numIters); + + ANKERL_NANOBENCH(NODISCARD) PerfCountSet const& val() const noexcept; + ANKERL_NANOBENCH(NODISCARD) PerfCountSet const& has() const noexcept; + +private: +#if ANKERL_NANOBENCH(PERF_COUNTERS) + LinuxPerformanceCounters* mPc = nullptr; +#endif + PerfCountSet mVal{}; + PerfCountSet mHas{}; +}; +ANKERL_NANOBENCH(IGNORE_PADDED_POP) + +// Gets the singleton +PerformanceCounters& performanceCounters(); + +// How long one epoch should take: the clock's resolution multiplied up until the measurement sits +// well above the clock's own noise, then clamped into [minEpochTime, maxEpochTime]. The minimum is +// applied last, so it wins when the two bounds conflict. +// +// Both run() and compare() size their epochs with this. It is a function rather than a line in each +// of them because it is a rule, and a rule with two copies is a rule with two behaviours. +Clock::duration targetRuntimePerEpoch(Bench const& bench); + +// Whether hideColumn() left this column visible. Takes the Config rather than the Bench because the +// table writers only have the Config - every Result carries a copy of the one it was measured under. +ANKERL_NANOBENCH(NODISCARD) bool isColumnVisible(Config const& config, Column column) noexcept; + +// The frequency-scaling and pyperf warnings, and the note that the kernel refused the performance +// counters. Declared up here because compare() is a template defined above the implementation block +// and has to be able to call them - a program whose only nanobench call is compare() needs the +// warning at least as much as one that calls run(). +void printStabilityInformationOnce(std::ostream* outStream); +void printPerformanceCounterHintOnce(std::ostream* outStream, bool wantsPerformanceCounters); + +// The last table settings written to this stream. When they change, a new header is written. +uint64_t& streamHeaderHash(std::ostream& os); + +// The arithmetic that turns what the kernel hands back into the numbers of the ins/op, cyc/op, IPC, +// bra/op and miss% columns. It lives out here, rather than inside the Linux-only counter class, +// because it is the part that can be silently wrong: a counter that is quietly 5% low looks exactly +// like a correct one. Out here it is compiled and testable on every platform, while +// perf_event_open() - which cannot be exercised in a container at all - stays where it is. +// +// All of these saturate at 0 rather than wrapping: a correction larger than the measurement means +// the measurement was noise, not that the value is enormous. + +// a - b, or 0 when b is the larger. The one place that saturation is written down: hand-rolled, it +// was six ternaries that had already drifted between '>' and '>=' without anyone noticing, because at +// a == b the two spellings agree. +uint64_t saturatingSub(uint64_t a, uint64_t b) noexcept; + +// Rounded integer division, for spreading a measured overhead over the iterations it covers. +uint64_t divRounded(uint64_t a, uint64_t divisor) noexcept; + +// How many uint64_t a PERF_FORMAT_GROUP | PERF_FORMAT_ID read needs for `numEvents` events: three +// header words (nr, time_enabled, time_running), then a value and an id per event. That layout used +// to be written down separately at the buffer sizing, at the buffer initialization and here - three +// places to keep in step on a code path most containers and VMs cannot run at all. +size_t perfReadFormatSize(size_t numEvents) noexcept; + +// Index of the i-th event's value in such a read, which is exactly where the i events before it end. +size_t perfValueIndex(uint64_t i) noexcept; + +// Extrapolates a counter that the kernel could only keep active for part of the measurement, the way +// `perf stat` does. Returns the value unchanged when it was active throughout. +uint64_t scaleMultiplexed(uint64_t value, uint64_t timeEnabled, uint64_t timeRunning) noexcept; + +// Subtracts the calibrated overhead of measuring, plus the per-iteration overhead of the loop doing +// the measuring, from a raw counter value. Pass 0 for a correction that does not apply. +uint64_t correctOverhead(uint64_t value, uint64_t measuringOverhead, uint64_t loopOverheadPerIteration, uint64_t numIters) noexcept; + +// The loop's own per-iteration cost, from a calibration run of `numIters` operations and one of +// twice as many: whatever the second run did not cost twice over is the loop rather than the work. +uint64_t loopOverheadPerIteration(uint64_t single, uint64_t doubled, uint64_t measuringOverhead, uint64_t numIters) noexcept; + +// Removes the branch that the measuring loop itself takes once per iteration, plus the one that ends +// it, from a raw branch count. +uint64_t correctBranchInstructions(uint64_t rawBranchInstructions, uint64_t numIters) noexcept; + +// Paired statistics, for Bench::compare(). Pure functions over the per-round measurements, so they are +// testable without running a benchmark at all - which matters, because the alternative is judging a +// confidence interval by looking at it. + +// ln(a[i]) - ln(b[i]) for each round. Working in log space turns a speedup, which is multiplicative, +// into a difference, which is what every statistic below assumes. Rounds where either side measured +// 0 - a clock too coarse for the operation - carry no ratio and are dropped. +std::vector pairedLogRatios(std::vector const& a, std::vector const& b); + +// The same, over two Results' per-round elapsed times. compare() runs one epoch of every alternative +// per round, so their measurements line up round for round and can be paired directly - not only +// against the baseline, but between any two of them. +std::vector pairedLogRatios(Result const& a, Result const& b); + +// Median of a copy of the values. 0.0 when there are none. +double medianOf(std::vector values); + +// The pair of 0-based order statistics that bracket the median with at least `confidence` +// probability: the largest k with P(Bin(n, 1/2) < k) <= (1 - confidence) / 2 gives the interval +// [x_(k), x_(n+1-k)]. Returns {1, 0} - an empty range - when no such interval exists, which is the +// case for fewer than six observations at 95%. +std::pair medianIntervalIndices(size_t n, double confidence); + +// Rounds whose two sides measured the same time to the last tick the clock could report, i.e. a log +// ratio of exactly 0. Not evidence of equality - evidence that the clock ran out of resolution. +size_t countTiedRounds(std::vector const& logRatios); + +// Confidence each individual interval is built at, so that all `numComparisons` of them together +// hold at 95%. Bonferroni: simple, assumption-free, and conservative here because the comparisons +// share a baseline and are therefore correlated. One function, because the table and the summary +// line underneath it must not be able to report intervals at two different confidences. +double bonferroniConfidence(size_t numComparisons) noexcept; + +// Distribution-free confidence interval for the median, from those order statistics. +// +// The sign test, and it is chosen for what it does *not* assume. It needs the observations to be +// independent and nothing else - no shape, no symmetry, no finite variance, no asymptotics - and it +// is exact at every n rather than approximately right for large ones. The alternatives all want +// more: a t-interval wants normality, a bootstrap wants its own asymptotics and converges slowly +// for a median in particular, and Wilcoxon wants the differences to be symmetric about their +// median. That last one is exactly what paired timings do not give you, because an operation can be +// arbitrarily slower but never faster than its floor, so the differences are skewed whenever one +// side has the heavier tail. Measured on such data: sign 96.7% coverage of a nominal 95%, Wilcoxon +// 91.5%. +// +// The price is width - roughly 10% wider than a bootstrap - and slight over-coverage from the +// discreteness. Both err towards saying "no difference resolved", which is the right direction for +// a tool whose output ends up in a pull request. +std::pair medianInterval(std::vector values, double confidence); + +// How many rounds a second stage needs so that its interval is no wider than `targetWidth`. +// +// The width of a sign-test interval for the median shrinks as 1/sqrt(n) -- the two order statistics +// that bracket it sit about z*sqrt(n)/2 ranks either side of the middle, so the distance between +// them falls off with the square root of the count. So the count that reaches a target width scales +// with the square of the ratio of widths, which is all this computes, plus the rounding the caller +// needs: up to a whole block of `numOps`, never below what was already run, never above `maxRounds`. +// +// `observedWidth` and `targetWidth` are both in log space, the space the interval is computed in. +// A degenerate pilot -- every round identical, so a width of zero -- asks for the minimum rather +// than dividing by zero. +size_t roundsForTargetWidth(size_t pilotRounds, double observedWidth, double targetWidth, size_t numOps, size_t maxRounds) noexcept; + +// Branch misses cannot exceed the branches they were taken from, and the loop is assumed to mispredict +// its own exit once - so at least one miss is always attributed to it. +double correctBranchMisses(uint64_t rawBranchMisses, double correctedBranchInstructions) noexcept; + +// Applies a "key=value,key=value" string - the contents of NANOBENCH_CONFIG - to bench, by calling +// the setter each key is named after. Separate from reading the environment so that it can be tested +// by handing it a string, rather than through a variable that is spelled putenv on one platform and +// _putenv_s on another. +// +// An entry that cannot be applied appends one message to errors and is skipped; every other entry +// still applies. Nothing throws, so a typo behaves the same for a consumer built -fno-exceptions. +void applyConfigString(Bench& bench, std::string const& configStr, std::vector& errors); + +} // namespace detail + +class BigO { +public: + using RangeMeasure = std::vector>; + + template + static RangeMeasure mapRangeMeasure(RangeMeasure data, Op op) { + for (auto& rangeMeasure : data) { + rangeMeasure.first = op(rangeMeasure.first); + } + return data; + } + + static RangeMeasure collectRangeMeasure(std::vector const& results); + + template + BigO(char const* bigOName, RangeMeasure const& rangeMeasure, Op rangeToN) + : BigO(bigOName, mapRangeMeasure(rangeMeasure, rangeToN)) {} + + template + BigO(std::string bigOName, RangeMeasure const& rangeMeasure, Op rangeToN) + : BigO(std::move(bigOName), mapRangeMeasure(rangeMeasure, rangeToN)) {} + + BigO(char const* bigOName, RangeMeasure const& scaledRangeMeasure); + BigO(std::string bigOName, RangeMeasure const& scaledRangeMeasure); + ANKERL_NANOBENCH(NODISCARD) std::string const& name() const noexcept; + ANKERL_NANOBENCH(NODISCARD) double constant() const noexcept; + ANKERL_NANOBENCH(NODISCARD) double normalizedRootMeanSquare() const noexcept; + ANKERL_NANOBENCH(NODISCARD) bool operator<(BigO const& other) const noexcept; + +private: + std::string mName{}; + double mConstant{}; + double mNormalizedRootMeanSquare{}; +}; +std::ostream& operator<<(std::ostream& os, BigO const& bigO); +std::ostream& operator<<(std::ostream& os, std::vector const& bigOs); + +} // namespace nanobench +} // namespace ankerl + +// implementation ///////////////////////////////////////////////////////////////////////////////// + +namespace ankerl { +namespace nanobench { + +constexpr uint64_t(Rng::min)() { + return 0; +} + +constexpr uint64_t(Rng::max)() { + return (std::numeric_limits::max)(); +} + +ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") +uint64_t Rng::operator()() noexcept { + auto x = mX; + + mX = UINT64_C(15241094284759029579) * mY; + mY = rotl(mY - x, 27); + + return x; +} + +ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") +uint32_t Rng::bounded(uint32_t range) noexcept { + uint64_t const r32 = static_cast(operator()()); + auto multiresult = r32 * range; + return static_cast(multiresult >> 32U); +} + +double Rng::uniform01() noexcept { + auto i = (UINT64_C(0x3ff) << 52U) | (operator()() >> 12U); + // can't use union in c++ here for type puning, it's undefined behavior. + // std::memcpy is optimized anyways. + double d{}; + std::memcpy(&d, &i, sizeof(double)); + return d - 1.0; +} + +template +void Rng::shuffle(Container& container) noexcept { + auto i = container.size(); + while (i > 1U) { + using std::swap; + auto n = operator()(); + // using decltype(i) instead of size_t to be compatible to containers with 32bit index (see #80) + auto b1 = static_cast((static_cast(n) * static_cast(i)) >> 32U); + swap(container[--i], container[b1]); + + auto b2 = static_cast(((n >> 32U) * static_cast(i)) >> 32U); + swap(container[--i], container[b2]); + } +} + +ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") +constexpr uint64_t Rng::rotl(uint64_t x, unsigned k) noexcept { + return (x << k) | (x >> (64U - k)); +} + +namespace detail { + +// A setup lambda that captures little - or nothing - is smaller than the alignment of the Bench +// reference next to it, so this pads. That is unavoidable here and harmless, but a consumer building +// with -Weverything sees it as an error in nanobench's own header. +ANKERL_NANOBENCH(IGNORE_PADDED_PUSH) +template +class SetupRunner { +public: + explicit SetupRunner(SetupOp setupOp, Bench& bench) + : mSetupOp(std::move(setupOp)) + , mBench(bench) {} + + template + ANKERL_NANOBENCH_NO_SANITIZE("integer") + Bench& run(Op&& op) { + return mBench.runImpl(mSetupOp, std::forward(op)); + } + + // Bench::run() takes a name, so setup().run() has to as well - otherwise adding a setup to an + // existing benchmark means rewriting its call site to use name() separately. + template + Bench& run(char const* benchmarkName, Op&& op) { + mBench.name(benchmarkName); + return run(std::forward(op)); + } + + template + Bench& run(std::string const& benchmarkName, Op&& op) { + mBench.name(benchmarkName); + return run(std::forward(op)); + } + +private: + SetupOp mSetupOp; + Bench& mBench; +}; +ANKERL_NANOBENCH(IGNORE_PADDED_POP) +} // namespace detail + +template +ANKERL_NANOBENCH_NO_SANITIZE("integer") +Bench& Bench::run(Op&& op) { + auto setupOp = [] {}; + return runImpl(setupOp, std::forward(op)); +} + +template +ANKERL_NANOBENCH_NO_SANITIZE("integer") +Bench& Bench::runImpl(SetupOp& setupOp, Op&& op) { + // It is important that this method is kept short so the compiler can do better optimizations/ inlining of op() + detail::IterationLogic iterationLogic(*this); + auto& pc = detail::performanceCounters(); + + while (auto n = iterationLogic.numIters()) { + setupOp(); + + pc.beginMeasure(); + Clock::time_point const before = Clock::now(); + while (n-- > 0) { + op(); + } + Clock::time_point const after = Clock::now(); + pc.endMeasure(); + pc.updateResults(iterationLogic.numIters()); + iterationLogic.add(after - before, pc); + } + iterationLogic.moveResultTo(mResults); + return *this; +} + +template +CompareResult Bench::compare(Args&&... args) { + static_assert(sizeof...(args) % 2 == 0, "compare() takes `name, op` pairs"); + static_assert(sizeof...(args) >= 4, "compare() needs at least two alternatives to compare"); + + std::vector names; + std::vector ops; + names.reserve(sizeof...(args) / 2); + ops.reserve(sizeof...(args) / 2); + compareCollect(names, ops, std::forward(args)...); + return compareImpl(names, ops); +} + +template +detail::SetupRunner Bench::setup(SetupOp setupOp) { + return detail::SetupRunner(std::move(setupOp), *this); +} + +// Performs all evaluations. +template +Bench& Bench::run(char const* benchmarkName, Op&& op) { + name(benchmarkName); + return run(std::forward(op)); +} + +template +Bench& Bench::run(std::string const& benchmarkName, Op&& op) { + name(benchmarkName); + return run(std::forward(op)); +} + +#if ANKERL_NANOBENCH(HAS_STRING_VIEW) +template +Bench& Bench::run(std::string_view benchmarkName, Op&& op) { + name(benchmarkName); + return run(std::forward(op)); +} +#endif + +template +BigO Bench::complexityBigO(char const* benchmarkName, Op op) const { + return BigO(benchmarkName, BigO::collectRangeMeasure(mResults), op); +} + +template +BigO Bench::complexityBigO(std::string const& benchmarkName, Op op) const { + return BigO(benchmarkName, BigO::collectRangeMeasure(mResults), op); +} + +// Set the batch size, e.g. number of processed bytes, or some other metric for the size of the processed data in each iteration. +// Any argument is cast to double. +template +Bench& Bench::batch(T b) noexcept { + mConfig.mBatch = static_cast(b); + return *this; +} + +// Sets the computation complexity of the next run. Any argument is cast to double. +template +Bench& Bench::complexityN(T n) noexcept { + mConfig.mComplexityN = static_cast(n); + return *this; +} + +// Convenience: makes sure none of the given arguments are optimized away by the compiler. +template +Bench& Bench::doNotOptimizeAway(Arg&& arg) { + detail::doNotOptimizeAway(std::forward(arg)); + return *this; +} + +// Makes sure none of the given arguments are optimized away by the compiler. +template +void doNotOptimizeAway(Arg&& arg) { + detail::doNotOptimizeAway(std::forward(arg)); +} + +namespace detail { + +#if !ANKERL_NANOBENCH(ASM_DONT_OPTIMIZE_AWAY) +template +void doNotOptimizeAway(T const& val) { + doNotOptimizeAwaySink(&val); +} + +#endif + +} // namespace detail +} // namespace nanobench +} // namespace ankerl + +#if defined(ANKERL_NANOBENCH_IMPLEMENT) + +/////////////////////////////////////////////////////////////////////////////////////////////////// +// implementation part - only visible in .cpp +/////////////////////////////////////////////////////////////////////////////////////////////////// + +# include // sort, reverse +# include // compare_exchange_strong in loop overhead +# include // getenv +# include // strstr, strncmp +# include // ifstream to parse proc files +# include // setw, setprecision +# include // cout +# include // numeric_limits, to parse NANOBENCH_CONFIG without overflowing +# include // accumulate +# include // random_device +# include // to_s in Number +# include // throw for rendering templates +# if defined(__linux__) +# include //sysconf +# endif +# if ANKERL_NANOBENCH(PERF_COUNTERS) +# include // map + +# include +# include +# include +# endif + +// declarations /////////////////////////////////////////////////////////////////////////////////// + +namespace ankerl { +namespace nanobench { + +// helper stuff that is only intended to be used internally +namespace detail { + +// formatting utilities +namespace fmt { + +// groups the integer digits of an already formatted number in threes +std::string addThousandsSeparators(std::string str, char sep); + +class StreamStateRestorer; +class Number; +class MarkDownColumn; +class MarkDownCode; + +} // namespace fmt +} // namespace detail +} // namespace nanobench +} // namespace ankerl + +// definitions //////////////////////////////////////////////////////////////////////////////////// + +namespace ankerl { +namespace nanobench { + +uint64_t splitMix64(uint64_t& state) noexcept; + +namespace detail { + +// helpers to get double values +template +inline double d(T t) noexcept { + return static_cast(t); +} +inline double d(Clock::duration duration) noexcept { + return std::chrono::duration_cast>(duration).count(); +} + +// Rounds to the nearest uint64_t. Casting a double that doesn't fit is undefined behavior, so the value is clamped +// into a safe range first. The comparisons are written so that a NaN takes the same path as a negative value. +inline uint64_t u64(double val) noexcept { + auto const maxVal = d((std::numeric_limits::max)() / 2U); + if (!(val > 0.0)) { + return 0; + } + // +0.5 for correct rounding when casting + // NOLINTNEXTLINE(bugprone-incorrect-roundings) + return static_cast((val < maxVal ? val : maxVal) + 0.5); +} + +// Calculates clock resolution once, and remembers the result +inline Clock::duration clockResolution() noexcept; + +} // namespace detail + +namespace templates { + +char const* csv() noexcept { + return R"DELIM("title";"name";"unit";"batch";"elapsed";"error %";"instructions";"branches";"branch misses";"total" +{{#result}}"{{title}}";"{{name}}";"{{unit}}";{{batch}};{{median(elapsed)}};{{medianAbsolutePercentError(elapsed)}};{{median(instructions)}};{{median(branchinstructions)}};{{median(branchmisses)}};{{sumProduct(iterations, elapsed)}} +{{/result}})DELIM"; +} + +char const* compareCsv() noexcept { + return R"DELIM("title";"name";"relative";"relative low";"relative high";"significant";"tied rounds";"rounds";"elapsed";"error %" +{{#alternative}}"{{title}}";"{{name}}";{{relative}};{{relativeLow}};{{relativeHigh}};{{significant}};{{tiedRounds}};{{rounds}};{{median(elapsed)}};{{medianAbsolutePercentError(elapsed)}} +{{/alternative}})DELIM"; +} + +char const* htmlBoxplot() noexcept { + return R"DELIM( + + + + + + +
+ + + +)DELIM"; +} + +char const* pyperf() noexcept { + return R"DELIM({ + "benchmarks": [ + { + "runs": [ + { + "values": [ +{{#measurement}} {{elapsed}}{{^-last}}, +{{/last}}{{/measurement}} + ] + } + ] + } + ], + "metadata": { + "loops": {{sum(iterations)}}, + "inner_loops": {{batch}}, + "name": "{{title}}", + "unit": "second" + }, + "version": "1.0" +})DELIM"; +} + +char const* json() noexcept { + return R"DELIM({ + "results": [ +{{#result}} { + "title": "{{title}}", + "name": "{{name}}", + "unit": "{{unit}}", + "batch": {{batch}}, + "complexityN": {{complexityN}}, + "epochs": {{epochs}}, + "clockResolution": {{clockResolution}}, + "clockResolutionMultiple": {{clockResolutionMultiple}}, + "maxEpochTime": {{maxEpochTime}}, + "minEpochTime": {{minEpochTime}}, + "minEpochIterations": {{minEpochIterations}}, + "epochIterations": {{epochIterations}}, + "warmup": {{warmup}}, + "relative": {{relative}}, + "median(elapsed)": {{median(elapsed)}}, + "medianAbsolutePercentError(elapsed)": {{medianAbsolutePercentError(elapsed)}}, + "median(instructions)": {{median(instructions)}}, + "medianAbsolutePercentError(instructions)": {{medianAbsolutePercentError(instructions)}}, + "median(cpucycles)": {{median(cpucycles)}}, + "median(contextswitches)": {{median(contextswitches)}}, + "median(pagefaults)": {{median(pagefaults)}}, + "median(branchinstructions)": {{median(branchinstructions)}}, + "median(branchmisses)": {{median(branchmisses)}}, + "totalTime": {{sumProduct(iterations, elapsed)}}, + "measurements": [ +{{#measurement}} { + "iterations": {{iterations}}, + "elapsed": {{elapsed}}, + "pagefaults": {{pagefaults}}, + "cpucycles": {{cpucycles}}, + "contextswitches": {{contextswitches}}, + "instructions": {{instructions}}, + "branchinstructions": {{branchinstructions}}, + "branchmisses": {{branchmisses}} + }{{^-last}},{{/-last}} +{{/measurement}} ] + }{{^-last}},{{/-last}} +{{/result}} ] +})DELIM"; +} + +ANKERL_NANOBENCH(IGNORE_PADDED_PUSH) +struct Node { + enum class Type { tag, content, section, inverted_section }; + + char const* begin; + char const* end; + std::vector children; + Type type; + + template + // NOLINTNEXTLINE(hicpp-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) + bool operator==(char const (&str)[N]) const noexcept { + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay) + return static_cast(std::distance(begin, end) + 1) == N && 0 == strncmp(str, begin, N - 1); + } +}; +ANKERL_NANOBENCH(IGNORE_PADDED_POP) + +// A node points into the template rather than owning its text, and these two are the only places that +// have to know it: everywhere else asks for the text or writes it out. Free functions rather than +// members, so Node stays the plain aggregate that parseMustacheTemplate() brace-initializes. +static std::string text(Node const& n) { + return {n.begin, n.end}; +} + +static void writeTo(Node const& n, std::ostream& out) { + out.write(n.begin, std::distance(n.begin, n.end)); +} + +// NOLINTNEXTLINE(misc-no-recursion) +static std::vector parseMustacheTemplate(char const** tpl) { + std::vector nodes; + + while (true) { + auto const* begin = std::strstr(*tpl, "{{"); + auto const* end = begin; + if (begin != nullptr) { + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + begin += 2; + end = std::strstr(begin, "}}"); + } + + if (begin == nullptr || end == nullptr) { + // nothing found, finish node + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + nodes.emplace_back(Node{*tpl, *tpl + std::strlen(*tpl), std::vector{}, Node::Type::content}); + return nodes; + } + + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + nodes.emplace_back(Node{*tpl, begin - 2, std::vector{}, Node::Type::content}); + + // we found a tag + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + *tpl = end + 2; + switch (*begin) { + case '/': + // finished! bail out + return nodes; + + case '#': + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + nodes.emplace_back(Node{begin + 1, end, parseMustacheTemplate(tpl), Node::Type::section}); + break; + + case '^': + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + nodes.emplace_back(Node{begin + 1, end, parseMustacheTemplate(tpl), Node::Type::inverted_section}); + break; + + default: + nodes.emplace_back(Node{begin, end, std::vector{}, Node::Type::tag}); + break; + } + } +} + +static bool generateFirstLast(Node const& n, size_t idx, size_t size, std::ostream& out) { + ANKERL_NANOBENCH_LOG("n.type=" << static_cast(n.type)); + bool const matchFirst = n == "-first"; + bool const matchLast = n == "-last"; + if (!matchFirst && !matchLast) { + return false; + } + + bool doWrite = false; + if (n.type == Node::Type::section) { + doWrite = (matchFirst && idx == 0) || (matchLast && idx == size - 1); + } else if (n.type == Node::Type::inverted_section) { + doWrite = (matchFirst && idx != 0) || (matchLast && idx != size - 1); + } + + if (doWrite) { + for (auto const& child : n.children) { + if (child.type == Node::Type::content) { + writeTo(child, out); + } + } + } + return true; +} + +static bool matchCmdArgs(std::string const& str, std::vector& matchResult) { + matchResult.clear(); + auto idxOpen = str.find('('); + auto idxClose = str.find(')', idxOpen); + if (idxClose == std::string::npos) { + return false; + } + + matchResult.emplace_back(str.substr(0, idxOpen)); + + // split by comma + matchResult.emplace_back(); + for (size_t i = idxOpen + 1; i != idxClose; ++i) { + if (str[i] == ' ' || str[i] == '\t') { + // skip whitespace + continue; + } + if (str[i] == ',') { + // got a comma => new string + matchResult.emplace_back(); + continue; + } + // no whitespace no comma, append + matchResult.back() += str[i]; + } + return true; +} + +// One tag: writes `value` when the node names it, and reports whether it did. +template +// NOLINTNEXTLINE(hicpp-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays) +static bool writeTag(Node const& n, char const (&tagName)[N], T const& value, std::ostream& out) { + if (!(n == tagName)) { + return false; + } + out << value; + return true; +} + +static bool generateConfigTag(Node const& n, Config const& config, std::ostream& out) { + using detail::d; + + // Not part of the chain below: the chain evaluates the arguments of every tag up to the one that + // matches, and measuring the clock's resolution the first time is the one that is not free. + if (n == "clockResolution") { + out << d(detail::clockResolution()); + return true; + } + + // '||' short-circuits, so at most one of these writes anything and nothing past the match is even + // evaluated. Written out as fourteen copies of writeTag()'s body, this was a four-line stanza per + // tag whose only compiler-visible mistake - forgetting the `return true` - fell through to + // "command not understood" at runtime. + return writeTag(n, "title", config.mBenchmarkTitle, out) || writeTag(n, "name", config.mBenchmarkName, out) || + writeTag(n, "unit", config.mUnit, out) || writeTag(n, "batch", config.mBatch, out) || + writeTag(n, "complexityN", config.mComplexityN, out) || writeTag(n, "epochs", config.mNumEpochs, out) || + writeTag(n, "clockResolutionMultiple", config.mClockResolutionMultiple, out) || + writeTag(n, "maxEpochTime", d(config.mMaxEpochTime), out) || writeTag(n, "minEpochTime", d(config.mMinEpochTime), out) || + writeTag(n, "minEpochIterations", config.mMinEpochIterations, out) || + writeTag(n, "epochIterations", config.mEpochIterations, out) || writeTag(n, "warmup", config.mWarmup, out) || + writeTag(n, "relative", config.mIsRelative, out); +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +// `elapsed` is seconds, which is rarely the unit anyone wants in a report, and the template language +// has no arithmetic to fix that afterwards (issue #107). So a time measure may carry a unit suffix - +// `elapsedms`, `elapsedus`, `elapsedns` - and this resolves it to the measure plus the factor to +// multiply by. Plain `elapsed` keeps its meaning, so existing templates render byte for byte as +// before. +static Result::Measure measureFromString(std::string const& str, double& scale) { + scale = 1.0; + auto m = Result::fromString(str); + if (Result::Measure::_size != m) { + return m; + } + + if (str == "elapsedms") { + scale = 1e3; + return Result::Measure::elapsed; + } + if (str == "elapsedus") { + scale = 1e6; + return Result::Measure::elapsed; + } + if (str == "elapsedns") { + scale = 1e9; + return Result::Measure::elapsed; + } + return Result::Measure::_size; +} + +// The one-argument measure commands - {{median(elapsed)}} and friends. Split out of +// generateResultTag so that adding the unit-scaled measures did not push its cognitive complexity +// past what clang-tidy accepts. +static bool generateMeasureTag(std::string const& command, Result const& r, Result::Measure m, double scale, std::ostream& out) { + if (command == "median") { + out << r.median(m) * scale; + return true; + } + if (command == "average") { + out << r.average(m) * scale; + return true; + } + if (command == "medianAbsolutePercentError") { + // a relative error is the same number whatever the unit, so this one is not scaled + out << r.medianAbsolutePercentError(m); + return true; + } + if (command == "sum") { + out << r.sum(m) * scale; + return true; + } + if (command == "minimum") { + out << r.minimum(m) * scale; + return true; + } + if (command == "maximum") { + out << r.maximum(m) * scale; + return true; + } + return false; +} + +static std::ostream& generateResultTag(Node const& n, Result const& r, std::ostream& out) { + if (generateConfigTag(n, r.config(), out)) { + return out; + } + // match e.g. "median(elapsed)" + // g++ 4.8 doesn't implement std::regex :( + // static std::regex const regOpArg1("^([a-zA-Z]+)\\(([a-zA-Z]*)\\)$"); + // std::cmatch matchResult; + // if (std::regex_match(n.begin, n.end, matchResult, regOpArg1)) { + std::vector matchResult; + if (matchCmdArgs(text(n), matchResult)) { + if (matchResult.size() == 2) { + if (matchResult[0] == "context") { + return out << r.context(matchResult[1]); + } + + double scale = 1.0; + auto m = measureFromString(matchResult[1], scale); + if (m == Result::Measure::_size) { + return out << 0.0; + } + if (generateMeasureTag(matchResult[0], r, m, scale, out)) { + return out; + } + } else if (matchResult.size() == 3) { + double scale1 = 1.0; + double scale2 = 1.0; + auto m1 = measureFromString(matchResult[1], scale1); + auto m2 = measureFromString(matchResult[2], scale2); + if (m1 == Result::Measure::_size || m2 == Result::Measure::_size) { + return out << 0.0; + } + + if (matchResult[0] == "sumProduct") { + return out << r.sumProduct(m1, m2) * scale1 * scale2; + } + } + } + + // match e.g. "sumProduct(elapsed, iterations)" + // static std::regex const regOpArg2("^([a-zA-Z]+)\\(([a-zA-Z]*)\\s*,\\s+([a-zA-Z]*)\\)$"); + + // nothing matches :( + ANKERL_NANOBENCH_THROW(std::runtime_error("command '" + text(n) + "' not understood")); +} + +static void generateResultMeasurement(std::vector const& nodes, size_t idx, Result const& r, std::ostream& out) { + for (auto const& n : nodes) { + if (!generateFirstLast(n, idx, r.size(), out)) { + ANKERL_NANOBENCH_LOG("n.type=" << static_cast(n.type)); + switch (n.type) { + case Node::Type::content: + writeTo(n, out); + break; + + case Node::Type::inverted_section: + ANKERL_NANOBENCH_THROW(std::runtime_error("got a inverted section inside measurement")); + + case Node::Type::section: + ANKERL_NANOBENCH_THROW(std::runtime_error("got a section inside measurement")); + + case Node::Type::tag: { + double scale = 1.0; + auto m = measureFromString(text(n), scale); + if (m == Result::Measure::_size || !r.has(m)) { + out << 0.0; + } else { + out << r.get(idx, m) * scale; + } + break; + } + } + } + } +} + +// The tags a comparison adds on top of a result's. Returns false for anything it does not know, so +// the caller can fall back to the result tags and a template keeps working whichever it names. +static bool generateCompareTag(Node const& n, CompareResult const& cmp, size_t idx, std::ostream& out) { + if (n == "relative") { + out << cmp[idx].relative; + } else if (n == "relativeLow") { + out << cmp[idx].relativeLow; + } else if (n == "relativeHigh") { + out << cmp[idx].relativeHigh; + } else if (n == "tiedRounds") { + out << cmp[idx].tiedRounds; + } else if (n == "significant") { + out << (cmp.isSignificant(idx) ? 1 : 0); + } else if (n == "rounds") { + out << cmp.rounds(); + } else if (n == "comparisons") { + out << cmp.comparisons(); + } else { + return false; + } + return true; +} + +static void generateCompareEntry(std::vector const& nodes, size_t idx, CompareResult const& cmp, std::ostream& out) { + auto const& r = cmp[idx].result; + for (auto const& n : nodes) { + if (!generateFirstLast(n, idx, cmp.size(), out)) { + ANKERL_NANOBENCH_LOG("n.type=" << static_cast(n.type)); + switch (n.type) { + case Node::Type::content: + writeTo(n, out); + break; + + case Node::Type::inverted_section: + ANKERL_NANOBENCH_THROW(std::runtime_error("got a inverted section inside alternative")); + + case Node::Type::section: + if (n == "measurement") { + for (size_t i = 0; i < r.size(); ++i) { + generateResultMeasurement(n.children, i, r, out); + } + } else { + ANKERL_NANOBENCH_THROW(std::runtime_error("got a section inside alternative")); + } + break; + + case Node::Type::tag: + if (!generateCompareTag(n, cmp, idx, out)) { + generateResultTag(n, r, out); + } + break; + } + } + } +} + +static void generateResult(std::vector const& nodes, size_t idx, std::vector const& results, std::ostream& out) { + auto const& r = results[idx]; + for (auto const& n : nodes) { + if (!generateFirstLast(n, idx, results.size(), out)) { + ANKERL_NANOBENCH_LOG("n.type=" << static_cast(n.type)); + switch (n.type) { + case Node::Type::content: + writeTo(n, out); + break; + + case Node::Type::inverted_section: + ANKERL_NANOBENCH_THROW(std::runtime_error("got a inverted section inside result")); + + case Node::Type::section: + if (n == "measurement") { + for (size_t i = 0; i < r.size(); ++i) { + generateResultMeasurement(n.children, i, r, out); + } + } else { + ANKERL_NANOBENCH_THROW(std::runtime_error("got a section inside result")); + } + break; + + case Node::Type::tag: + generateResultTag(n, r, out); + break; + } + } + } +} + +} // namespace templates + +// helper stuff that only intended to be used internally +namespace detail { + +char const* getEnv(char const* name); +bool isEndlessRunning(std::string const& name); +bool isWarningsEnabled(); + +// Applies NANOBENCH_CONFIG to a freshly built Bench, printing whatever it could not use. Every Bench +// re-reads and re-parses the variable; only the complaining is once per process. +void applyEnvConfig(Bench& bench); + +template +T parseFile(std::string const& filename, bool* fail); + +void gatherStabilityInformation(std::vector& warnings, std::vector& recommendations); + +// determines resolution of the given clock. This is done by measuring multiple times and returning the minimum time difference. +Clock::duration calcClockResolution(size_t numEvaluations) noexcept; + +// formatting utilities +namespace fmt { + +// RAII to save & restore a stream's state +ANKERL_NANOBENCH(IGNORE_PADDED_PUSH) +class StreamStateRestorer { +public: + explicit StreamStateRestorer(std::ostream& s); + ~StreamStateRestorer(); + + // sets back all stream info that we remembered at construction + void restore(); + + // don't allow copying / moving + StreamStateRestorer(StreamStateRestorer const&) = delete; + StreamStateRestorer& operator=(StreamStateRestorer const&) = delete; + StreamStateRestorer(StreamStateRestorer&&) = delete; + StreamStateRestorer& operator=(StreamStateRestorer&&) = delete; + +private: + std::ostream& mStream; + std::locale mLocale; + std::streamsize const mPrecision; + std::streamsize const mWidth; + std::ostream::char_type const mFill; + std::ostream::fmtflags const mFmtFlags; +}; +ANKERL_NANOBENCH(IGNORE_PADDED_POP) + +// Number formatter +class Number { +public: + Number(int width, int precision, double value); + ANKERL_NANOBENCH(NODISCARD) std::string to_s() const; + +private: + friend std::ostream& operator<<(std::ostream& os, Number const& n); + std::ostream& write(std::ostream& os) const; + + int mWidth; + int mPrecision; + double mValue; +}; + +// helper replacement for std::to_string of signed/unsigned numbers so we are locale independent +std::string to_s(uint64_t n); + +std::ostream& operator<<(std::ostream& os, Number const& n); + +ANKERL_NANOBENCH(IGNORE_PADDED_PUSH) +class MarkDownColumn { +public: + MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val) noexcept; + // a column holding text instead of a number, for the context columns + MarkDownColumn(int w, std::string tit, std::string text) noexcept; + ANKERL_NANOBENCH(NODISCARD) std::string title() const; + ANKERL_NANOBENCH(NODISCARD) std::string separator() const; + ANKERL_NANOBENCH(NODISCARD) std::string invalid() const; + ANKERL_NANOBENCH(NODISCARD) std::string value() const; + +private: + // the column's padding convention, stated once: right aligned in mWidth, one trailing space + ANKERL_NANOBENCH(NODISCARD) std::string padded(std::string const& text) const; + + int mWidth; + int mPrecision; + std::string mTitle; + std::string mSuffix; + double mValue; + std::string mText{}; + bool mIsText{false}; +}; +ANKERL_NANOBENCH(IGNORE_PADDED_POP) + +// Formats any text as markdown code, escaping backticks. +class MarkDownCode { +public: + explicit MarkDownCode(std::string const& what); + +private: + friend std::ostream& operator<<(std::ostream& os, MarkDownCode const& mdCode); + std::ostream& write(std::ostream& os) const; + + std::string mWhat{}; +}; + +std::ostream& operator<<(std::ostream& os, MarkDownCode const& mdCode); + +} // namespace fmt +} // namespace detail +} // namespace nanobench +} // namespace ankerl + +// implementation ///////////////////////////////////////////////////////////////////////////////// + +namespace ankerl { +namespace nanobench { + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +void render(char const* mustacheTemplate, std::vector const& results, std::ostream& out) { + detail::fmt::StreamStateRestorer const restorer(out); + + out.precision(std::numeric_limits::digits10); + auto nodes = templates::parseMustacheTemplate(&mustacheTemplate); + + for (auto const& n : nodes) { + ANKERL_NANOBENCH_LOG("n.type=" << static_cast(n.type)); + switch (n.type) { + case templates::Node::Type::content: + templates::writeTo(n, out); + break; + + case templates::Node::Type::inverted_section: + ANKERL_NANOBENCH_THROW(std::runtime_error("unknown list '" + templates::text(n) + "'")); + + case templates::Node::Type::section: + if (n == "result") { + const size_t nbResults = results.size(); + for (size_t i = 0; i < nbResults; ++i) { + generateResult(n.children, i, results, out); + } + } else if (n == "measurement") { + if (results.size() != 1) { + ANKERL_NANOBENCH_THROW(std::runtime_error( + "render: can only use section 'measurement' here if there is a single result, but there are " + + detail::fmt::to_s(results.size()))); + } + // when we only have a single result, we can immediately go into its measurement. + auto const& r = results.front(); + for (size_t i = 0; i < r.size(); ++i) { + generateResultMeasurement(n.children, i, r, out); + } + } else { + ANKERL_NANOBENCH_THROW(std::runtime_error("render: unknown section '" + templates::text(n) + "'")); + } + break; + + case templates::Node::Type::tag: + if (results.size() == 1) { + // result & config are both supported there + generateResultTag(n, results.front(), out); + } else { + // This just uses the last result's config. + if (!generateConfigTag(n, results.back().config(), out)) { + ANKERL_NANOBENCH_THROW(std::runtime_error("unknown tag '" + templates::text(n) + "'")); + } + } + break; + } + } +} + +void render(char const* mustacheTemplate, CompareResult const& cmp, std::ostream& out) { + detail::fmt::StreamStateRestorer const restorer(out); + + out.precision(std::numeric_limits::digits10); + auto nodes = templates::parseMustacheTemplate(&mustacheTemplate); + + for (auto const& n : nodes) { + ANKERL_NANOBENCH_LOG("n.type=" << static_cast(n.type)); + switch (n.type) { + case templates::Node::Type::content: + templates::writeTo(n, out); + break; + + case templates::Node::Type::inverted_section: + ANKERL_NANOBENCH_THROW(std::runtime_error("unknown list '" + templates::text(n) + "'")); + + case templates::Node::Type::section: + if (n == "alternative") { + for (size_t i = 0; i < cmp.size(); ++i) { + templates::generateCompareEntry(n.children, i, cmp, out); + } + } else { + ANKERL_NANOBENCH_THROW(std::runtime_error("render: unknown section '" + templates::text(n) + "'")); + } + break; + + case templates::Node::Type::tag: + // Outside the section only the experiment-wide tags mean anything; the baseline's config + // answers for the rest, which is what makes a bare {{title}} in a header line work. + if (!templates::generateCompareTag(n, cmp, 0, out) && !templates::generateConfigTag(n, cmp[0].result.config(), out)) { + ANKERL_NANOBENCH_THROW(std::runtime_error("unknown tag '" + templates::text(n) + "'")); + } + break; + } + } +} + +void render(std::string const& mustacheTemplate, CompareResult const& cmp, std::ostream& out) { + render(mustacheTemplate.c_str(), cmp, out); +} + +void render(std::string const& mustacheTemplate, std::vector const& results, std::ostream& out) { + render(mustacheTemplate.c_str(), results, out); +} + +void render(char const* mustacheTemplate, const Bench& bench, std::ostream& out) { + render(mustacheTemplate, bench.results(), out); +} + +void render(std::string const& mustacheTemplate, const Bench& bench, std::ostream& out) { + render(mustacheTemplate.c_str(), bench.results(), out); +} + +namespace detail { + +PerformanceCounters& performanceCounters() { +# if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wexit-time-destructors" +# endif + static PerformanceCounters pc; +# if defined(__clang__) +# pragma clang diagnostic pop +# endif + return pc; +} + +Clock::duration targetRuntimePerEpoch(Bench const& bench) { + auto target = detail::clockResolution() * bench.clockResolutionMultiple(); + if (target > bench.maxEpochTime()) { + target = bench.maxEpochTime(); + } + if (target < bench.minEpochTime()) { + target = bench.minEpochTime(); + } + return target; +} + +// Windows version of doNotOptimizeAway +// see https://github.com/google/benchmark/blob/v1.7.1/include/benchmark/benchmark.h#L514 +// see https://github.com/facebook/folly/blob/v2023.01.30.00/folly/lang/Hint-inl.h#L54-L58 +// see https://learn.microsoft.com/en-us/cpp/preprocessor/optimize +# if !ANKERL_NANOBENCH(ASM_DONT_OPTIMIZE_AWAY) +# if defined(_MSC_VER) +# pragma optimize("", off) +# endif +void doNotOptimizeAwaySink(void const*) {} +# if defined(_MSC_VER) +# pragma optimize("", on) +# endif +# endif + +template +T parseFile(std::string const& filename, bool* fail) { + std::ifstream fin(filename); // NOLINT(misc-const-correctness) + T num{}; + fin >> num; + if (fail != nullptr) { + *fail = fin.fail(); + } + return num; +} + +char const* getEnv(char const* name) { +# if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable : 4996) // getenv': This function or variable may be unsafe. +# endif + return std::getenv(name); // NOLINT(concurrency-mt-unsafe) +# if defined(_MSC_VER) +# pragma warning(pop) +# endif +} + +bool isEndlessRunning(std::string const& name) { + auto const* const endless = getEnv("NANOBENCH_ENDLESS"); + return nullptr != endless && endless == name; +} + +// True when environment variable NANOBENCH_SUPPRESS_WARNINGS is either not set at all, or set to "0" +bool isWarningsEnabled() { + auto const* const suppression = getEnv("NANOBENCH_SUPPRESS_WARNINGS"); + return nullptr == suppression || suppression == std::string("0"); +} + +// NANOBENCH_CONFIG /////////////////////////////////////////////////////////////////////////////// +// +// The parsing below reports a bad value by returning the reason for it as a string, empty when the +// value was fine. That is what keeps this usable from a Bench constructor: throwing would be worse +// than the mistake it reports, and impossible for a consumer built with -fno-exceptions. +// +// The numbers are parsed by hand rather than through a stream or std::stoull. Both of those accept +// "-1" as a uint64_t - the standard has them read a signed value and negate it - which would turn a +// typo into 18446744073709551615 epochs. Doing it here also keeps the decimal point out of the +// hands of the global locale, and lets a duration be scaled by an exact power of ten instead of a +// double multiplication that turns 1.5s into 1499999999ns. + +// Everything before the unit of a duration, and every count: "1.5" gives digits 15 and one fraction +// digit. A sign, an exponent, a second dot and anything that does not fit are all rejected - these +// are epoch times and iteration counts, not a general purpose number format. +static std::string parseDecimal(std::string const& str, uint64_t& digits, int& fractionDigits) { + digits = 0; + fractionDigits = 0; + bool seenDot = false; + bool seenDigit = false; + for (auto ch : str) { + if ('.' == ch) { + if (seenDot) { + return "is not a number"; + } + seenDot = true; + continue; + } + if (ch < '0' || ch > '9') { + return "is not a number"; + } + auto const digit = static_cast(ch - '0'); + if (digits > (std::numeric_limits::max() - digit) / 10U) { + return "is too large"; + } + digits = digits * 10U + digit; + seenDigit = true; + if (seenDot) { + ++fractionDigits; + } + } + return seenDigit ? std::string() : std::string("is not a number"); +} + +// Multiplies by ten `exponent` times, or divides that often when it is negative. Only the last +// division rounds, so that the digits thrown away before it cannot round up twice. +static bool scaleByPowerOfTen(uint64_t& value, int exponent) { + while (exponent > 0) { + if (value > std::numeric_limits::max() / 10U) { + return false; + } + value *= 10U; + --exponent; + } + while (exponent < 0) { + auto const remainder = value % 10U; + value /= 10U; + if (-1 == exponent && remainder >= 5U) { + ++value; + } + ++exponent; + } + return true; +} + +// The trailing ns/us/ms/s of a duration. exponent is then the power of ten that turns that unit into +// nanoseconds, and numberLen how much of value is left in front of it. +// +// Every unit is an "s" with an optional metric prefix, which is worth saying that way rather than as +// a list of four suffixes to try: a list has to put the two character units first, since every one +// of them also ends in a match for "s", and nothing about the list says so. +static bool splitTimeUnit(std::string const& value, int& exponent, size_t& numberLen) { + if (value.empty() || 's' != value[value.size() - 1]) { + return false; + } + numberLen = value.size() - 1; + exponent = 9; + if (numberLen > 0) { + auto const prefix = value[numberLen - 1]; + if ('n' == prefix) { + exponent = 0; + --numberLen; + } else if ('u' == prefix) { + exponent = 3; + --numberLen; + } else if ('m' == prefix) { + exponent = 6; + --numberLen; + } + } + return true; +} + +// A whole number of iterations or epochs. +static std::string parseCount(std::string const& value, uint64_t& out) { + int fractionDigits = 0; + std::string reason = parseDecimal(value, out, fractionDigits); + if (reason.empty() && fractionDigits > 0) { + reason = "is not a whole number"; + } + return reason; +} + +// A number with a required unit, e.g. "1.5s". The unit is required because a bare number reads as +// whatever the writer had in mind and would be taken as nanoseconds, so "minEpochTime=5" would +// quietly become a 5ns floor - which is the trap that made one variable per setting, in nanoseconds, +// the wrong shape for this. +static std::string parseDuration(std::string const& value, std::chrono::nanoseconds& out) { + int unitExponent = 0; + size_t numberLen = 0; + if (!splitTimeUnit(value, unitExponent, numberLen)) { + return "is missing a time unit - use ns, us, ms or s"; + } + + // Nested, rather than the flatter chain of early returns this wants to be. `return reason;` is + // only elided when it is the *only* return of a named local, and clang's -Wnrvo is an error + // here; the early return above is fine because it returns a temporary built from a literal. + uint64_t digits = 0; + int fractionDigits = 0; + std::string reason = parseDecimal(value.substr(0, numberLen), digits, fractionDigits); + if (reason.empty()) { + using Rep = std::chrono::nanoseconds::rep; + if (scaleByPowerOfTen(digits, unitExponent - fractionDigits) && + digits <= static_cast(std::numeric_limits::max())) { + out = std::chrono::nanoseconds(static_cast(digits)); + } else { + reason = "is too large"; + } + } + return reason; +} + +// One key, applied by calling the Bench setter it is named after. Going through the setter is what +// "the keys are the Bench setter names" has to mean to be worth writing down: minEpochIterations +// turns 0 into 1, and a setter that grows a rule later gets it for a value from the environment too. +// +// Arg is spelled at the call site rather than deduced. Every one of these names a setter *and* a +// getter, and naming the parameter type outright makes picking between them ordinary overload +// resolution rather than deduction against an overload set. +template +static bool setCount(Bench& bench, std::string const& key, char const* name, Bench& (Bench::*setter)(Arg), std::string const& value, + std::string& reason) { + if (key != name) { + return false; + } + uint64_t parsed = 0; + reason = parseCount(value, parsed); + // size_t is 32 bits on a third of the matrix, and a count that does not survive the trip would + // otherwise run a quietly different benchmark than the one that was asked for + if (reason.empty() && static_cast(static_cast(parsed)) != parsed) { + reason = "is too large"; + } + if (reason.empty()) { + (bench.*setter)(static_cast(parsed)); + } + return true; +} + +static bool setDuration(Bench& bench, std::string const& key, char const* name, Bench& (Bench::*setter)(std::chrono::nanoseconds), + std::string const& value, std::string& reason) { + if (key != name) { + return false; + } + std::chrono::nanoseconds parsed{}; + reason = parseDuration(value, parsed); + if (reason.empty()) { + (bench.*setter)(parsed); + } + return true; +} + +// Everything NANOBENCH_CONFIG understands, each key named exactly once. '||' short-circuits, so at +// most one of these runs and nothing past the match is even evaluated. +// +// This is the shape generateConfigTag() uses for the same seven names in the other direction, and +// for the reason its comment gives: as a stanza per key, the one mistake the compiler cannot see is +// a key you classified and then forgot to assign, which falls through to whichever branch was last. +static bool applyKnownKey(Bench& bench, std::string const& key, std::string const& value, std::string& reason) { + return setDuration(bench, key, "minEpochTime", &Bench::minEpochTime, value, reason) || + setDuration(bench, key, "maxEpochTime", &Bench::maxEpochTime, value, reason) || + setCount(bench, key, "epochs", &Bench::epochs, value, reason) || + setCount(bench, key, "warmup", &Bench::warmup, value, reason) || + setCount(bench, key, "minEpochIterations", &Bench::minEpochIterations, value, reason) || + setCount(bench, key, "epochIterations", &Bench::epochIterations, value, reason) || + setCount(bench, key, "clockResolutionMultiple", &Bench::clockResolutionMultiple, value, reason); +} + +// Sorted, and next to the chain above so that a key added to one and not the other is visible on one +// screen. Only tuning knobs are there at all: name, title, unit, batch and timeUnit say what a +// benchmark *is*, and letting a shell variable change those for a whole process would change what +// the numbers mean rather than how long they take. +static char const* configKeyList() { + return "clockResolutionMultiple, epochIterations, epochs, maxEpochTime, minEpochIterations, minEpochTime, warmup"; +} + +static void applyConfigEntry(Bench& bench, std::string const& key, std::string const& value, std::vector& errors) { + std::string reason; + if (!applyKnownKey(bench, key, value, reason)) { + errors.push_back("NANOBENCH_CONFIG: unknown key '" + key + "' - valid keys are " + configKeyList()); + } else if (!reason.empty()) { + errors.push_back("NANOBENCH_CONFIG: '" + key + "=" + value + "' " + reason); + } +} + +// Whitespace around a key or a value is somebody laying the variable out to be read, not part of +// what they typed. +static std::string trimWhitespace(std::string const& str) { + char const* const spaces = " \t\n\r\f\v"; + auto const first = str.find_first_not_of(spaces); + if (std::string::npos == first) { + return {}; + } + return str.substr(first, str.find_last_not_of(spaces) - first + 1); +} + +void applyConfigString(Bench& bench, std::string const& configStr, std::vector& errors) { + size_t pos = 0; + while (pos < configStr.size()) { + auto const comma = configStr.find(',', pos); + auto const end = std::string::npos == comma ? configStr.size() : comma; + auto const entry = trimWhitespace(configStr.substr(pos, end - pos)); + pos = end + 1; + + // an empty variable, or a trailing comma, is nothing to complain about + if (entry.empty()) { + continue; + } + auto const equals = entry.find('='); + if (std::string::npos == equals) { + errors.push_back("NANOBENCH_CONFIG: '" + entry + "' is not a key=value pair"); + continue; + } + applyConfigEntry(bench, trimWhitespace(entry.substr(0, equals)), trimWhitespace(entry.substr(equals + 1)), errors); + } +} + +// Applies NANOBENCH_CONFIG to a freshly built Bench, so that anything the benchmark sets afterwards +// still wins: the environment moves the default, it does not overrule a benchmark that needs a +// particular setting to mean anything. +void applyEnvConfig(Bench& bench) { + auto const* const configStr = getEnv("NANOBENCH_CONFIG"); + if (nullptr == configStr) { + return; + } + + std::vector errors; + applyConfigString(bench, configStr, errors); + + // Every Bench parses the variable, but a typo in it is one mistake and gets one message. This is + // deliberately not behind NANOBENCH_SUPPRESS_WARNINGS: that hides statements about the machine + // being unsuitable, whereas this is a mistake in something the user typed moments ago, and + // swallowing it is how somebody ends up believing a sweep ran when it did not. + static bool isFirstBench = true; + if (isFirstBench) { + isFirstBench = false; + for (auto const& error : errors) { + std::cerr << error << std::endl; + } + } +} + +void gatherStabilityInformation(std::vector& warnings, std::vector& recommendations) { + warnings.clear(); + recommendations.clear(); + +# if defined(DEBUG) + warnings.emplace_back("DEBUG defined"); + bool const recommendCheckFlags = true; +# else + bool const recommendCheckFlags = false; +# endif + + bool recommendPyPerf = false; +# if defined(__linux__) + auto nprocs = sysconf(_SC_NPROCESSORS_CONF); + if (nprocs <= 0) { + warnings.emplace_back("couldn't figure out number of processors - no governor, turbo check possible"); + } else { + // check frequency scaling + for (long id = 0; id < nprocs; ++id) { + auto idStr = detail::fmt::to_s(static_cast(id)); + auto sysCpu = "/sys/devices/system/cpu/cpu" + idStr; + auto minFreq = parseFile(sysCpu + "/cpufreq/scaling_min_freq", nullptr); + auto maxFreq = parseFile(sysCpu + "/cpufreq/scaling_max_freq", nullptr); + if (minFreq != maxFreq) { + auto minMHz = d(minFreq) / 1000.0; + auto maxMHz = d(maxFreq) / 1000.0; + warnings.emplace_back("CPU frequency scaling enabled: CPU " + idStr + " between " + + detail::fmt::Number(1, 1, minMHz).to_s() + " and " + detail::fmt::Number(1, 1, maxMHz).to_s() + + " MHz"); + recommendPyPerf = true; + break; + } + } + + auto fail = false; + auto currentGovernor = parseFile("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor", &fail); + if (!fail && "performance" != currentGovernor) { + warnings.emplace_back("CPU governor is '" + currentGovernor + "' but should be 'performance'"); + recommendPyPerf = true; + } + + auto noTurbo = parseFile("/sys/devices/system/cpu/intel_pstate/no_turbo", &fail); + if (!fail && noTurbo == 0) { + warnings.emplace_back("Turbo is enabled, CPU frequency will fluctuate"); + recommendPyPerf = true; + } + } +# endif + + if (recommendCheckFlags) { + recommendations.emplace_back("Make sure you compile for Release"); + } + if (recommendPyPerf) { + recommendations.emplace_back("Use 'pyperf system tune' before benchmarking. See https://github.com/psf/pyperf"); + } +} + +void printStabilityInformationOnce(std::ostream* outStream) { + static bool shouldPrint = true; + if (shouldPrint && (nullptr != outStream) && isWarningsEnabled()) { + auto& os = *outStream; + shouldPrint = false; + std::vector warnings; + std::vector recommendations; + gatherStabilityInformation(warnings, recommendations); + if (warnings.empty()) { + return; + } + + os << "Warning, results might be unstable:" << std::endl; + for (auto const& w : warnings) { + os << "* " << w << std::endl; + } + + os << std::endl << "Recommendations" << std::endl; + for (auto const& r : recommendations) { + os << "* " << r << std::endl; + } + } +} + +// When perf_event_open is refused, the table simply comes out five columns narrower. That reads like a +// nanobench bug rather than a setting of the machine, and people have gone looking for it more than once +// (issues #106, #123). So say what happened - but only where those columns could have appeared at all: +// on a platform that has no perf events in the first place nothing is wrong, and a note on every single +// run would be pure noise. +void printPerformanceCounterHintOnce(std::ostream* outStream, bool wantsPerformanceCounters) { +# if ANKERL_NANOBENCH(PERF_COUNTERS) + static bool shouldPrint = true; + if (!shouldPrint || !wantsPerformanceCounters || (nullptr == outStream) || !isWarningsEnabled()) { + return; + } + shouldPrint = false; + + auto const& has = performanceCounters().has(); + if (has.instructions || has.cpuCycles || has.branchInstructions || has.branchMisses) { + return; + } + + *outStream << "Note: perf_event_open failed, so the ins/op, cyc/op, IPC, bra/op and miss% columns are missing." << std::endl + << "This is usually a container or VM without virtualized performance counters, or" << std::endl + << "/proc/sys/kernel/perf_event_paranoid being too restrictive." << std::endl + << std::endl; +# else + (void)outStream; + (void)wantsPerformanceCounters; +# endif +} + +// Which index of the pword/iword array every stream carries is ours. Allocated once per process. +static int streamHeaderHashIndex() { + static int const index = std::ios_base::xalloc(); + return index; +} + +// pword holds a raw void*, so what it points at has to be freed when the stream goes away. +static void streamHeaderHashCallback(std::ios_base::event ev, std::ios_base& ios, int index) { + if (std::ios_base::erase_event == ev) { + // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) + delete static_cast(ios.pword(index)); + ios.pword(index) = nullptr; + } else if (std::ios_base::copyfmt_event == ev) { + // copyfmt just copied our pointer into this stream; only one of the two may own it + ios.pword(index) = nullptr; + } +} + +// This used to be a single process-wide value, so two Bench objects writing to different streams +// overwrote each other's settings and the second table came out with no header at all (issue #112). +// +// The obvious fix - a static map keyed on the ostream* - grows for the lifetime of the process and, +// worse, hands a freshly constructed stream the state of a destroyed one that happened to be +// allocated at the same address, which shows up as a header going missing at random. iostreams +// already provide per-stream storage whose lifetime is exactly the stream's, which is what this +// wants. +uint64_t& streamHeaderHash(std::ostream& os) { + auto const index = streamHeaderHashIndex(); + void*& slot = os.pword(index); + if (nullptr == slot) { + // iword is a separate array at the same index; it records that the callback is registered so + // a stream never registers it twice. copyfmt copies iword and the callbacks together, so a + // stream copied from another is already covered. + if (0 == os.iword(index)) { + os.iword(index) = 1; + os.register_callback(streamHeaderHashCallback, index); + } + // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) + slot = new uint64_t{}; + } + return *static_cast(slot); +} + +ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") +inline uint64_t hash_combine(uint64_t seed, uint64_t val) { + return seed ^ (val + UINT64_C(0x9e3779b9) + (seed << 6U) + (seed >> 2U)); +} + +// determines resolution of the given clock. This is done by measuring multiple times and returning the minimum time difference. +Clock::duration calcClockResolution(size_t numEvaluations) noexcept { + auto bestDuration = Clock::duration::max(); + Clock::time_point tBegin; + Clock::time_point tEnd; + for (size_t i = 0; i < numEvaluations; ++i) { + tBegin = Clock::now(); + do { + tEnd = Clock::now(); + } while (tBegin == tEnd); + bestDuration = (std::min)(bestDuration, tEnd - tBegin); + } + return bestDuration; +} + +// Calculates clock resolution once, and remembers the result +Clock::duration clockResolution() noexcept { + static Clock::duration const sResolution = calcClockResolution(20); + return sResolution; +} + +// Appends a column unless hideColumn() switched it off. +// +// Whether a column *can* be shown - is the counter available at all - and whether the caller wants it +// are different questions, and interleaving them at every site made it easy to answer one where the +// other was meant. The availability checks stay next to the values; the visibility policy is here, +// for every builder below, so a change to how a column is gated is one edit rather than three. +static void addColumn(std::vector& columns, Config const& config, Column column, int width, int precision, + std::string title, std::string suffix, double value) { + if (isColumnVisible(config, column)) { + columns.emplace_back(width, precision, std::move(title), std::move(suffix), value); + } +} + +// The ratio-to-baseline column both table writers put in front of the measurements. Its geometry is +// here rather than at the two call sites: a header written at one width above rows written at +// another lines up nowhere. +static fmt::MarkDownColumn relativeColumn(double percent) { + return {11, 1, "relative", "%", percent}; +} + +// The performance counter columns of a row, empty when the counters are switched off or the kernel +// refused them. Split out of measurementColumns() below only to keep both of them under clang-tidy's +// cognitive complexity limit - the two are read as one list. +static std::vector counterColumns(Config const& config, Result const& result) { + std::vector columns; + if (!config.mShowPerformanceCounters) { + return columns; + } + + auto const hasIns = result.has(Result::Measure::instructions); + auto const hasCyc = result.has(Result::Measure::cpucycles); + double const rInsMedian = hasIns ? result.median(Result::Measure::instructions) : 0.0; + double const rCycMedian = hasCyc ? result.median(Result::Measure::cpucycles) : 0.0; + + if (hasIns) { + addColumn(columns, config, Column::instructions, 18, 2, "ins/" + config.mUnit, "", rInsMedian / config.mBatch); + } + if (hasCyc) { + addColumn(columns, config, Column::cycles, 18, 2, "cyc/" + config.mUnit, "", rCycMedian / config.mBatch); + } + + // Which columns a row has must depend only on what the machine can measure, never on what it + // measured: keying IPC off the values instead meant a row whose op reported zero instructions + // came out one cell short of the header it sits under, and the rest of the table with it. + if (hasIns && hasCyc) { + addColumn(columns, config, Column::ipc, 9, 3, "IPC", "", rCycMedian <= 0.0 ? 0.0 : rInsMedian / rCycMedian); + } + + if (result.has(Result::Measure::branchinstructions)) { + double const rBraMedian = result.median(Result::Measure::branchinstructions); + addColumn(columns, config, Column::branches, 17, 2, "bra/" + config.mUnit, "", rBraMedian / config.mBatch); + if (result.has(Result::Measure::branchmisses)) { + double const misses = result.median(Result::Measure::branchmisses); + addColumn(columns, config, Column::branchMisses, 10, 1, "miss%", "%", + rBraMedian < 1e-9 ? 0.0 : 100.0 * misses / rBraMedian); + } + } + return columns; +} + +// The columns that describe what was measured, as opposed to how one row relates to another. Both +// table writers are built out of these: IterationLogic prepends `relative`, a comparison prepends +// `relative` and `95% CI`, and everything from here on is the same table in both. +// +// It is one function because it used to be two copies, and the copies drifted exactly the way copies +// do: the comparison table silently lost the performance counters it collects anyway and the context +// columns, and hideColumn() went inert there until it was patched separately. Adding a column here +// adds it to both. +// +// Takes the Config rather than the Bench because a comparison has no Bench to ask - every Result +// carries a copy of the config it was measured under, which is the same object either way. +static std::vector measurementColumns(Config const& config, Result const& result) { + std::vector columns; + + auto const median = result.median(Result::Measure::elapsed); + + if (config.mComplexityN > 0.0) { + addColumn(columns, config, Column::complexityN, 14, 0, "complexityN", "", config.mComplexityN); + } + + // context columns come before the measurements: they say which benchmark this row is, and that + // reads better on the left. A row without the variable gets a blank cell rather than a missing + // column, so the table stays rectangular. + for (auto const& variableName : config.mContextColumns) { + auto it = config.mContext.find(variableName); + auto const width = static_cast((std::max)(variableName.size() + 3, static_cast(11))); + columns.emplace_back(width, variableName, it == config.mContext.end() ? std::string() : it->second); + } + + addColumn(columns, config, Column::timePerUnit, 22, 2, config.mTimeUnitName + "/" + config.mUnit, "", + median / (config.mTimeUnit.count() * config.mBatch)); + addColumn(columns, config, Column::unitPerSecond, 22, 2, config.mUnit + "/s", "", median <= 0.0 ? 0.0 : config.mBatch / median); + addColumn(columns, config, Column::error, 10, 1, "err%", "%", result.medianAbsolutePercentError(Result::Measure::elapsed) * 100.0); + + auto const counters = counterColumns(config, result); + columns.insert(columns.end(), counters.begin(), counters.end()); + + addColumn(columns, config, Column::total, 12, 2, "total", "", + result.sumProduct(Result::Measure::iterations, Result::Measure::elapsed)); + return columns; +} + +// The markdown table's header and separator lines. +// +// Both table writers go through this, because the table is not rectangular in the obvious sense: the +// last cell holds the table's title on the header line and a benchmark's name on a data line, and the +// separator's last cell is one dash per title character, so it is one character wider than the header +// cell above it. What lines up - and what the tests assert on - is the offset of the last '|'. Two +// spellings of that convention agree only until one of them changes. +static void writeTableHeaderLines(std::ostream& os, std::vector const& columns, std::string const& title) { + os << std::endl; + for (auto const& col : columns) { + os << col.title(); + } + os << "| " << title << std::endl; + + for (auto const& col : columns) { + os << col.separator(); + } + os << "|:" << std::string(title.size() + 1U, '-') << std::endl; +} + +ANKERL_NANOBENCH(IGNORE_PADDED_PUSH) +struct IterationLogic::Impl { + enum class State { warmup, upscaling_runtime, measuring, endless }; + + explicit Impl(Bench const& bench) + : mBench(bench) + , mResult(bench.config()) { + printStabilityInformationOnce(mBench.output()); + printPerformanceCounterHintOnce(mBench.output(), mBench.performanceCounters()); + + // determine target runtime per epoch + mTargetRuntimePerEpoch = detail::targetRuntimePerEpoch(mBench); + + if (isEndlessRunning(mBench.name())) { + std::cerr << "NANOBENCH_ENDLESS set: running '" << mBench.name() << "' endlessly" << std::endl; + mNumIters = (std::numeric_limits::max)(); + mState = State::endless; + } else if (0 != mBench.warmup()) { + mNumIters = mBench.warmup(); + mState = State::warmup; + } else if (hasExactNumIters()) { + mNumIters = mBench.epochIterations(); + mState = State::measuring; + } else { + mNumIters = mBench.minEpochIterations(); + mState = State::upscaling_runtime; + } + } + + // True when an exact number of iterations per epoch was requested, which overrides any calculated one. + ANKERL_NANOBENCH(NODISCARD) bool hasExactNumIters() const noexcept { + return 0 != mBench.epochIterations(); + } + + // The number of iterations the next epoch should run, honoring an exact count if one was requested. + ANKERL_NANOBENCH(NODISCARD) uint64_t calcNextNumIters(std::chrono::nanoseconds elapsed, uint64_t iters) noexcept { + return hasExactNumIters() ? mBench.epochIterations() : calcBestNumIters(elapsed, iters); + } + + // directly calculates new iters based on elapsed&iters, and stretches the epoch by 0-20% of random noise so epochs + // don't get into lockstep with periodic disturbances. Makes sure we don't under- or overflow. + ANKERL_NANOBENCH(NODISCARD) uint64_t calcBestNumIters(std::chrono::nanoseconds elapsed, uint64_t iters) noexcept { + auto doubleElapsed = d(elapsed); + auto doubleTargetRuntimePerEpoch = d(mTargetRuntimePerEpoch); + auto doubleNewIters = doubleTargetRuntimePerEpoch / doubleElapsed * d(iters); + + // An elapsed time of 0 (possible with a coarse clock) makes the division above produce infinity, or even NaN + // when the target runtime is 0 as well. Comparing with '!(a >= b)' instead of 'a < b' replaces NaN with the + // minimum too, so an epoch never ends up with a nonsensical number of iterations. + auto doubleMinEpochIters = d(mBench.minEpochIterations()); + if (!(doubleNewIters >= doubleMinEpochIters)) { + doubleNewIters = doubleMinEpochIters; + } + doubleNewIters *= 1.0 + 0.2 * mRng.uniform01(); + + return u64(doubleNewIters); + } + + ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") void upscale(std::chrono::nanoseconds elapsed) { + if (elapsed * 10 < mTargetRuntimePerEpoch) { + // we are far below the target runtime. Multiply iterations by 10 (with overflow check) + if (mNumIters > (std::numeric_limits::max)() / 10) { + // overflow :-( + showResult("iterations overflow. Maybe your code got optimized away?"); + mNumIters = 0; + return; + } + mNumIters *= 10; + } else { + mNumIters = calcBestNumIters(elapsed, mNumIters); + } + } + + void add(std::chrono::nanoseconds elapsed, PerformanceCounters const& pc) noexcept { +# if defined(ANKERL_NANOBENCH_LOG_ENABLED) + auto oldIters = mNumIters; +# endif + + switch (mState) { + case State::warmup: + // an exact number of iterations makes upscaling pointless, so warmup is over either way + if (hasExactNumIters() || isCloseEnoughForMeasurements(elapsed)) { + // if elapsed is close enough, we can skip upscaling and go right to measurements + // still, we don't add the result to the measurements. + mState = State::measuring; + mNumIters = calcNextNumIters(elapsed, mNumIters); + } else { + // not close enough: switch to upscaling + mState = State::upscaling_runtime; + upscale(elapsed); + } + break; + + case State::upscaling_runtime: + if (isCloseEnoughForMeasurements(elapsed)) { + // if we are close enough, add measurement and switch to always measuring + mState = State::measuring; + mTotalElapsed += elapsed; + mTotalNumIters += mNumIters; + mResult.add(elapsed, mNumIters, pc); + mNumIters = calcNextNumIters(mTotalElapsed, mTotalNumIters); + } else { + upscale(elapsed); + } + break; + + case State::measuring: + // just add measurements - no questions asked. Even when runtime is low. But we can't ignore + // that fluctuation, or else we would bias the result + mTotalElapsed += elapsed; + mTotalNumIters += mNumIters; + mResult.add(elapsed, mNumIters, pc); + mNumIters = calcNextNumIters(mTotalElapsed, mTotalNumIters); + break; + + case State::endless: + mNumIters = (std::numeric_limits::max)(); + break; + } + + if (static_cast(mResult.size()) == mBench.epochs()) { + // we got all the results that we need, finish it + showResult(""); + mNumIters = 0; + } + + ANKERL_NANOBENCH_LOG(mBench.name() << ": " << detail::fmt::Number(20, 3, d(elapsed.count())) << " elapsed, " + << detail::fmt::Number(20, 3, d(mTargetRuntimePerEpoch.count())) << " target. oldIters=" + << oldIters << ", mNumIters=" << mNumIters << ", mState=" << static_cast(mState)); + } + + void showResult(std::string const& errorMessage) const { + ANKERL_NANOBENCH_LOG(errorMessage); + + if (nullptr == mBench.output()) { + return; + } + + // prepare column data /////// + std::vector columns; + + if (mBench.relative()) { + double relativePercent = 100.0; + if (!mBench.results().empty()) { + // Every other column is per unit, so this one has to be as well. Comparing the raw epoch times would + // give a skewed percentage as soon as the baseline was run with a different batch size. + // See https://github.com/martinus/nanobench/issues/131 + // This is (baseline / baselineBatch) / (rMedian / batch) as a single fraction, so one guard does. + auto const rMedian = mResult.median(Result::Measure::elapsed); + auto const& baseline = mBench.results().front(); + auto const num = baseline.median(Result::Measure::elapsed) * mBench.batch(); + auto const den = rMedian * baseline.config().mBatch; + relativePercent = den <= 0.0 ? 0.0 : num / den * 100.0; + } + if (isColumnVisible(mBench.config(), Column::relative)) { + columns.push_back(relativeColumn(relativePercent)); + } + } + + // everything a comparison table shows too + auto const measurements = measurementColumns(mResult.config(), mResult); + columns.insert(columns.end(), measurements.begin(), measurements.end()); + + double const rErrorMedian = mResult.medianAbsolutePercentError(Result::Measure::elapsed); + + // write everything + auto& os = *mBench.output(); + + // The shape of the table *is* the columns it has, so that is what decides whether the header + // above the previous row still fits this one. Hashing a hand-kept list of the config fields + // that influence the columns meant every conditional column had a second place to be + // remembered, two hundred lines from the one that adds it - and complexityN, which + // measurementColumns() adds only when it is set, had already been forgotten there. + uint64_t hash = 0; + hash = hash_combine(std::hash{}(mBench.title()), hash); + for (auto const& col : columns) { + hash = hash_combine(std::hash{}(col.title()), hash); + } + + auto& lastHeaderHash = streamHeaderHash(os); + if (hash != lastHeaderHash) { + lastHeaderHash = hash; + writeTableHeaderLines(os, columns, mBench.title()); + } + + if (!errorMessage.empty()) { + for (auto const& col : columns) { + os << col.invalid(); + } + os << "| :boom: " << fmt::MarkDownCode(mBench.name()) << " (" << errorMessage << ')' << std::endl; + return; + } + + for (auto const& col : columns) { + os << col.value(); + } + os << "| "; + auto showUnstable = isWarningsEnabled() && rErrorMedian >= 0.05; + if (showUnstable) { + os << ":wavy_dash: "; + } + os << fmt::MarkDownCode(mBench.name()); + if (showUnstable) { + auto avgIters = d(mTotalNumIters) / d(mBench.epochs()); + auto suggestedIters = u64(avgIters * 10); + + os << " (Unstable with ~" << detail::fmt::Number(1, 1, avgIters) << " iters. Increase `minEpochIterations` to e.g. " + << suggestedIters << ")"; + } + os << std::endl; + } + + ANKERL_NANOBENCH(NODISCARD) bool isCloseEnoughForMeasurements(std::chrono::nanoseconds elapsed) const noexcept { + return elapsed * 3 >= mTargetRuntimePerEpoch * 2; + } + + uint64_t mNumIters = 1; // NOLINT(misc-non-private-member-variables-in-classes) + Bench const& mBench; // NOLINT(misc-non-private-member-variables-in-classes) + std::chrono::nanoseconds mTargetRuntimePerEpoch{}; // NOLINT(misc-non-private-member-variables-in-classes) + Result mResult; // NOLINT(misc-non-private-member-variables-in-classes) + Rng mRng{123}; // NOLINT(misc-non-private-member-variables-in-classes) + std::chrono::nanoseconds mTotalElapsed{}; // NOLINT(misc-non-private-member-variables-in-classes) + uint64_t mTotalNumIters = 0; // NOLINT(misc-non-private-member-variables-in-classes) + State mState = State::upscaling_runtime; // NOLINT(misc-non-private-member-variables-in-classes) +}; +ANKERL_NANOBENCH(IGNORE_PADDED_POP) + +IterationLogic::IterationLogic(Bench const& bench) + : mPimpl(new Impl(bench)) {} + +IterationLogic::~IterationLogic() { + delete mPimpl; +} + +uint64_t IterationLogic::numIters() const noexcept { + ANKERL_NANOBENCH_LOG(mPimpl->mBench.name() << ": mNumIters=" << mPimpl->mNumIters); + return mPimpl->mNumIters; +} + +void IterationLogic::add(std::chrono::nanoseconds elapsed, PerformanceCounters const& pc) noexcept { + mPimpl->add(elapsed, pc); +} + +void IterationLogic::moveResultTo(std::vector& results) noexcept { + results.emplace_back(std::move(mPimpl->mResult)); +} + +// The counter arithmetic, deliberately outside the PERF_COUNTERS guard - see the declarations. + +uint64_t saturatingSub(uint64_t a, uint64_t b) noexcept { + return a >= b ? a - b : UINT64_C(0); +} + +uint64_t divRounded(uint64_t a, uint64_t divisor) noexcept { + return (a + divisor / 2) / divisor; +} + +size_t perfReadFormatSize(size_t numEvents) noexcept { + return 3U + numEvents * 2U; +} + +size_t perfValueIndex(uint64_t i) noexcept { + return perfReadFormatSize(static_cast(i)); +} + +uint64_t scaleMultiplexed(uint64_t value, uint64_t timeEnabled, uint64_t timeRunning) noexcept { + if (0U == timeRunning || timeEnabled <= timeRunning) { + // the event was active the whole time, nothing to extrapolate + return value; + } + // counts stay far below 2^53, so double is exact enough here and cannot overflow the way the + // integer multiplication would + return u64(d(value) * (d(timeEnabled) / d(timeRunning))); +} + +ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") +uint64_t correctOverhead(uint64_t value, uint64_t measuringOverhead, uint64_t loopOverheadPerIter, uint64_t numIters) noexcept { + return saturatingSub(saturatingSub(value, measuringOverhead), loopOverheadPerIter * numIters); +} + +ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") +uint64_t loopOverheadPerIteration(uint64_t single, uint64_t doubled, uint64_t measuringOverhead, uint64_t numIters) noexcept { + auto const m1 = saturatingSub(single, measuringOverhead); + auto const m2 = saturatingSub(doubled, measuringOverhead); + // the second run does the work twice, so twice the first run minus the second is what the loop + // costs on its own + return divRounded(saturatingSub(m1 * 2, m2), numIters); +} + +ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") +uint64_t correctBranchInstructions(uint64_t rawBranchInstructions, uint64_t numIters) noexcept { + // one branch per iteration for the loop, plus the one that ends it + return saturatingSub(rawBranchInstructions, numIters + 1U); +} + +size_t roundsForTargetWidth(size_t pilotRounds, double observedWidth, double targetWidth, size_t numOps, size_t maxRounds) noexcept { + auto rounds = pilotRounds; + if (targetWidth > 0.0 && observedWidth > targetWidth) { + auto const factor = (observedWidth / targetWidth) * (observedWidth / targetWidth); + // d() is only defined further down; this stays in double throughout and clamps before the + // cast back, so a pilot that measured almost nothing cannot overflow the count. + auto const wanted = static_cast(pilotRounds) * factor; + auto const capped = wanted > static_cast(maxRounds) ? static_cast(maxRounds) : wanted; + rounds = static_cast(capped); + if (rounds < pilotRounds) { + rounds = pilotRounds; + } + } + if (rounds > maxRounds) { + rounds = maxRounds; + } + // whole blocks, so every alternative is measured the same number of times + if (0U != numOps && 0U != rounds % numOps) { + rounds += numOps - (rounds % numOps); + } + return rounds; +} + +double correctBranchMisses(uint64_t rawBranchMisses, double correctedBranchInstructions) noexcept { + auto branchMisses = d(rawBranchMisses); + if (branchMisses > correctedBranchInstructions) { + // can't have more branch misses than there were branches + branchMisses = correctedBranchInstructions; + } + + // assuming at least one missed branch for the loop + branchMisses -= 1.0; + if (branchMisses < 1.0) { + branchMisses = 1.0; + } + return branchMisses; +} + +# if ANKERL_NANOBENCH(PERF_COUNTERS) + +// glibc declares ioctl()'s request parameter as unsigned long, musl as int. PERF_EVENT_IOC_ID and +// friends carry the direction bits in the high end, so they don't fit into an int and passing one +// straight through is a value-changing conversion that -Werror rejects on musl (issue #92). Deduce +// the declared parameter type from ioctl() itself, so the same cast is right for either libc. +# if defined(__BIONIC__) +// Except on bionic, which declares ioctl() *twice* - once taking int and once taking unsigned - for +// exactly the reason above, so that either signedness compiles without a cast. Its header says out +// loud that the overload breaks anyone taking ioctl()'s address, and points at naming a concrete +// type instead: &::ioctl is an overload set, and nothing can be deduced from one. That deduction +// failure is the whole of issue microsoft/vcpkg#53422 - nanobench built for no Android ABI at all. +// unsigned is the overload to name, since it is the one PERF_EVENT_IOC_ID fits into. +using IoctlRequest = unsigned; +# else +// Never defined - only ever asked for its return type. +template +Request ioctlRequestType(Ret (*)(Fd, Request, ...)); + +# if defined(__cpp_noexcept_function_type) +// Since C++17 noexcept is part of a function's type, and glibc declares ioctl() with __THROW - so +// without this second overload the deduction above stops matching at -std=c++17. +template +Request ioctlRequestType(Ret (*)(Fd, Request, ...) noexcept); +# endif + +using IoctlRequest = decltype(ioctlRequestType(&::ioctl)); +# endif + +template +int perfIoctl(int fd, unsigned long request, Arg arg) { + // NOLINTNEXTLINE(hicpp-signed-bitwise,cppcoreguidelines-pro-type-vararg) + return ioctl(fd, static_cast(request), arg); +} + +ANKERL_NANOBENCH(IGNORE_PADDED_PUSH) +class LinuxPerformanceCounters { +public: + struct Target { + Target(uint64_t* targetValue_, bool correctMeasuringOverhead_, bool correctLoopOverhead_) + : targetValue(targetValue_) + , correctMeasuringOverhead(correctMeasuringOverhead_) + , correctLoopOverhead(correctLoopOverhead_) {} + + uint64_t* targetValue{}; // NOLINT(misc-non-private-member-variables-in-classes) + bool correctMeasuringOverhead{}; // NOLINT(misc-non-private-member-variables-in-classes) + bool correctLoopOverhead{}; // NOLINT(misc-non-private-member-variables-in-classes) + }; + + LinuxPerformanceCounters() = default; + LinuxPerformanceCounters(LinuxPerformanceCounters const&) = delete; + LinuxPerformanceCounters(LinuxPerformanceCounters&&) = delete; + LinuxPerformanceCounters& operator=(LinuxPerformanceCounters const&) = delete; + LinuxPerformanceCounters& operator=(LinuxPerformanceCounters&&) = delete; + ~LinuxPerformanceCounters(); + + bool monitor(perf_sw_ids swId, Target target); + bool monitor(perf_hw_id hwId, Target target); + + ANKERL_NANOBENCH(NODISCARD) bool hasError() const noexcept { + return mHasError; + } + + // Just reading data is faster than enable & disabling. + // we subtract data ourselves. + inline void beginMeasure() { + if (mHasError) { + return; + } + + mHasError = -1 == perfIoctl(mFd, PERF_EVENT_IOC_RESET, PERF_IOC_FLAG_GROUP); + if (mHasError) { + return; + } + + mHasError = -1 == perfIoctl(mFd, PERF_EVENT_IOC_ENABLE, PERF_IOC_FLAG_GROUP); + } + + inline void endMeasure() { + if (mHasError) { + return; + } + + mHasError = (-1 == perfIoctl(mFd, PERF_EVENT_IOC_DISABLE, PERF_IOC_FLAG_GROUP)); + if (mHasError) { + return; + } + + auto const numBytes = sizeof(uint64_t) * mCounters.size(); + auto ret = read(mFd, mCounters.data(), numBytes); + mHasError = ret != static_cast(numBytes); + if (!mHasError) { + scaleCounters(); + } + } + + void updateResults(uint64_t numIters); + + // Compensates the counters that were just read for multiplexing. + // + // When more events are monitored than the hardware can count at the same time, the kernel time-shares the counters + // between them. An event is then only active for a fraction of the measurement, and the value read back is just what + // happened while it was active. Scaling it up by enabled/running extrapolates to the whole measurement, which is + // exactly what `perf stat` does. Without this, cycles/instructions/branches are silently underreported - e.g. when + // the NMI watchdog occupies one of the counters, or in a VM that exposes a restricted PMU. + inline void scaleCounters() noexcept { + auto const enabled = timeEnabled() - mTotalTimeEnabledNanos; + auto const running = timeRunning() - mTotalTimeRunningNanos; + mTotalTimeEnabledNanos = timeEnabled(); + mTotalTimeRunningNanos = timeRunning(); + + for (uint64_t i = 0; i < numEvents(); ++i) { + auto const idx = perfValueIndex(i); + mCounters[idx] = scaleMultiplexed(mCounters[idx], enabled, running); + } + } + + ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") + static inline uint32_t mix(uint32_t x) noexcept { + x ^= x << 13U; + x ^= x >> 17U; + x ^= x << 5U; + return x; + } + + template + ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") + void calibrate(Op&& op) { + // clear current calibration data, so a measurement that fails below subtracts nothing rather + // than something stale + for (auto& v : mCalibratedOverhead) { + v = UINT64_C(0); + } + + // the new data is a minimum over 100 runs, so it starts at the maximum + std::vector newCalibration(mCalibratedOverhead.size(), (std::numeric_limits::max)()); + for (size_t iter = 0; iter < 100; ++iter) { + beginMeasure(); + op(); + endMeasure(); + if (mHasError) { + return; + } + + for (size_t i = 0; i < newCalibration.size(); ++i) { + newCalibration[i] = (std::min)(newCalibration[i], mCounters[i]); + } + } + + mCalibratedOverhead = std::move(newCalibration); + + { + // calibrate loop overhead. For branches & instructions this makes sense, not so much for everything else like cycles. + // marsaglia's xorshift: mov, sal/shr, xor. Times 3. + // This has the nice property that the compiler doesn't seem to be able to optimize multiple calls any further. + // see https://godbolt.org/z/49RVQ5 + uint64_t const numIters = 100000U + (std::random_device{}() & 3U); + uint64_t n = numIters; + uint32_t x = 1234567; + + beginMeasure(); + while (n-- > 0) { + x = mix(x); + } + endMeasure(); + detail::doNotOptimizeAway(x); + auto measure1 = mCounters; + + n = numIters; + beginMeasure(); + while (n-- > 0) { + // we now run *twice* so we can easily calculate the overhead + x = mix(x); + x = mix(x); + } + endMeasure(); + detail::doNotOptimizeAway(x); + auto measure2 = mCounters; + + for (size_t i = 0; i < mCounters.size(); ++i) { + mLoopOverhead[i] = loopOverheadPerIteration(measure1[i], measure2[i], mCalibratedOverhead[i], numIters); + } + } + } + +private: + bool monitor(uint32_t type, uint64_t eventid, Target target); + + // The three header words of the last read, by name rather than by index. + ANKERL_NANOBENCH(NODISCARD) uint64_t numEvents() const noexcept { + return mCounters[0]; + } + ANKERL_NANOBENCH(NODISCARD) uint64_t timeEnabled() const noexcept { + return mCounters[1]; + } + ANKERL_NANOBENCH(NODISCARD) uint64_t timeRunning() const noexcept { + return mCounters[2]; + } + + std::map mIdToTarget{}; + + // no events monitored yet, so just the read_format header + std::vector mCounters = std::vector(perfReadFormatSize(0)); + std::vector mCalibratedOverhead = std::vector(perfReadFormatSize(0)); + std::vector mLoopOverhead = std::vector(perfReadFormatSize(0)); + + // PERF_EVENT_IOC_RESET resets the counters, but not time_enabled/time_running: those keep accumulating over the + // whole lifetime of the event. So the times of a single measurement are the difference to the previous read. + uint64_t mTotalTimeEnabledNanos = 0; + uint64_t mTotalTimeRunningNanos = 0; + int mFd = -1; + bool mHasError = false; +}; +ANKERL_NANOBENCH(IGNORE_PADDED_POP) + +LinuxPerformanceCounters::~LinuxPerformanceCounters() { + if (-1 != mFd) { + close(mFd); + } +} + +bool LinuxPerformanceCounters::monitor(perf_sw_ids swId, LinuxPerformanceCounters::Target target) { + return monitor(PERF_TYPE_SOFTWARE, swId, target); +} + +bool LinuxPerformanceCounters::monitor(perf_hw_id hwId, LinuxPerformanceCounters::Target target) { + return monitor(PERF_TYPE_HARDWARE, hwId, target); +} + +// overflow is ok, it's checked +ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") +void LinuxPerformanceCounters::updateResults(uint64_t numIters) { + // clear old data + for (auto& id_value : mIdToTarget) { + *id_value.second.targetValue = UINT64_C(0); + } + + if (mHasError) { + return; + } + + // mCounters has already been compensated for multiplexing in endMeasure(), and so has mCalibratedOverhead + for (uint64_t i = 0; i < numEvents(); ++i) { + auto idx = perfValueIndex(i); + auto id = mCounters[idx + 1U]; + + auto it = mIdToTarget.find(id); + if (it != mIdToTarget.end()) { + auto& tgt = it->second; + // a correction the target did not ask for is applied as a correction of zero + *tgt.targetValue = correctOverhead(mCounters[idx], tgt.correctMeasuringOverhead ? mCalibratedOverhead[idx] : UINT64_C(0), + tgt.correctLoopOverhead ? mLoopOverhead[idx] : UINT64_C(0), numIters); + } + } +} + +bool LinuxPerformanceCounters::monitor(uint32_t type, uint64_t eventid, Target target) { + *target.targetValue = (std::numeric_limits::max)(); + if (mHasError) { + return false; + } + + auto pea = perf_event_attr(); + std::memset(&pea, 0, sizeof(perf_event_attr)); + pea.type = type; + pea.size = sizeof(perf_event_attr); + pea.config = eventid; + pea.disabled = 1; // start counter as disabled + pea.exclude_kernel = 1; + pea.exclude_hv = 1; + + // NOLINTNEXTLINE(hicpp-signed-bitwise) + pea.read_format = PERF_FORMAT_GROUP | PERF_FORMAT_ID | PERF_FORMAT_TOTAL_TIME_ENABLED | PERF_FORMAT_TOTAL_TIME_RUNNING; + + const int pid = 0; // the current process + const int cpu = -1; // all CPUs +# if defined(PERF_FLAG_FD_CLOEXEC) // since Linux 3.14 + const unsigned long flags = PERF_FLAG_FD_CLOEXEC; +# else + const unsigned long flags = 0; +# endif + + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) + auto fd = static_cast(syscall(__NR_perf_event_open, &pea, pid, cpu, mFd, flags)); + if (-1 == fd) { + return false; + } + if (-1 == mFd) { + // first call: set to fd, and use this from now on + mFd = fd; + } + uint64_t id = 0; + if (-1 == perfIoctl(fd, PERF_EVENT_IOC_ID, &id)) { + // couldn't get id + return false; + } + + // insert into map, rely on the fact that map's references are constant. + mIdToTarget.emplace(id, target); + + // prepare readformat with the correct size (after the insert) + auto size = perfReadFormatSize(mIdToTarget.size()); + mCounters.resize(size); + mCalibratedOverhead.resize(size); + mLoopOverhead.resize(size); + + return true; +} + +PerformanceCounters::PerformanceCounters() + : mPc(new LinuxPerformanceCounters()) + , mVal() + , mHas() { + + // HW events + mHas.cpuCycles = mPc->monitor(PERF_COUNT_HW_REF_CPU_CYCLES, LinuxPerformanceCounters::Target(&mVal.cpuCycles, true, false)); + if (!mHas.cpuCycles) { + // Fallback to cycles counter, reference cycles not available in many systems. + mHas.cpuCycles = mPc->monitor(PERF_COUNT_HW_CPU_CYCLES, LinuxPerformanceCounters::Target(&mVal.cpuCycles, true, false)); + } + mHas.instructions = mPc->monitor(PERF_COUNT_HW_INSTRUCTIONS, LinuxPerformanceCounters::Target(&mVal.instructions, true, true)); + mHas.branchInstructions = + mPc->monitor(PERF_COUNT_HW_BRANCH_INSTRUCTIONS, LinuxPerformanceCounters::Target(&mVal.branchInstructions, true, false)); + mHas.branchMisses = mPc->monitor(PERF_COUNT_HW_BRANCH_MISSES, LinuxPerformanceCounters::Target(&mVal.branchMisses, true, false)); + + // SW events + mHas.pageFaults = mPc->monitor(PERF_COUNT_SW_PAGE_FAULTS, LinuxPerformanceCounters::Target(&mVal.pageFaults, true, false)); + mHas.contextSwitches = + mPc->monitor(PERF_COUNT_SW_CONTEXT_SWITCHES, LinuxPerformanceCounters::Target(&mVal.contextSwitches, true, false)); + + mPc->calibrate([] { + auto before = ankerl::nanobench::Clock::now(); + auto after = ankerl::nanobench::Clock::now(); + (void)before; + (void)after; + }); + + if (mPc->hasError()) { + // something failed, don't monitor anything. + mHas = PerfCountSet{}; + } +} + +PerformanceCounters::~PerformanceCounters() { + // no need to check for nullptr, delete nullptr has no effect + delete mPc; +} + +void PerformanceCounters::beginMeasure() { + mPc->beginMeasure(); +} + +void PerformanceCounters::endMeasure() { + mPc->endMeasure(); +} + +void PerformanceCounters::updateResults(uint64_t numIters) { + mPc->updateResults(numIters); +} + +# else + +PerformanceCounters::PerformanceCounters() = default; +PerformanceCounters::~PerformanceCounters() = default; +void PerformanceCounters::beginMeasure() {} +void PerformanceCounters::endMeasure() {} +void PerformanceCounters::updateResults(uint64_t) {} + +# endif + +ANKERL_NANOBENCH(NODISCARD) PerfCountSet const& PerformanceCounters::val() const noexcept { + return mVal; +} +ANKERL_NANOBENCH(NODISCARD) PerfCountSet const& PerformanceCounters::has() const noexcept { + return mHas; +} + +// formatting utilities +namespace fmt { + +// RAII to save & restore a stream's state +StreamStateRestorer::StreamStateRestorer(std::ostream& s) + : mStream(s) + , mLocale(s.getloc()) + , mPrecision(s.precision()) + , mWidth(s.width()) + , mFill(s.fill()) + , mFmtFlags(s.flags()) {} + +StreamStateRestorer::~StreamStateRestorer() { + restore(); +} + +// sets back all stream info that we remembered at construction +void StreamStateRestorer::restore() { + mStream.imbue(mLocale); + mStream.precision(mPrecision); + mStream.width(mWidth); + mStream.fill(mFill); + mStream.flags(mFmtFlags); +} + +Number::Number(int width, int precision, double value) + : mWidth(width) + , mPrecision(precision) + , mValue(value) {} + +// Groups the integer digits of an already formatted number in threes: "1234.50" -> "1,234.50". +// +// This used to be a std::numpunct facet imbued into the stream, which is the idiomatic way and works +// right up until someone builds with -fno-rtti: installing a facet goes through __dynamic_cast, which +// without RTTI reads through a null pointer and takes the process with it (issue #122). Grouping the +// digits by hand costs a few lines, produces the same output for everyone, and drops an allocation +// per formatted number along the way. +// +// Only a run of digits is touched, so "inf" and "nan" pass through unchanged. +std::string addThousandsSeparators(std::string str, char sep) { + size_t const start = (!str.empty() && ('-' == str[0] || '+' == str[0])) ? 1U : 0U; + size_t end = start; + while (end < str.size() && str[end] >= '0' && str[end] <= '9') { + ++end; + } + + // insert from the right, so the positions still to be visited stay valid + for (size_t pos = end; pos > start + 3;) { + pos -= 3; + str.insert(pos, 1, sep); + } + return str; +} + +std::ostream& Number::write(std::ostream& os) const { + // No StreamStateRestorer: the only thing this used to change on os was the imbued locale, and + // std::setw below is consumed by the very next insertion. + + // format without the stream's locale, so a global locale that already groups digits cannot group + // them a second time + std::stringstream ss; + ss.imbue(std::locale::classic()); + ss << std::setprecision(mPrecision) << std::fixed << mValue; + + os << std::setw(mWidth) << addThousandsSeparators(ss.str(), ','); + return os; +} + +std::string Number::to_s() const { + std::stringstream ss; + write(ss); + return ss.str(); +} + +std::string to_s(uint64_t n) { + std::string str; + do { + str += static_cast('0' + static_cast(n % 10)); + n /= 10; + } while (n != 0); + std::reverse(str.begin(), str.end()); + return str; +} + +std::ostream& operator<<(std::ostream& os, Number const& n) { + return n.write(os); +} + +MarkDownColumn::MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val) noexcept + : mWidth(w) + , mPrecision(prec) + , mTitle(std::move(tit)) + , mSuffix(std::move(suff)) + , mValue(val) {} + +MarkDownColumn::MarkDownColumn(int w, std::string tit, std::string text) noexcept + : mWidth(w) + , mPrecision(0) + , mTitle(std::move(tit)) + , mSuffix() + , mValue(0.0) + , mText(std::move(text)) + , mIsText(true) {} + +std::string MarkDownColumn::padded(std::string const& text) const { + std::stringstream ss; + ss << '|' << std::setw(mWidth - 2) << std::right << text << ' '; + return ss.str(); +} + +std::string MarkDownColumn::title() const { + return padded(mTitle); +} + +std::string MarkDownColumn::separator() const { + std::string sep(static_cast(mWidth), '-'); + sep.front() = '|'; + sep.back() = ':'; + return sep; +} + +std::string MarkDownColumn::invalid() const { + std::string sep(static_cast(mWidth), ' '); + sep.front() = '|'; + sep[sep.size() - 2] = '-'; + return sep; +} + +std::string MarkDownColumn::value() const { + if (mIsText) { + // aligned like the header, so the ':' the separator puts on the right stays honest + return padded(mText); + } + std::stringstream ss; + auto width = mWidth - 2 - static_cast(mSuffix.size()); + ss << '|' << Number(width, mPrecision, mValue) << mSuffix << ' '; + return ss.str(); +} + +// Formats any text as markdown code, escaping backticks. +MarkDownCode::MarkDownCode(std::string const& what) { + mWhat.reserve(what.size() + 2); + mWhat.push_back('`'); + for (char const c : what) { + mWhat.push_back(c); + if ('`' == c) { + mWhat.push_back('`'); + } + } + mWhat.push_back('`'); +} + +std::ostream& MarkDownCode::write(std::ostream& os) const { + return os << mWhat; +} + +std::ostream& operator<<(std::ostream& os, MarkDownCode const& mdCode) { + return mdCode.write(os); +} +} // namespace fmt +} // namespace detail + +// provide implementation here so it's only generated once +Config::Config() = default; +Config::~Config() = default; +Config& Config::operator=(Config const&) = default; +Config& Config::operator=(Config&&) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)) = default; +Config::Config(Config const&) = default; +Config::Config(Config&&) noexcept = default; + +// provide implementation here so it's only generated once +Result::~Result() = default; +Result& Result::operator=(Result const&) = default; +Result& Result::operator=(Result&&) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)) = default; +Result::Result(Result const&) = default; +Result::Result(Result&&) noexcept = default; + +namespace detail { +template +inline constexpr typename std::underlying_type::type u(T val) noexcept { + return static_cast::type>(val); +} +} // namespace detail + +// Result returned after a benchmark has finished. Can be used as a baseline for relative(). +Result::Result(Config benchmarkConfig) + : mConfig(std::move(benchmarkConfig)) + // One slot per measure, plus one for _size itself. Result::fromString() returns _size for a name + // it does not know, so it reaches these accessors whenever a caller resolves a measure from + // user input - and reading one slot past the end of the storage is not the answer to a typo. + // With the extra slot it is a permanently empty measurement list, which reads as "nothing was + // measured": 0.0 from the statistics and false from has(), the same thing the mustache renderer + // already does with a measure it does not recognise. + , mNameToMeasurements{detail::u(Result::Measure::_size) + 1U} {} + +void Result::add(Clock::duration totalElapsed, uint64_t iters, detail::PerformanceCounters const& pc) { + using detail::d; + using detail::u; + + double const dIters = d(iters); + mNameToMeasurements[u(Result::Measure::iterations)].push_back(dIters); + + mNameToMeasurements[u(Result::Measure::elapsed)].push_back(d(totalElapsed) / dIters); + if (pc.has().pageFaults) { + mNameToMeasurements[u(Result::Measure::pagefaults)].push_back(d(pc.val().pageFaults) / dIters); + } + if (pc.has().cpuCycles) { + mNameToMeasurements[u(Result::Measure::cpucycles)].push_back(d(pc.val().cpuCycles) / dIters); + } + if (pc.has().contextSwitches) { + mNameToMeasurements[u(Result::Measure::contextswitches)].push_back(d(pc.val().contextSwitches) / dIters); + } + if (pc.has().instructions) { + mNameToMeasurements[u(Result::Measure::instructions)].push_back(d(pc.val().instructions) / dIters); + } + if (pc.has().branchInstructions) { + double const branchInstructions = d(detail::correctBranchInstructions(pc.val().branchInstructions, iters)); + mNameToMeasurements[u(Result::Measure::branchinstructions)].push_back(branchInstructions / dIters); + + if (pc.has().branchMisses) { + auto const branchMisses = detail::correctBranchMisses(pc.val().branchMisses, branchInstructions); + mNameToMeasurements[u(Result::Measure::branchmisses)].push_back(branchMisses / dIters); + } + } +} + +Config const& Result::config() const noexcept { + return mConfig; +} + +inline double calcMedian(std::vector& data) { + if (data.empty()) { + return 0.0; + } + std::sort(data.begin(), data.end()); + + auto midIdx = data.size() / 2U; + if (1U == (data.size() & 1U)) { + return data[midIdx]; + } + return (data[midIdx - 1U] + data[midIdx]) / 2U; +} + +namespace detail { + +std::vector pairedLogRatios(std::vector const& a, std::vector const& b) { + std::vector logRatios; + auto const numPairs = (std::min)(a.size(), b.size()); + logRatios.reserve(numPairs); + for (size_t i = 0; i < numPairs; ++i) { + // A round where either side measured 0 carries no ratio at all - the logarithm of it is not a + // large number, it is negative infinity, and one of those poisons every statistic downstream. + if (a[i] > 0.0 && b[i] > 0.0) { + logRatios.push_back(std::log(a[i]) - std::log(b[i])); + } + } + return logRatios; +} + +std::vector pairedLogRatios(Result const& a, Result const& b) { + auto const perRound = [](Result const& result) { + std::vector values; + values.reserve(result.size()); + for (size_t r = 0; r < result.size(); ++r) { + values.push_back(result.get(r, Result::Measure::elapsed)); + } + return values; + }; + return pairedLogRatios(perRound(a), perRound(b)); +} + +double medianOf(std::vector values) { + return calcMedian(values); +} + +double bonferroniConfidence(size_t numComparisons) noexcept { + return 1.0 - 0.05 / d((std::max)(numComparisons, static_cast(1))); +} + +std::pair medianIntervalIndices(size_t n, double confidence) { + auto const alphaHalf = (1.0 - confidence) / 2.0; + size_t k = 0; + + if (n <= 1024U) { + // Exact: walk the binomial tail up from P(X = 0) = 2^-n, each term from the one before it. + // 2^-1024 is still a representable double, so nothing underflows into a wrong answer. + double p = std::pow(0.5, d(n)); + double cumulative = 0.0; + for (size_t i = 0; i < n; ++i) { + if (i > 0U) { + p = p * d(n - i + 1U) / d(i); + } + if (cumulative + p > alphaHalf) { + break; + } + cumulative += p; + k = i + 1U; + } + } else { + // well past the point where the normal approximation of a binomial is worth arguing about + auto const z = 1.959963985; + auto const approx = d(n) / 2.0 - z * std::sqrt(d(n)) / 2.0; + k = approx <= 0.0 ? 0U : static_cast(approx); + } + + if (0U == k || 2U * k > n) { + // no interval at this confidence for this few observations + return {1U, 0U}; + } + return {k - 1U, n - k}; +} + +size_t countTiedRounds(std::vector const& logRatios) { + size_t tied = 0; + for (auto logRatio : logRatios) { + // written without == so that -Wfloat-equal stays on for everyone else + if (!(logRatio < 0.0) && !(logRatio > 0.0)) { + ++tied; + } + } + return tied; +} + +std::pair medianInterval(std::vector values, double confidence) { + auto const indices = medianIntervalIndices(values.size(), confidence); + if (indices.first > indices.second) { + return {0.0, 0.0}; + } + std::sort(values.begin(), values.end()); + return {values[indices.first], values[indices.second]}; +} + +} // namespace detail + +CompareResult::Entry::Entry(std::string entryName, Result entryResult, double entryRelative, double entryRelativeLow, + double entryRelativeHigh, size_t entryTiedRounds) noexcept + : name(std::move(entryName)) + , result(std::move(entryResult)) + , relative(entryRelative) + , relativeLow(entryRelativeLow) + , relativeHigh(entryRelativeHigh) + , tiedRounds(entryTiedRounds) {} + +CompareResult::CompareResult(std::vector entries, size_t numRounds) + : mEntries(std::move(entries)) + , mRounds(numRounds) {} + +size_t CompareResult::size() const noexcept { + return mEntries.size(); +} + +CompareResult::Entry const& CompareResult::operator[](size_t idx) const { + return mEntries.at(idx); +} + +size_t CompareResult::rounds() const noexcept { + return mRounds; +} + +size_t CompareResult::comparisons() const noexcept { + return mEntries.empty() ? 0U : mEntries.size() - 1U; +} + +bool CompareResult::isSignificant(size_t idx) const { + if (0U == idx) { + // the baseline is not compared against itself + return false; + } + auto const& entry = mEntries.at(idx); + return entry.relativeLow > 1.0 || entry.relativeHigh < 1.0; +} + +// The entry with the lowest median, ignoring index `skip` - pass size() to ignore nothing. Returns +// size() when that leaves no entry at all. The winner and the runner-up are the same question asked +// twice, and asking it twice is how the two answers come to disagree about ties. +static size_t fastestExcept(CompareResult const& compareResult, size_t skip) { + size_t best = compareResult.size(); + auto bestMedian = (std::numeric_limits::max)(); + for (size_t i = 0; i < compareResult.size(); ++i) { + if (i == skip) { + continue; + } + auto const median = compareResult[i].result.median(Result::Measure::elapsed); + if (median < bestMedian) { + bestMedian = median; + best = i; + } + } + return best; +} + +size_t CompareResult::fastest() const { + auto const best = fastestExcept(*this, mEntries.size()); + // with no entries there is nothing to pick, and 0 is what this has always answered + return best == mEntries.size() ? 0U : best; +} + +// The table half of a comparison: an ordinary benchmark table with the ratio to the baseline and an +// interval for it in front. Split out of operator<< because clang-tidy caps cognitive complexity at +// 25 and the two halves together sat well over it. +static void writeCompareTable(std::ostream& os, CompareResult const& compareResult) { + auto const& config = compareResult[0].result.config(); + auto ratioText = [](double low, double high) { + std::ostringstream ss; + ss << detail::fmt::Number(1, 1, low * 100.0) << "% .. " << detail::fmt::Number(1, 1, high * 100.0) << "%"; + return ss.str(); + }; + + // The two extra columns are the comparison itself, so they are not hideable - without them this + // is just a table. Everything after them is the same set an ordinary table shows, performance + // counters included: a comparison collects them around every epoch either way, so leaving them + // out was paying for them and showing nothing. + auto columnsFor = [&](CompareResult::Entry const& entry, bool isBaseline) { + std::vector columns; + columns.push_back(detail::relativeColumn(entry.relative * 100.0)); + columns.emplace_back(22, "95% CI", isBaseline ? std::string() : ratioText(entry.relativeLow, entry.relativeHigh)); + auto const measurements = detail::measurementColumns(entry.result.config(), entry.result); + columns.insert(columns.end(), measurements.begin(), measurements.end()); + return columns; + }; + + // An ordinary table only reprints its header when the shape changes, and that state lives on the + // stream. A comparison table always prints its own header, so the remembered shape is now wrong: + // clear it, or the next run() on this stream matches against a header it never wrote and streams + // its rows straight under the comparison's summary with no header of their own. + detail::streamHeaderHash(os) = 0; + + detail::writeTableHeaderLines(os, columnsFor(compareResult[0], true), config.mBenchmarkTitle); + + for (size_t i = 0; i < compareResult.size(); ++i) { + auto const columns = columnsFor(compareResult[i], 0U == i); + for (auto const& col : columns) { + os << col.value(); + } + os << "| " << detail::fmt::MarkDownCode(compareResult[i].name) << std::endl; + } +} + +// Two alternatives read better as a sentence than as a row to look up. +// One spelling of the interval, so the table and the two summary shapes cannot drift apart. +static void writeCompareInterval(std::ostream& os, double low, double high) { + os << "95% CI [" << detail::fmt::Number(1, 2, low) << " .. " << detail::fmt::Number(1, 2, high) << "]"; +} + +static void writeComparePair(std::ostream& os, CompareResult const& compareResult) { + auto const baseline = detail::fmt::MarkDownCode(compareResult[0].name); + auto const& entry = compareResult[1]; + auto const other = detail::fmt::MarkDownCode(entry.name); + + if (!compareResult.isSignificant(1)) { + os << " no difference resolved between " << baseline << " and " << other << std::endl + << " ratio " << detail::fmt::Number(1, 2, entry.relative) << "x, "; + writeCompareInterval(os, entry.relativeLow, entry.relativeHigh); + return; + } + + // stated the way round the reader asked the question, so the interval inverts with it + auto const faster = entry.relative > 1.0; + os << " " << other << " ran " << detail::fmt::Number(1, 2, faster ? entry.relative : 1.0 / entry.relative) + << (faster ? "x faster than " : "x slower than ") << baseline << std::endl + << " "; + writeCompareInterval(os, faster ? entry.relativeLow : 1.0 / entry.relativeHigh, + faster ? entry.relativeHigh : 1.0 / entry.relativeLow); +} + +// Picking the winner out of many is a selection, not a test: whichever came first is flattered by the +// same luck that made it first. So the claim is not "this one is fastest" on its own, it is how far +// ahead of the *runner-up* it is - and if that interval contains 1, the top two were not separated. +static void writeCompareWinner(std::ostream& os, CompareResult const& compareResult) { + auto const best = compareResult.fastest(); + auto runnerUp = fastestExcept(compareResult, best); + if (runnerUp == compareResult.size()) { + // a comparison of one: there is nothing to be ahead of, so it is compared against itself and + // the empty interval that comes out reads as "not separated" + runnerUp = best; + } + + // The rounds are paired for every alternative, not only against the baseline, so the top two can + // be compared to each other after the fact from the per-round measurements. + auto const logRatios = detail::pairedLogRatios(compareResult[runnerUp].result, compareResult[best].result); + auto const confidence = detail::bonferroniConfidence(compareResult.comparisons()); + auto const interval = detail::medianInterval(logRatios, confidence); + auto const lead = std::exp(detail::medianOf(logRatios)); + auto const leadLow = std::exp(interval.first); + auto const leadHigh = std::exp(interval.second); + + auto const bestCode = detail::fmt::MarkDownCode(compareResult[best].name); + auto const runnerUpCode = detail::fmt::MarkDownCode(compareResult[runnerUp].name); + if (leadLow > 1.0) { + os << " " << bestCode << " is fastest of " << compareResult.size() << ", " << detail::fmt::Number(1, 2, lead) + << "x ahead of " << runnerUpCode << std::endl; + } else { + os << " " << bestCode << " and " << runnerUpCode << " are the two fastest of " << compareResult.size() + << ", and were not separated" << std::endl; + } + os << " "; + writeCompareInterval(os, leadLow, leadHigh); + os << ", intervals corrected for " << compareResult.comparisons() << " comparisons"; +} + +std::ostream& operator<<(std::ostream& os, CompareResult const& compareResult) { + detail::fmt::StreamStateRestorer const restorer(os); + if (0U == compareResult.size()) { + return os; + } + + // The same table an ordinary benchmark prints, plus the ratio to the baseline and an interval for + // it. A ratio with no scale beside it cannot be told from the same ratio on a completely + // different scale, and a side that was wildly unstable is invisible in one - so the absolute + // numbers come first and the verdict after. + writeCompareTable(os, compareResult); + + os << std::endl << " Summary" << std::endl; + if (2U == compareResult.size()) { + writeComparePair(os, compareResult); + } else { + writeCompareWinner(os, compareResult); + } + os << ", " << compareResult.rounds() << " paired rounds, interleaved" << std::endl; + + size_t tied = 0; + for (size_t i = 1; i < compareResult.size(); ++i) { + tied = (std::max)(tied, compareResult[i].tiedRounds); + } + if (0U != tied) { + os << " up to " << tied << " of " << compareResult.rounds() << " rounds tied at the clock's resolution" << std::endl; + } + return os; +} + +uint64_t Bench::compareIterations(std::vector const& ops) const { + // An exact epochIterations() overrides any calculated count, exactly as it does in a normal run. + if (0 != epochIterations()) { + return epochIterations(); + } + + // The count that fills a normal epoch, per alternative. The smallest of them belongs to the + // slowest alternative - it needs the fewest iterations to fill an epoch - and the largest to the + // fastest. + auto const target = detail::targetRuntimePerEpoch(*this); + auto fewest = (std::numeric_limits::max)(); + uint64_t most = 0; + for (auto const& op : ops) { + auto const calibrated = compareCalibrate(op, target); + fewest = (std::min)(fewest, calibrated); + most = (std::max)(most, calibrated); + } + + // Every count above fills the same target, so they are inversely proportional to the + // per-iteration times and convert to any other epoch length by scaling: a count that fills + // `target` times `wanted / target` is the count that fills `wanted`. Calibration stops at the + // first count that reaches the target rather than at the exact one, so the per-iteration time + // behind it is if anything underestimated and the scaled counts err upwards. + auto const targetSeconds = detail::d(target); + if (!(targetSeconds > 0.0)) { + // Nothing to scale by. A maxEpochTime of zero gets here, and every alternative then ran + // minEpochIterations during calibration, which is what `fewest` holds. + return fewest; + } + auto const scaled = [&](uint64_t iters, Clock::duration wanted) { + return detail::u64(detail::d(iters) * (detail::d(wanted) / targetSeconds)); + }; + + // Taking `fewest` is what keeps the slowest alternative inside maxEpochTime, but on its own it + // makes every faster alternative run an epoch shorter than asked for by however much faster it + // is - a 1ns operation against a 50ns one gets a fiftieth of an epoch. So the count is raised + // until the *fastest* alternative clears the epoch this library calls long enough to measure: + // the clock's resolution times the multiple, which is where targetRuntimePerEpoch() starts from. + // That sits far below minEpochTime wherever the clock is good, so this binds only when the + // alternatives are orders of magnitude apart or the clock is coarse. + auto const measurable = scaled(most, detail::clockResolution() * clockResolutionMultiple()); + + // Raising it stretches the slowest alternative's epoch by the same factor, and maxEpochTime is + // the bound on that. A 1ns operation against a 1ms one cannot have both, and epochs a million + // times longer than asked for are the worse of the two failures. + auto const affordable = scaled(fewest, maxEpochTime()); + + return (std::max)(fewest, (std::min)(measurable, affordable)); +} + +uint64_t Bench::compareCalibrate(detail::ErasedOp const& op, Clock::duration target) const { + uint64_t numIters = minEpochIterations(); + uint64_t reached = 0; + for (size_t attempt = 0; attempt < 64; ++attempt) { + Clock::time_point const before = Clock::now(); + op.run(op.op, numIters); + auto const elapsed = Clock::now() - before; + if (elapsed >= target) { + // Measured again at the same count before it is believed. A single reading is one + // interruption away from being far too high, and this loop grows in steps of up to 10x - + // so one preempted attempt ends the search an order of magnitude early, and every + // alternative then runs epochs that much shorter than they were asked to be. That is not + // hypothetical: an Alpine leg on a shared two core runner returned 11111 for an operation + // whose 11111 iterations take 14us against a 100us target, because one attempt was + // interrupted for 86us. Two readings in a row have to cross. + if (reached == numIters) { + return numIters; + } + reached = numIters; + continue; + } + reached = 0; + + // grow towards the target, but never by more than 10x at once - the same shape as the normal + // upscaling, so a wildly wrong first guess still converges in a few steps + auto const elapsedSeconds = detail::d(elapsed); + auto factor = elapsedSeconds > 0.0 ? detail::d(target) / elapsedSeconds : 10.0; + factor = (std::min)(10.0, (std::max)(1.5, factor)); + if (numIters > (std::numeric_limits::max)() / 16U) { + break; + } + numIters = detail::u64(detail::d(numIters) * factor) + 1U; + } + return numIters; +} + +CompareResult Bench::compareImpl(std::vector const& names, std::vector const& ops) { + auto const numOps = ops.size(); + + // A comparison on a machine with frequency scaling on needs these at least as much as a run() + // does, and a program that only ever calls compare() would otherwise never see them. + detail::printStabilityInformationOnce(output()); + detail::printPerformanceCounterHintOnce(output(), performanceCounters()); + + // Honored, though it is not the tool it looks like here: calibration below already runs each side + // for about a full epoch, and the serial correlation a warmup would be aimed at is removed by the + // pairing rather than by running longer first. Measured over 200 rounds, the raw per-round times + // carry a lag-1 autocorrelation around +0.10 while the paired differences carry -0.01 to -0.08, + // and dropping the first twenty rounds changes neither. + for (auto const& op : ops) { + op.run(op.op, warmup()); + } + + // One count for every alternative - see compareIterations() for why that matters so much. + auto const iters = compareIterations(ops); + + std::vector results; + results.reserve(numOps); + for (auto const& name : names) { + Config entryConfig = mConfig; + entryConfig.mBenchmarkName = name; + results.emplace_back(std::move(entryConfig)); + } + + // Every alternative besides the baseline is one more chance to be wrong, so the intervals are + // widened to keep the whole table at 95% rather than each row separately. + auto const confidence = detail::bonferroniConfidence(numOps - 1U); + + // Rounds come in blocks of one epoch per alternative, and the count is raised until the interval + // is possible at all - with many alternatives the corrected confidence needs more rounds before + // any pair of order statistics reaches it. + auto numRounds = ((epochs() + numOps - 1U) / numOps) * numOps; + while (true) { + auto const indices = detail::medianIntervalIndices(numRounds, confidence); + if (indices.first <= indices.second) { + break; + } + numRounds += numOps; + } + + // Rounds, into whichever results are handed in. A stage is one call. + auto runRounds = [&](size_t rounds, std::vector& into) { + Rng orderRng; + std::vector order(numOps); + for (size_t i = 0; i < numOps; ++i) { + order[i] = static_cast(i); + } + for (size_t round = 0; round < rounds; ++round) { + auto const positionInBlock = round % numOps; + if (0 == positionInBlock) { + // A fresh random permutation per block, then rotated one step per round: every + // alternative occupies every position exactly once over the block, so their mean + // positions are equal and a drift that is linear over the block cancels. For two + // alternatives this produces exactly ABBA or BAAB. + orderRng.shuffle(order); + } + for (size_t slot = 0; slot < numOps; ++slot) { + auto const which = order[(slot + positionInBlock) % numOps]; + compareEpoch(ops[which], iters, into[which]); + } + } + }; + + if (targetIntervalWidth() > 0.0) { + // A pilot, whose only job is to say how many rounds the target needs. Its measurements are + // then dropped: choosing the count from the same data that the interval is computed from is + // the optional-stopping mistake, and it makes an interval that covers less often than it + // claims. See targetIntervalWidth() for the argument. + std::vector pilot; + pilot.reserve(numOps); + for (auto const& name : names) { + Config pilotConfig = mConfig; + pilotConfig.mBenchmarkName = name; + pilot.emplace_back(std::move(pilotConfig)); + } + runRounds(numRounds, pilot); + + // The widest interval over the alternatives, so that every row reaches the target, not just + // the luckiest one. + auto widest = 0.0; + for (size_t i = 1; i < numOps; ++i) { + auto const logRatios = detail::pairedLogRatios(pilot[0], pilot[i]); + auto const interval = detail::medianInterval(logRatios, confidence); + auto const width = interval.second - interval.first; + if (width > widest) { + widest = width; + } + } + numRounds = detail::roundsForTargetWidth(numRounds, widest, targetIntervalWidth(), numOps, maxEpochs()); + } + + runRounds(numRounds, results); + + std::vector entries; + entries.reserve(numOps); + entries.emplace_back(names[0], std::move(results[0]), 1.0, 1.0, 1.0, static_cast(0)); + for (size_t i = 1; i < numOps; ++i) { + // ln(t_baseline) - ln(t_i), so a positive difference means the alternative was quicker and + // the ratio comes out above 1 - the same direction relative() reports. The baseline was moved + // into entries[0] above, which is where it is read from now. + auto const logRatios = detail::pairedLogRatios(entries[0].result, results[i]); + auto const interval = detail::medianInterval(logRatios, confidence); + entries.emplace_back(names[i], std::move(results[i]), std::exp(detail::medianOf(logRatios)), std::exp(interval.first), + std::exp(interval.second), detail::countTiedRounds(logRatios)); + } + + CompareResult compareResult{std::move(entries), numRounds}; + if (nullptr != output()) { + *output() << compareResult; + } + return compareResult; +} + +void Bench::compareEpoch(detail::ErasedOp const& op, uint64_t numIters, Result& result) { + auto& pc = detail::performanceCounters(); + + pc.beginMeasure(); + Clock::time_point const before = Clock::now(); + op.run(op.op, numIters); + Clock::time_point const after = Clock::now(); + pc.endMeasure(); + pc.updateResults(numIters); + + result.add(after - before, numIters, pc); +} + +std::vector const& Result::measurements(Measure m) const { + return mNameToMeasurements.at(detail::u(m)); +} + +double Result::median(Measure m) const { + // create a copy so we can sort + auto data = measurements(m); + return calcMedian(data); +} + +double Result::average(Measure m) const { + using detail::d; + auto const& data = measurements(m); + if (data.empty()) { + return 0.0; + } + + return sum(m) / d(data.size()); +} + +double Result::medianAbsolutePercentError(Measure m) const { + // create copy + auto data = measurements(m); + + // calculates MdAPE which is the median of percentage error + // see https://support.numxl.com/hc/en-us/articles/115001223503-MdAPE-Median-Absolute-Percentage-Error + auto med = calcMedian(data); + + // The error is relative to the measurement itself, so a measurement of 0 has none that is finite: it is 0 when the + // median is 0 as well (they agree), and infinite otherwise, which is the limit of |(x - med) / x| for x -> 0. + // Calculating it directly would give NaN, and a NaN in the data breaks the ordering that calcMedian's sort needs. + auto const zeroError = med <= 0.0 ? 0.0 : std::numeric_limits::infinity(); + + // transform the data to absolute error + for (auto& x : data) { + if (x <= 0.0) { + x = zeroError; + continue; + } + x = (x - med) / x; + if (x < 0) { + x = -x; + } + } + return calcMedian(data); +} + +double Result::sum(Measure m) const noexcept { + auto const& data = measurements(m); + return std::accumulate(data.begin(), data.end(), 0.0); +} + +double Result::sumProduct(Measure m1, Measure m2) const noexcept { + auto const& data1 = measurements(m1); + auto const& data2 = measurements(m2); + + if (data1.size() != data2.size()) { + return 0.0; + } + + double result = 0.0; + for (size_t i = 0, s = data1.size(); i != s; ++i) { + result += data1[i] * data2[i]; + } + return result; +} + +bool Result::has(Measure m) const noexcept { + return !measurements(m).empty(); +} + +double Result::get(size_t idx, Measure m) const { + return measurements(m).at(idx); +} + +bool Result::empty() const noexcept { + return 0U == size(); +} + +size_t Result::size() const noexcept { + return measurements(Measure::elapsed).size(); +} + +double Result::minimum(Measure m) const noexcept { + auto const& data = measurements(m); + if (data.empty()) { + return 0.0; + } + + // here its save to assume that at least one element is there + return *std::min_element(data.begin(), data.end()); +} + +double Result::maximum(Measure m) const noexcept { + auto const& data = measurements(m); + if (data.empty()) { + return 0.0; + } + + // here its save to assume that at least one element is there + return *std::max_element(data.begin(), data.end()); +} + +std::string const& Result::context(char const* variableName) const { + return mConfig.mContext.at(variableName); +} + +std::string const& Result::context(std::string const& variableName) const { + return mConfig.mContext.at(variableName); +} + +Result::Measure Result::fromString(std::string const& str) { + if (str == "elapsed") { + return Measure::elapsed; + } + if (str == "iterations") { + return Measure::iterations; + } + if (str == "pagefaults") { + return Measure::pagefaults; + } + if (str == "cpucycles") { + return Measure::cpucycles; + } + if (str == "contextswitches") { + return Measure::contextswitches; + } + if (str == "instructions") { + return Measure::instructions; + } + if (str == "branchinstructions") { + return Measure::branchinstructions; + } + if (str == "branchmisses") { + return Measure::branchmisses; + } + // not found, return _size + return Measure::_size; +} + +// Configuration of a microbenchmark. +Bench::Bench() { + mConfig.mOut = &std::cout; + detail::applyEnvConfig(*this); +} + +Bench::Bench(Bench&&) noexcept = default; +Bench& Bench::operator=(Bench&&) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)) = default; +Bench::Bench(Bench const&) = default; +Bench& Bench::operator=(Bench const&) = default; +Bench::~Bench() noexcept = default; + +double Bench::batch() const noexcept { + return mConfig.mBatch; +} + +double Bench::complexityN() const noexcept { + return mConfig.mComplexityN; +} + +// Set a baseline to compare it to. 100% it is exactly as fast as the baseline, >100% means it is faster than the baseline, <100% +// means it is slower than the baseline. +Bench& Bench::relative(bool isRelativeEnabled) noexcept { + mConfig.mIsRelative = isRelativeEnabled; + return *this; +} +bool Bench::relative() const noexcept { + return mConfig.mIsRelative; +} + +Bench& Bench::performanceCounters(bool showPerformanceCounters) noexcept { + mConfig.mShowPerformanceCounters = showPerformanceCounters; + return *this; +} +bool Bench::performanceCounters() const noexcept { + return mConfig.mShowPerformanceCounters; +} + +namespace detail { + +// One bit of mHiddenColumns per Column, so there had better be at most 32 of them. +static_assert(static_cast(Column::_size) <= 32, "Column no longer fits in Config::mHiddenColumns"); + +ANKERL_NANOBENCH(NODISCARD) inline uint32_t columnBit(Column column) noexcept { + return UINT32_C(1) << static_cast(column); +} + +bool isColumnVisible(Config const& config, Column column) noexcept { + return 0 == (config.mHiddenColumns & columnBit(column)); +} + +} // namespace detail + +Bench& Bench::hideColumn(Column column) noexcept { + mConfig.mHiddenColumns |= detail::columnBit(column); + return *this; +} + +Bench& Bench::showColumn(Column column) noexcept { + mConfig.mHiddenColumns &= ~detail::columnBit(column); + return *this; +} + +bool Bench::isColumnVisible(Column column) const noexcept { + return detail::isColumnVisible(mConfig, column); +} + +Bench& Bench::contextColumn(std::string const& variableName) { + // adding the same name twice would print the same column twice + if (mConfig.mContextColumns.end() != std::find(mConfig.mContextColumns.begin(), mConfig.mContextColumns.end(), variableName)) { + return *this; + } + mConfig.mContextColumns.push_back(variableName); + return *this; +} + +Bench& Bench::clearContextColumns() noexcept { + mConfig.mContextColumns.clear(); + return *this; +} + +// Operation unit. Defaults to "op", could be e.g. "byte" for string processing. +// If u differs from currently set unit, the stored results will be cleared. +// Use singular (byte, not bytes). +Bench& Bench::unit(char const* u) { + if (u != mConfig.mUnit) { + mResults.clear(); + } + mConfig.mUnit = u; + return *this; +} + +Bench& Bench::unit(std::string const& u) { + return unit(u.c_str()); +} + +std::string const& Bench::unit() const noexcept { + return mConfig.mUnit; +} + +Bench& Bench::timeUnit(std::chrono::duration const& tu, std::string const& tuName) { + mConfig.mTimeUnit = tu; + mConfig.mTimeUnitName = tuName; + return *this; +} + +std::string const& Bench::timeUnitName() const noexcept { + return mConfig.mTimeUnitName; +} + +std::chrono::duration const& Bench::timeUnit() const noexcept { + return mConfig.mTimeUnit; +} + +Bench& Bench::title(const char* benchmarkTitle) { + return title(std::string(benchmarkTitle)); +} + +// If benchmarkTitle differs from currently set title, the stored results will be cleared. One body +// for both overloads, so that rule cannot come to mean two things. +Bench& Bench::title(std::string const& benchmarkTitle) { + if (benchmarkTitle != mConfig.mBenchmarkTitle) { + mResults.clear(); + } + mConfig.mBenchmarkTitle = benchmarkTitle; + return *this; +} + +std::string const& Bench::title() const noexcept { + return mConfig.mBenchmarkTitle; +} + +Bench& Bench::name(const char* benchmarkName) { + mConfig.mBenchmarkName = benchmarkName; + return *this; +} + +Bench& Bench::name(std::string const& benchmarkName) { + mConfig.mBenchmarkName = benchmarkName; + return *this; +} + +# if ANKERL_NANOBENCH(HAS_STRING_VIEW) +Bench& Bench::name(std::string_view benchmarkName) { + mConfig.mBenchmarkName.assign(benchmarkName.data(), benchmarkName.size()); + return *this; +} +# endif + +std::string const& Bench::name() const noexcept { + return mConfig.mBenchmarkName; +} + +Bench& Bench::context(char const* variableName, char const* variableValue) { + mConfig.mContext[variableName] = variableValue; + return *this; +} + +Bench& Bench::context(std::string const& variableName, std::string const& variableValue) { + mConfig.mContext[variableName] = variableValue; + return *this; +} + +Bench& Bench::clearContext() { + mConfig.mContext.clear(); + return *this; +} + +// Number of epochs to evaluate. The reported result will be the median of evaluation of each epoch. +Bench& Bench::epochs(size_t numEpochs) noexcept { + mConfig.mNumEpochs = numEpochs; + return *this; +} +size_t Bench::epochs() const noexcept { + return mConfig.mNumEpochs; +} + +Bench& Bench::targetIntervalWidth(double width) noexcept { + mConfig.mTargetIntervalWidth = width; + return *this; +} +double Bench::targetIntervalWidth() const noexcept { + return mConfig.mTargetIntervalWidth; +} + +Bench& Bench::maxEpochs(size_t numEpochs) noexcept { + mConfig.mMaxEpochs = numEpochs; + return *this; +} +size_t Bench::maxEpochs() const noexcept { + return mConfig.mMaxEpochs; +} + +// Desired evaluation time is a multiple of clock resolution. Default is to be 1000 times above this measurement precision. +Bench& Bench::clockResolutionMultiple(size_t multiple) noexcept { + mConfig.mClockResolutionMultiple = multiple; + return *this; +} +size_t Bench::clockResolutionMultiple() const noexcept { + return mConfig.mClockResolutionMultiple; +} + +// Sets the maximum time each epoch should take. Default is 100ms. +Bench& Bench::maxEpochTime(std::chrono::nanoseconds t) noexcept { + mConfig.mMaxEpochTime = t; + return *this; +} +std::chrono::nanoseconds Bench::maxEpochTime() const noexcept { + return mConfig.mMaxEpochTime; +} + +// Sets the minimum time each epoch should take. Default is 1ms. +Bench& Bench::minEpochTime(std::chrono::nanoseconds t) noexcept { + mConfig.mMinEpochTime = t; + return *this; +} +std::chrono::nanoseconds Bench::minEpochTime() const noexcept { + return mConfig.mMinEpochTime; +} + +Bench& Bench::minEpochIterations(uint64_t numIters) noexcept { + mConfig.mMinEpochIterations = (numIters == 0) ? 1 : numIters; + return *this; +} +uint64_t Bench::minEpochIterations() const noexcept { + return mConfig.mMinEpochIterations; +} + +Bench& Bench::epochIterations(uint64_t numIters) noexcept { + mConfig.mEpochIterations = numIters; + return *this; +} +uint64_t Bench::epochIterations() const noexcept { + return mConfig.mEpochIterations; +} + +Bench& Bench::warmup(uint64_t numWarmupIters) noexcept { + mConfig.mWarmup = numWarmupIters; + return *this; +} +uint64_t Bench::warmup() const noexcept { + return mConfig.mWarmup; +} + +Bench& Bench::config(Config const& benchmarkConfig) { + mConfig = benchmarkConfig; + return *this; +} +Config const& Bench::config() const noexcept { + return mConfig; +} + +Bench& Bench::output(std::ostream* outstream) noexcept { + mConfig.mOut = outstream; + return *this; +} + +ANKERL_NANOBENCH(NODISCARD) std::ostream* Bench::output() const noexcept { + return mConfig.mOut; +} + +std::vector const& Bench::results() const noexcept { + return mResults; +} + +Bench& Bench::render(char const* templateContent, std::ostream& os) { + ::ankerl::nanobench::render(templateContent, *this, os); + return *this; +} + +Bench& Bench::render(std::string const& templateContent, std::ostream& os) { + ::ankerl::nanobench::render(templateContent, *this, os); + return *this; +} + +std::vector Bench::complexityBigO() const { + std::vector bigOs; + auto rangeMeasure = BigO::collectRangeMeasure(mResults); + bigOs.emplace_back("O(1)", rangeMeasure, [](double) { + return 1.0; + }); + bigOs.emplace_back("O(n)", rangeMeasure, [](double n) { + return n; + }); + bigOs.emplace_back("O(log n)", rangeMeasure, [](double n) { + return std::log2(n); + }); + bigOs.emplace_back("O(n log n)", rangeMeasure, [](double n) { + return n * std::log2(n); + }); + bigOs.emplace_back("O(n^2)", rangeMeasure, [](double n) { + return n * n; + }); + bigOs.emplace_back("O(n^3)", rangeMeasure, [](double n) { + return n * n * n; + }); + std::sort(bigOs.begin(), bigOs.end()); + return bigOs; +} + +Rng::Rng() + : mX(0) + , mY(0) { + std::random_device rd; + std::uniform_int_distribution dist; + do { + mX = dist(rd); + mY = dist(rd); + } while (mX == 0 && mY == 0); +} + +ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") +uint64_t splitMix64(uint64_t& state) noexcept { + uint64_t z = (state += UINT64_C(0x9e3779b97f4a7c15)); + z = (z ^ (z >> 30U)) * UINT64_C(0xbf58476d1ce4e5b9); + z = (z ^ (z >> 27U)) * UINT64_C(0x94d049bb133111eb); + return z ^ (z >> 31U); +} + +// Seeded as described in romu paper (update april 2020) +Rng::Rng(uint64_t seed) noexcept + : mX(splitMix64(seed)) + , mY(splitMix64(seed)) { + for (size_t i = 0; i < 10; ++i) { + operator()(); + } +} + +// only internally used to copy the RNG. +Rng::Rng(uint64_t x, uint64_t y) noexcept + : mX(x) + , mY(y) {} + +Rng Rng::copy() const noexcept { + return Rng{mX, mY}; +} + +Rng::Rng(std::vector const& data) + : mX(0) + , mY(0) { + if (data.size() != 2) { + ANKERL_NANOBENCH_THROW(std::runtime_error("ankerl::nanobench::Rng::Rng: needed exactly 2 entries in data, but got " + + detail::fmt::to_s(data.size()))); + } + mX = data[0]; + mY = data[1]; +} + +std::vector Rng::state() const { + std::vector data(2); + data[0] = mX; + data[1] = mY; + return data; +} + +BigO::RangeMeasure BigO::collectRangeMeasure(std::vector const& results) { + BigO::RangeMeasure rangeMeasure; + for (auto const& result : results) { + if (result.config().mComplexityN > 0.0) { + rangeMeasure.emplace_back(result.config().mComplexityN, result.median(Result::Measure::elapsed)); + } + } + return rangeMeasure; +} + +BigO::BigO(std::string bigOName, RangeMeasure const& rangeMeasure) + : mName(std::move(bigOName)) { + + // Nothing to fit against: leave the constant and the error at 0 rather than dividing by zero + // three times over. complexityBigO() reaches this whenever it is called on a Bench where no + // complexityN was ever set, since collectRangeMeasure() then returns nothing - the six models + // used to come out all-NaN, which prints as "nan" and, being unordered, is a poor thing to hand + // to std::sort. + if (rangeMeasure.empty()) { + return; + } + + // estimate the constant factor + double sumRangeMeasure = 0.0; + double sumRangeRange = 0.0; + + for (const auto& rm : rangeMeasure) { + sumRangeMeasure += rm.first * rm.second; + sumRangeRange += rm.first * rm.first; + } + // a sum of squares, so this is only 0 when every n is + mConstant = sumRangeRange <= 0.0 ? 0.0 : sumRangeMeasure / sumRangeRange; + + // calculate root mean square + double err = 0.0; + double sumMeasure = 0.0; + for (const auto& rm : rangeMeasure) { + auto diff = mConstant * rm.first - rm.second; + err += diff * diff; + + sumMeasure += rm.second; + } + + auto n = detail::d(rangeMeasure.size()); + auto mean = sumMeasure / n; + // the error is relative to the mean measurement, so there is no relative error to report when + // everything measured as 0 - and 0/0 is not it + mNormalizedRootMeanSquare = mean <= 0.0 ? 0.0 : std::sqrt(err / n) / mean; +} + +BigO::BigO(const char* bigOName, RangeMeasure const& rangeMeasure) + : BigO(std::string(bigOName), rangeMeasure) {} + +std::string const& BigO::name() const noexcept { + return mName; +} + +double BigO::constant() const noexcept { + return mConstant; +} + +double BigO::normalizedRootMeanSquare() const noexcept { + return mNormalizedRootMeanSquare; +} + +bool BigO::operator<(BigO const& other) const noexcept { + return (mNormalizedRootMeanSquare < other.mNormalizedRootMeanSquare) || + (!(mNormalizedRootMeanSquare > other.mNormalizedRootMeanSquare) && mName < other.mName); +} + +std::ostream& operator<<(std::ostream& os, BigO const& bigO) { + return os << bigO.constant() << " * " << bigO.name() << ", rms=" << bigO.normalizedRootMeanSquare(); +} + +std::ostream& operator<<(std::ostream& os, std::vector const& bigOs) { + detail::fmt::StreamStateRestorer const restorer(os); + os << std::endl << "| coefficient | err% | complexity" << std::endl << "|--------------:|-------:|------------" << std::endl; + for (auto const& bigO : bigOs) { + os << "|" << std::setw(14) << std::setprecision(7) << std::scientific << bigO.constant() << " "; + os << "|" << detail::fmt::Number(6, 1, bigO.normalizedRootMeanSquare() * 100.0) << "% "; + os << "| " << bigO.name(); + os << std::endl; + } + return os; +} + +} // namespace nanobench +} // namespace ankerl + +#endif // ANKERL_NANOBENCH_IMPLEMENT +#endif // ANKERL_NANOBENCH_H_INCLUDED