- Literal transliteration of `Statement.closedForm` plus an equivalent summarised implementation - Borrowed input keys (`std::string_view` / `std::span`) and owned result keys (`std::string`) - Golden-vector test, differential/safety fuzzers, and a nanobench comparison The literal and summarised implementations agree on 40,000 Lean golden vectors, 18,971 sorted corpus cases, and the readable fuzzer; the raw fuzzer runs both implementations under ASan/UBSan. `build/` and the generated `oracle/golden.txt` are ignored.
5474 lines
222 KiB
C++
5474 lines
222 KiB
C++
// __ _ _______ __ _ _____ ______ _______ __ _ _______ _ _
|
|
// | \ | |_____| | \ | | | |_____] |______ | \ | | |_____|
|
|
// | \_| | | | \_| |_____| |_____] |______ | \_| |_____ | |
|
|
//
|
|
// Microbenchmark framework for C++11/14/17/20
|
|
// https://github.com/martinus/nanobench
|
|
//
|
|
// Licensed under the MIT License <http://opensource.org/licenses/MIT>.
|
|
// SPDX-License-Identifier: MIT
|
|
// Copyright (c) 2019-2023 Martin Leitner-Ankerl <martin.ankerl@gmail.com>
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
// of this software and associated documentation files (the "Software"), to deal
|
|
// in the Software without restriction, including without limitation the rights
|
|
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
// copies of the Software, and to permit persons to whom the Software is
|
|
// furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in all
|
|
// copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
// SOFTWARE.
|
|
|
|
#ifndef ANKERL_NANOBENCH_H_INCLUDED
|
|
#define ANKERL_NANOBENCH_H_INCLUDED
|
|
|
|
// see https://semver.org/
|
|
#define ANKERL_NANOBENCH_VERSION_MAJOR 4 // incompatible API changes
|
|
#define ANKERL_NANOBENCH_VERSION_MINOR 6 // backwards-compatible changes
|
|
#define ANKERL_NANOBENCH_VERSION_PATCH 0 // backwards-compatible bug fixes
|
|
|
|
///////////////////////////////////////////////////////////////////////////////////////////////////
|
|
// public facing api - as minimal as possible
|
|
///////////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
#include <chrono> // high_resolution_clock
|
|
#include <cmath> // log & exp for the paired A/B statistics
|
|
#include <cstring> // memcpy
|
|
#include <iosfwd> // for std::ostream* custom output target in Config
|
|
#include <string> // all names
|
|
#include <unordered_map> // holds context information of results
|
|
#include <vector> // holds all results
|
|
|
|
#define ANKERL_NANOBENCH(x) ANKERL_NANOBENCH_PRIVATE_##x()
|
|
|
|
// MSVC reports __cplusplus as 199711L whatever /std: says, unless /Zc:__cplusplus is passed - and it
|
|
// is not passed by default. _MSVC_LANG carries the real value. Getting this wrong does not fail to
|
|
// compile, it silently takes the pre-C++17 branch on MSVC, which is how [[nodiscard]] was missing
|
|
// there.
|
|
#if defined(_MSVC_LANG)
|
|
# define ANKERL_NANOBENCH_PRIVATE_CXX() _MSVC_LANG
|
|
#else
|
|
# define ANKERL_NANOBENCH_PRIVATE_CXX() __cplusplus
|
|
#endif
|
|
#define ANKERL_NANOBENCH_PRIVATE_CXX98() 199711L
|
|
#define ANKERL_NANOBENCH_PRIVATE_CXX11() 201103L
|
|
#define ANKERL_NANOBENCH_PRIVATE_CXX14() 201402L
|
|
#define ANKERL_NANOBENCH_PRIVATE_CXX17() 201703L
|
|
|
|
#if ANKERL_NANOBENCH(CXX) >= ANKERL_NANOBENCH(CXX17)
|
|
# define ANKERL_NANOBENCH_PRIVATE_NODISCARD() [[nodiscard]]
|
|
# define ANKERL_NANOBENCH_PRIVATE_HAS_STRING_VIEW() 1
|
|
#else
|
|
# define ANKERL_NANOBENCH_PRIVATE_NODISCARD()
|
|
# define ANKERL_NANOBENCH_PRIVATE_HAS_STRING_VIEW() 0
|
|
#endif
|
|
|
|
#if ANKERL_NANOBENCH(HAS_STRING_VIEW)
|
|
# include <string_view> // the name() and run() overloads
|
|
#endif
|
|
|
|
#if defined(__clang__)
|
|
# define ANKERL_NANOBENCH_PRIVATE_IGNORE_PADDED_PUSH() \
|
|
_Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wpadded\"")
|
|
# define ANKERL_NANOBENCH_PRIVATE_IGNORE_PADDED_POP() _Pragma("clang diagnostic pop")
|
|
#else
|
|
# define ANKERL_NANOBENCH_PRIVATE_IGNORE_PADDED_PUSH()
|
|
# define ANKERL_NANOBENCH_PRIVATE_IGNORE_PADDED_POP()
|
|
#endif
|
|
|
|
#if defined(__GNUC__)
|
|
# define ANKERL_NANOBENCH_PRIVATE_IGNORE_EFFCPP_PUSH() _Pragma("GCC diagnostic push") _Pragma("GCC diagnostic ignored \"-Weffc++\"")
|
|
# define ANKERL_NANOBENCH_PRIVATE_IGNORE_EFFCPP_POP() _Pragma("GCC diagnostic pop")
|
|
#else
|
|
# define ANKERL_NANOBENCH_PRIVATE_IGNORE_EFFCPP_PUSH()
|
|
# define ANKERL_NANOBENCH_PRIVATE_IGNORE_EFFCPP_POP()
|
|
#endif
|
|
|
|
#if defined(ANKERL_NANOBENCH_LOG_ENABLED)
|
|
# include <iostream>
|
|
# define ANKERL_NANOBENCH_LOG(x) \
|
|
do { \
|
|
std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << x << std::endl; \
|
|
} while (0)
|
|
#else
|
|
# define ANKERL_NANOBENCH_LOG(x) \
|
|
do { \
|
|
} while (0)
|
|
#endif
|
|
|
|
#define ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS() 0
|
|
#if defined(__linux__) && !defined(ANKERL_NANOBENCH_DISABLE_PERF_COUNTERS)
|
|
# include <linux/version.h>
|
|
# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0)
|
|
// PERF_COUNT_HW_REF_CPU_CYCLES only available since kernel 3.3
|
|
// PERF_FLAG_FD_CLOEXEC since kernel 3.14
|
|
# undef ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS
|
|
# define ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS() 1
|
|
# endif
|
|
#endif
|
|
|
|
#if defined(__clang__)
|
|
# define ANKERL_NANOBENCH_NO_SANITIZE(...) __attribute__((no_sanitize(__VA_ARGS__)))
|
|
#else
|
|
# define ANKERL_NANOBENCH_NO_SANITIZE(...)
|
|
#endif
|
|
|
|
#if defined(_MSC_VER)
|
|
# define ANKERL_NANOBENCH_PRIVATE_NOINLINE() __declspec(noinline)
|
|
#else
|
|
# define ANKERL_NANOBENCH_PRIVATE_NOINLINE() __attribute__((noinline))
|
|
#endif
|
|
|
|
// Whether doNotOptimizeAway can use GCC-style inline assembly, which is both faster and a stronger barrier than the
|
|
// function call it falls back to. Testing for _MSC_VER is not enough to rule it out: clang-cl and clang in MSVC
|
|
// compatibility mode define _MSC_VER but not __GNUC__, even though they do understand the assembly.
|
|
// See https://github.com/martinus/nanobench/issues/111
|
|
#if defined(__clang__) || defined(__GNUC__)
|
|
# define ANKERL_NANOBENCH_PRIVATE_ASM_DONT_OPTIMIZE_AWAY() 1
|
|
#else
|
|
# define ANKERL_NANOBENCH_PRIVATE_ASM_DONT_OPTIMIZE_AWAY() 0
|
|
#endif
|
|
|
|
// workaround missing "is_trivially_copyable" in g++ < 5.0
|
|
// See https://stackoverflow.com/a/31798726/48181
|
|
#if defined(__GNUC__) && __GNUC__ < 5
|
|
# define ANKERL_NANOBENCH_IS_TRIVIALLY_COPYABLE(...) __has_trivial_copy(__VA_ARGS__)
|
|
#else
|
|
# define ANKERL_NANOBENCH_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value
|
|
#endif
|
|
|
|
// noexcept may be missing for std::string.
|
|
// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58265
|
|
#define ANKERL_NANOBENCH_PRIVATE_NOEXCEPT_STRING_MOVE() std::is_nothrow_move_assignable<std::string>::value
|
|
|
|
// nanobench throws only for programming errors in a render template - a tag that does not exist, a
|
|
// section in the wrong place. With exceptions turned off there is nothing sensible left to do but
|
|
// stop, so the argument is never evaluated and the message building disappears entirely.
|
|
//
|
|
// std::abort() rather than a per-compiler intrinsic on purpose: the branch that would pick one is
|
|
// the branch no CI leg can reach, and an untested call into CRT internals is exactly the kind of
|
|
// thing that turns out not to compile the first time someone actually needs it.
|
|
#if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)
|
|
# define ANKERL_NANOBENCH_THROW(...) throw __VA_ARGS__
|
|
#else
|
|
# define ANKERL_NANOBENCH_THROW(...) std::abort()
|
|
#endif
|
|
|
|
// declarations ///////////////////////////////////////////////////////////////////////////////////
|
|
|
|
namespace ankerl {
|
|
namespace nanobench {
|
|
|
|
using Clock = std::conditional<std::chrono::high_resolution_clock::is_steady, std::chrono::high_resolution_clock,
|
|
std::chrono::steady_clock>::type;
|
|
class Bench;
|
|
struct Config;
|
|
class Result;
|
|
class Rng;
|
|
class BigO;
|
|
class CompareResult;
|
|
|
|
namespace detail {
|
|
template <typename SetupOp>
|
|
class SetupRunner;
|
|
|
|
// One of compare()'s alternatives, with its type forgotten so that they can live in a vector.
|
|
//
|
|
// The alternatives all have different types, so holding them needs either type erasure or a template
|
|
// recursion that walks the pack at every step. This is the erasure, and it costs nothing that
|
|
// matters: `run` points at an instantiation that knows the concrete type, so the measuring loop is
|
|
// inside a function where the operation still inlines exactly as it would otherwise. What is added
|
|
// is one indirect call per *epoch*, against an epoch of a millisecond.
|
|
//
|
|
// std::function would not do, because it would put that indirection in the inner loop instead.
|
|
struct ErasedOp {
|
|
void (*run)(void* op, uint64_t numIters); // NOLINT(misc-non-private-member-variables-in-classes)
|
|
void* op; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
};
|
|
|
|
template <typename Op>
|
|
void runErasedOp(void* op, uint64_t numIters) {
|
|
auto& concrete = *static_cast<Op*>(op);
|
|
for (uint64_t i = 0; i < numIters; ++i) {
|
|
concrete();
|
|
}
|
|
}
|
|
|
|
template <typename Op>
|
|
ErasedOp eraseOp(Op&& op) {
|
|
using Bare = typename std::remove_reference<Op>::type;
|
|
// The operation is only ever called, never written to, so casting a const one back to non-const
|
|
// to fit it through void* is safe - a const lambda's operator() is const or it could not be
|
|
// called at all.
|
|
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast)
|
|
return ErasedOp{&runErasedOp<Bare>, const_cast<void*>(static_cast<void const*>(&op))};
|
|
}
|
|
} // namespace detail
|
|
|
|
/**
|
|
* @brief Renders output from a mustache-like template and benchmark results.
|
|
*
|
|
* The templating facility here is heavily inspired by [mustache - logic-less templates](https://mustache.github.io/).
|
|
* It adds a few more features that are necessary to get all of the captured data out of nanobench. Please read the
|
|
* excellent [mustache manual](https://mustache.github.io/mustache.5.html) to see what this is all about.
|
|
*
|
|
* nanobench output has two nested layers, *result* and *measurement*. Here is a hierarchy of the allowed tags:
|
|
*
|
|
* * `{{#result}}` Marks the begin of the result layer. Whatever comes after this will be instantiated as often as
|
|
* a benchmark result is available. Within it, you can use these tags:
|
|
*
|
|
* * `{{title}}` See Bench::title.
|
|
*
|
|
* * `{{name}}` Benchmark name, usually directly provided with Bench::run, but can also be set with Bench::name.
|
|
*
|
|
* * `{{unit}}` Unit, e.g. `byte`. Defaults to `op`, see Bench::unit.
|
|
*
|
|
* * `{{batch}}` Batch size, see Bench::batch.
|
|
*
|
|
* * `{{complexityN}}` Value used for asymptotic complexity calculation. See Bench::complexityN.
|
|
*
|
|
* * `{{epochs}}` Number of epochs, see Bench::epochs.
|
|
*
|
|
* * `{{clockResolution}}` Accuracy of the clock, i.e. what's the smallest time possible to measure with the clock.
|
|
* For modern systems, this can be around 20 ns. This value is automatically determined by nanobench at the first
|
|
* benchmark that is run, and used as a static variable throughout the application's runtime.
|
|
*
|
|
* * `{{clockResolutionMultiple}}` Configuration multiplier for `clockResolution`. See Bench::clockResolutionMultiple.
|
|
* This is the target runtime for each measurement (epoch). That means the more accurate your clock is, the faster
|
|
* will be the benchmark. Basing the measurement's runtime on the clock resolution is the main reason why nanobench is so fast.
|
|
*
|
|
* * `{{maxEpochTime}}` Configuration for a maximum time each measurement (epoch) is allowed to take. Note that at least
|
|
* a single iteration will be performed, even when that takes longer than maxEpochTime. See Bench::maxEpochTime.
|
|
*
|
|
* * `{{minEpochTime}}` Minimum epoch time, defaults to 1ms. See Bench::minEpochTime.
|
|
*
|
|
* * `{{minEpochIterations}}` See Bench::minEpochIterations.
|
|
*
|
|
* * `{{epochIterations}}` See Bench::epochIterations.
|
|
*
|
|
* * `{{warmup}}` Number of iterations used before measuring starts. See Bench::warmup.
|
|
*
|
|
* * `{{relative}}` True or false, depending on the setting you have used. See Bench::relative.
|
|
*
|
|
* * `{{context(variableName)}}` See Bench::context.
|
|
*
|
|
* Apart from these tags, it is also possible to use some mathematical operations on the measurement data. The operations
|
|
* are of the form `{{command(name)}}`. Currently `name` can be one of `elapsed`, `iterations`. If performance counters
|
|
* are available (currently only on current Linux systems), you also have `pagefaults`, `cpucycles`,
|
|
* `contextswitches`, `instructions`, `branchinstructions`, and `branchmisses`. All the measures (except `iterations`) are
|
|
* provided for a single iteration (so `elapsed` is the time a single iteration took). The following tags are available:
|
|
*
|
|
* `elapsed` is in **seconds**, which is rarely the unit a report wants. Since the template language has no arithmetic
|
|
* to scale it afterwards, the time measure also comes in `elapsedms`, `elapsedus` and `elapsedns` — the same value in
|
|
* milliseconds, microseconds and nanoseconds. So `{{minimum(elapsedms)}}` is the fastest iteration in milliseconds.
|
|
* Note this is unrelated to Bench::timeUnit(), which only sets the unit of the `ns/op` column in the table.
|
|
* `{{medianAbsolutePercentError(...)}}` is a relative error, so it is the same number whichever of them you ask for.
|
|
*
|
|
* * `{{median(<name>)}}` Calculate median of a measurement data set, e.g. `{{median(elapsed)}}`.
|
|
*
|
|
* * `{{average(<name>)}}` Average (mean) calculation.
|
|
*
|
|
* * `{{medianAbsolutePercentError(<name>)}}` Calculates MdAPE, the Median Absolute Percentage Error. The MdAPE is an excellent
|
|
* metric for the variation of measurements. It is more robust to outliers than the
|
|
* [Mean absolute percentage error (M-APE)](https://en.wikipedia.org/wiki/Mean_absolute_percentage_error).
|
|
* @f[
|
|
* \mathrm{MdAPE}(e) = \mathrm{med}\{| \frac{e_i - \mathrm{med}\{e\}}{e_i}| \}
|
|
* @f]
|
|
* E.g. for *elapsed*: First, @f$ \mathrm{med}\{e\} @f$ calculates the median by sorting and then taking the middle element
|
|
* of all *elapsed* measurements. This is used to calculate the absolute percentage
|
|
* error to this median for each measurement, as in @f$ | \frac{e_i - \mathrm{med}\{e\}}{e_i}| @f$. All these results
|
|
* are sorted, and the middle value is chosen as the median absolute percent error.
|
|
*
|
|
* This measurement is a bit hard to interpret, but it is very robust against outliers. E.g. a value of 5% means that half of the
|
|
* measurements deviate less than 5% from the median, and the other deviate more than 5% from the median.
|
|
*
|
|
* * `{{sum(<name>)}}` Sum of all the measurements. E.g. `{{sum(iterations)}}` will give you the total number of iterations
|
|
* measured in this benchmark.
|
|
*
|
|
* * `{{minimum(<name>)}}` Minimum of all measurements.
|
|
*
|
|
* * `{{maximum(<name>)}}` Maximum of all measurements.
|
|
*
|
|
* * `{{sumProduct(<first>, <second>)}}` Calculates the sum of the products of corresponding measures:
|
|
* @f[
|
|
* \mathrm{sumProduct}(a,b) = \sum_{i=1}^{n}a_i\cdot b_i
|
|
* @f]
|
|
* E.g. to calculate total runtime of the benchmark, you multiply iterations with elapsed time for each measurement, and
|
|
* sum these results up:
|
|
* `{{sumProduct(iterations, elapsed)}}`.
|
|
*
|
|
* * `{{#measurement}}` To access individual measurement results, open the begin tag for measurements.
|
|
*
|
|
* * `{{elapsed}}` Average elapsed wall clock time per iteration, in seconds.
|
|
*
|
|
* * `{{iterations}}` Number of iterations in the measurement. The number of iterations will fluctuate due
|
|
* to some applied randomness, to enhance accuracy.
|
|
*
|
|
* * `{{pagefaults}}` Average number of pagefaults per iteration.
|
|
*
|
|
* * `{{cpucycles}}` Average number of CPU cycles processed per iteration.
|
|
*
|
|
* * `{{contextswitches}}` Average number of context switches per iteration.
|
|
*
|
|
* * `{{instructions}}` Average number of retired instructions per iteration.
|
|
*
|
|
* * `{{branchinstructions}}` Average number of branches executed per iteration.
|
|
*
|
|
* * `{{branchmisses}}` Average number of branches that were missed per iteration.
|
|
*
|
|
* * `{{/measurement}}` Ends the measurement tag.
|
|
*
|
|
* * `{{/result}}` Marks the end of the result layer. This is the end marker for the template part that will be instantiated
|
|
* for each benchmark result.
|
|
*
|
|
*
|
|
* For the layer tags *result* and *measurement* you additionally can use these special markers:
|
|
*
|
|
* * ``{{#-first}}`` - Begin marker of a template that will be instantiated *only for the first* entry in the layer. Use is only
|
|
* allowed between the begin and end marker of the layer. So between ``{{#result}}`` and ``{{/result}}``, or between
|
|
* ``{{#measurement}}`` and ``{{/measurement}}``. Finish the template with ``{{/-first}}``.
|
|
*
|
|
* * ``{{^-first}}`` - Begin marker of a template that will be instantiated *for each except the first* entry in the layer. This,
|
|
* this is basically the inversion of ``{{#-first}}``. Use is only allowed between the begin and end marker of the layer.
|
|
* So between ``{{#result}}`` and ``{{/result}}``, or between ``{{#measurement}}`` and ``{{/measurement}}``.
|
|
*
|
|
* * ``{{/-first}}`` - End marker for either ``{{#-first}}`` or ``{{^-first}}``.
|
|
*
|
|
* * ``{{#-last}}`` - Begin marker of a template that will be instantiated *only for the last* entry in the layer. Use is only
|
|
* allowed between the begin and end marker of the layer. So between ``{{#result}}`` and ``{{/result}}``, or between
|
|
* ``{{#measurement}}`` and ``{{/measurement}}``. Finish the template with ``{{/-last}}``.
|
|
*
|
|
* * ``{{^-last}}`` - Begin marker of a template that will be instantiated *for each except the last* entry in the layer. This,
|
|
* this is basically the inversion of ``{{#-last}}``. Use is only allowed between the begin and end marker of the layer.
|
|
* So between ``{{#result}}`` and ``{{/result}}``, or between ``{{#measurement}}`` and ``{{/measurement}}``.
|
|
*
|
|
* * ``{{/-last}}`` - End marker for either ``{{#-last}}`` or ``{{^-last}}``.
|
|
*
|
|
@verbatim embed:rst
|
|
|
|
For an overview of all the possible data you can get out of nanobench, please see the tutorial at :ref:`tutorial-template-json`.
|
|
|
|
The templates that ship with nanobench are:
|
|
|
|
* :cpp:func:`templates::csv() <ankerl::nanobench::templates::csv()>`
|
|
* :cpp:func:`templates::json() <ankerl::nanobench::templates::json()>`
|
|
* :cpp:func:`templates::htmlBoxplot() <ankerl::nanobench::templates::htmlBoxplot()>`
|
|
* :cpp:func:`templates::pyperf() <ankerl::nanobench::templates::pyperf()>`
|
|
|
|
@endverbatim
|
|
*
|
|
* @param mustacheTemplate The template.
|
|
* @param bench Benchmark, containing all the results.
|
|
* @param out Output for the generated output.
|
|
*/
|
|
void render(char const* mustacheTemplate, Bench const& bench, std::ostream& out);
|
|
void render(std::string const& mustacheTemplate, Bench const& bench, std::ostream& out);
|
|
|
|
/**
|
|
* Same as render(char const* mustacheTemplate, Bench const& bench, std::ostream& out), but for when
|
|
* you only have results available.
|
|
*
|
|
* @param mustacheTemplate The template.
|
|
* @param results All the results to be used for rendering.
|
|
* @param out Output for the generated output.
|
|
*/
|
|
void render(char const* mustacheTemplate, std::vector<Result> const& results, std::ostream& out);
|
|
void render(std::string const& mustacheTemplate, std::vector<Result> const& results, std::ostream& out);
|
|
|
|
/**
|
|
* @brief Same, for the outcome of a Bench::compare().
|
|
*
|
|
@verbatim embed:rst
|
|
The section is ``{{#alternative}}``, one iteration per alternative with the baseline first, and
|
|
inside it the ratio tags ``{{relative}}``, ``{{relativeLow}}``, ``{{relativeHigh}}``,
|
|
``{{tiedRounds}}`` and ``{{significant}}`` are available on top of everything a result tag
|
|
already offers, so ``{{median(elapsed)}}`` and ``{{title}}`` work as they do elsewhere.
|
|
``{{rounds}}`` and ``{{comparisons}}`` describe the experiment and can be used anywhere.
|
|
|
|
Without this a comparison could only be read by a human out of the printed table:
|
|
:cpp:func:`templates::csv() <ankerl::nanobench::templates::csv()>` and its neighbours take a
|
|
:cpp:class:`Bench <ankerl::nanobench::Bench>` or a vector of
|
|
:cpp:class:`Result <ankerl::nanobench::Result>`, neither of which carries a ratio or its
|
|
interval. See :ref:`tutorial-template-compare-csv`.
|
|
@endverbatim
|
|
*/
|
|
void render(char const* mustacheTemplate, CompareResult const& cmp, std::ostream& out);
|
|
void render(std::string const& mustacheTemplate, CompareResult const& cmp, std::ostream& out);
|
|
|
|
// Contains mustache-like templates
|
|
namespace templates {
|
|
|
|
/*!
|
|
@brief CSV data for the benchmark results.
|
|
|
|
Generates a comma-separated values dataset. First line is the header, each following line is a summary of each benchmark run.
|
|
|
|
@verbatim embed:rst
|
|
See the tutorial at :ref:`tutorial-template-csv` for an example.
|
|
@endverbatim
|
|
*/
|
|
char const* csv() noexcept;
|
|
|
|
/*!
|
|
@brief CSV data for a paired comparison, one row per alternative.
|
|
|
|
Carries the ratio and the bounds of its confidence interval, which is what separates a comparison
|
|
from a set of independent runs. Meant for ``render(templates::compareCsv(), cmp, out)``.
|
|
|
|
@verbatim embed:rst
|
|
See the tutorial at :ref:`tutorial-template-compare-csv` for an example.
|
|
@endverbatim
|
|
*/
|
|
char const* compareCsv() noexcept;
|
|
|
|
/*!
|
|
@brief HTML output that uses plotly to generate an interactive boxplot chart. See the tutorial for an example output.
|
|
|
|
The output uses only the elapsed wall clock time, and displays each epoch as a single dot.
|
|
@verbatim embed:rst
|
|
See the tutorial at :ref:`tutorial-template-html` for an example.
|
|
@endverbatim
|
|
|
|
@see also ankerl::nanobench::render()
|
|
*/
|
|
char const* htmlBoxplot() noexcept;
|
|
|
|
/*!
|
|
@brief Output in pyperf compatible JSON format, which can be used for more analyzation.
|
|
@verbatim embed:rst
|
|
See the tutorial at :ref:`tutorial-template-pyperf` for an example how to further analyze the output.
|
|
@endverbatim
|
|
*/
|
|
char const* pyperf() noexcept;
|
|
|
|
/*!
|
|
@brief Template to generate JSON data.
|
|
|
|
The generated JSON data contains *all* data that has been generated. All times are as double values, in seconds. The output can get
|
|
quite large.
|
|
@verbatim embed:rst
|
|
See the tutorial at :ref:`tutorial-template-json` for an example.
|
|
@endverbatim
|
|
*/
|
|
char const* json() noexcept;
|
|
|
|
} // namespace templates
|
|
|
|
namespace detail {
|
|
|
|
template <typename T>
|
|
struct PerfCountSet;
|
|
|
|
class IterationLogic;
|
|
class PerformanceCounters;
|
|
|
|
#if ANKERL_NANOBENCH(PERF_COUNTERS)
|
|
class LinuxPerformanceCounters;
|
|
#endif
|
|
|
|
} // namespace detail
|
|
} // namespace nanobench
|
|
} // namespace ankerl
|
|
|
|
// definitions ////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
namespace ankerl {
|
|
namespace nanobench {
|
|
namespace detail {
|
|
|
|
template <typename T>
|
|
struct PerfCountSet {
|
|
T pageFaults{};
|
|
T cpuCycles{};
|
|
T contextSwitches{};
|
|
T instructions{};
|
|
T branchInstructions{};
|
|
T branchMisses{};
|
|
};
|
|
|
|
} // namespace detail
|
|
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_PUSH)
|
|
struct Config {
|
|
// actual benchmark config
|
|
std::string mBenchmarkTitle = "benchmark"; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
std::string mBenchmarkName = "noname"; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
std::string mUnit = "op"; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
double mBatch = 1.0; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
double mComplexityN = -1.0; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
size_t mNumEpochs = 11; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
size_t mClockResolutionMultiple = static_cast<size_t>(1000); // NOLINT(misc-non-private-member-variables-in-classes)
|
|
std::chrono::nanoseconds mMaxEpochTime = std::chrono::milliseconds(100); // NOLINT(misc-non-private-member-variables-in-classes)
|
|
std::chrono::nanoseconds mMinEpochTime = std::chrono::milliseconds(1); // NOLINT(misc-non-private-member-variables-in-classes)
|
|
uint64_t mMinEpochIterations{1}; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
// If not 0, run *exactly* these number of iterations per epoch.
|
|
uint64_t mEpochIterations{0}; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
uint64_t mWarmup = 0; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
std::ostream* mOut = nullptr; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
std::chrono::duration<double> mTimeUnit = std::chrono::nanoseconds{1}; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
std::string mTimeUnitName = "ns"; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
bool mShowPerformanceCounters = true; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
bool mIsRelative = false; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
std::unordered_map<std::string, std::string> mContext{}; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
// One bit per Column, set when that column is hidden - so the default of 0 shows everything and
|
|
// stays the table nanobench has always printed.
|
|
uint32_t mHiddenColumns{}; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
std::vector<std::string> mContextColumns{}; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
// 0 disables: compare() then runs exactly mNumEpochs rounds, as it always has.
|
|
double mTargetIntervalWidth{0.0}; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
size_t mMaxEpochs{1000}; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
|
|
Config();
|
|
~Config();
|
|
Config& operator=(Config const& other);
|
|
Config& operator=(Config&& other) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE));
|
|
Config(Config const& other);
|
|
Config(Config&& other) noexcept;
|
|
};
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_POP)
|
|
|
|
// Result returned after a benchmark has finished. Can be used as a baseline for relative().
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_PUSH)
|
|
class Result {
|
|
public:
|
|
enum class Measure : size_t {
|
|
elapsed,
|
|
iterations,
|
|
pagefaults,
|
|
cpucycles,
|
|
contextswitches,
|
|
instructions,
|
|
branchinstructions,
|
|
branchmisses,
|
|
|
|
/// Number of measures, and what fromString() returns for a name it does not know. Passing it
|
|
/// to the accessors below is not an error: it reads as a measure that was never recorded.
|
|
_size
|
|
};
|
|
|
|
explicit Result(Config benchmarkConfig);
|
|
|
|
~Result();
|
|
Result& operator=(Result const& other);
|
|
Result& operator=(Result&& other) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE));
|
|
Result(Result const& other);
|
|
Result(Result&& other) noexcept;
|
|
|
|
// adds new measurement results
|
|
// all values are scaled by iters (except iters...)
|
|
void add(Clock::duration totalElapsed, uint64_t iters, detail::PerformanceCounters const& pc);
|
|
|
|
ANKERL_NANOBENCH(NODISCARD) Config const& config() const noexcept;
|
|
|
|
ANKERL_NANOBENCH(NODISCARD) double median(Measure m) const;
|
|
ANKERL_NANOBENCH(NODISCARD) double medianAbsolutePercentError(Measure m) const;
|
|
ANKERL_NANOBENCH(NODISCARD) double average(Measure m) const;
|
|
ANKERL_NANOBENCH(NODISCARD) double sum(Measure m) const noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) double sumProduct(Measure m1, Measure m2) const noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) double minimum(Measure m) const noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) double maximum(Measure m) const noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) std::string const& context(char const* variableName) const;
|
|
ANKERL_NANOBENCH(NODISCARD) std::string const& context(std::string const& variableName) const;
|
|
|
|
ANKERL_NANOBENCH(NODISCARD) bool has(Measure m) const noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) double get(size_t idx, Measure m) const;
|
|
ANKERL_NANOBENCH(NODISCARD) bool empty() const noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) size_t size() const noexcept;
|
|
|
|
// Finds string, if not found, returns _size.
|
|
static Measure fromString(std::string const& str);
|
|
|
|
private:
|
|
// The per-epoch values of one measure. Every accessor above goes through this, so how the
|
|
// measures are stored - including the extra slot the constructor explains - is written down once
|
|
// rather than asserted at a dozen call sites.
|
|
ANKERL_NANOBENCH(NODISCARD) std::vector<double> const& measurements(Measure m) const;
|
|
|
|
Config mConfig{};
|
|
std::vector<std::vector<double>> mNameToMeasurements{};
|
|
};
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_POP)
|
|
|
|
/**
|
|
* @brief The outcome of a paired comparison, see Bench::compare().
|
|
*
|
|
@verbatim embed:rst
|
|
:cpp:func:`relative() <ankerl::nanobench::Bench::relative()>` compares the median of one benchmark
|
|
run entirely before another, which measures the machine's drift as much as the code: nanobench's
|
|
own test suite has a case where two *identical* workloads came out 38% apart on a CI runner, while
|
|
each reported an ``err%`` of 0.5. Whatever ``err%`` is measuring there, it is not the uncertainty
|
|
of the comparison.
|
|
|
|
A paired comparison measures the alternatives against each other within the same slice of time, so
|
|
anything that affects all of them - a frequency ramp, a noisy neighbour, thermal throttling -
|
|
cancels out of the ratios. What is reported is therefore an uncertainty about *the ratio*, which
|
|
is the number the caller actually wanted.
|
|
|
|
Entry 0 is the baseline that everything else is measured against.
|
|
@endverbatim
|
|
*/
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_PUSH)
|
|
class CompareResult {
|
|
public:
|
|
/// One alternative: its measurements, and how it compares to the baseline.
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_PUSH)
|
|
struct Entry {
|
|
// noexcept because it only moves, and both std::string's and Result's move constructors are:
|
|
// gcc's -Wnoexcept fails the build over the vector growth path otherwise.
|
|
Entry(std::string entryName, Result entryResult, double entryRelative, double entryRelativeLow, double entryRelativeHigh,
|
|
size_t entryTiedRounds) noexcept;
|
|
|
|
std::string name; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
Result result; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
double relative; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
double relativeLow; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
double relativeHigh; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
size_t tiedRounds; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
};
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_POP)
|
|
|
|
CompareResult(std::vector<Entry> entries, size_t numRounds);
|
|
|
|
/// Number of alternatives compared, including the baseline.
|
|
ANKERL_NANOBENCH(NODISCARD) size_t size() const noexcept;
|
|
|
|
/// Entry 0 is the baseline.
|
|
ANKERL_NANOBENCH(NODISCARD) Entry const& operator[](size_t idx) const;
|
|
|
|
/// Number of paired rounds each alternative was measured over.
|
|
ANKERL_NANOBENCH(NODISCARD) size_t rounds() const noexcept;
|
|
|
|
/**
|
|
* @brief How many comparisons the intervals were corrected for: one per alternative besides the
|
|
* baseline.
|
|
*
|
|
@verbatim embed:rst
|
|
With ten alternatives there are nine chances to be wrong at 5% each, so the intervals are
|
|
widened to keep the *whole table* at 95% rather than each row separately. See
|
|
:ref:`ab-comparison` for what that costs - less than it sounds, because the binomial tail is
|
|
steep.
|
|
@endverbatim
|
|
*/
|
|
ANKERL_NANOBENCH(NODISCARD) size_t comparisons() const noexcept;
|
|
|
|
/**
|
|
* @brief True when this alternative's interval excludes the baseline, i.e. the measurement told
|
|
* them apart. Always false for the baseline itself.
|
|
*/
|
|
ANKERL_NANOBENCH(NODISCARD) bool isSignificant(size_t idx) const;
|
|
|
|
/// Index of the alternative with the lowest median time.
|
|
ANKERL_NANOBENCH(NODISCARD) size_t fastest() const;
|
|
|
|
private:
|
|
std::vector<Entry> mEntries{};
|
|
size_t mRounds{};
|
|
};
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_POP)
|
|
|
|
/// Writes the comparison as a markdown table, in the same shape as an ordinary benchmark table, with
|
|
/// a summary line under it.
|
|
std::ostream& operator<<(std::ostream& os, CompareResult const& compareResult);
|
|
|
|
/**
|
|
* An extremely fast random generator. Currently, this implements *RomuDuoJr*, developed by Mark Overton. Source:
|
|
* http://www.romu-random.org/
|
|
*
|
|
* RomuDuoJr is extremely fast and provides reasonable good randomness. Not enough for large jobs, but definitely
|
|
* good enough for a benchmarking framework.
|
|
*
|
|
* * Estimated capacity: @f$ 2^{51} @f$ bytes
|
|
* * Register pressure: 4
|
|
* * State size: 128 bits
|
|
*
|
|
* This random generator is a drop-in replacement for the generators supplied by ``<random>``. It is not
|
|
* cryptographically secure. It's intended purpose is to be very fast so that benchmarks that make use
|
|
* of randomness are not distorted too much by the random generator.
|
|
*
|
|
* Rng also provides a few non-standard helpers, optimized for speed.
|
|
*/
|
|
class Rng final {
|
|
public:
|
|
/**
|
|
* @brief This RNG provides 64bit randomness.
|
|
*/
|
|
using result_type = uint64_t;
|
|
|
|
static constexpr uint64_t(min)();
|
|
static constexpr uint64_t(max)();
|
|
|
|
/**
|
|
* As a safety precaution, we don't allow copying. Copying a PRNG would mean you would have two random generators that produce the
|
|
* same sequence, which is generally not what one wants. Instead create a new rng with the default constructor Rng(), which is
|
|
* automatically seeded from `std::random_device`. If you really need a copy, use `copy()`.
|
|
*/
|
|
Rng(Rng const&) = delete;
|
|
|
|
/**
|
|
* Same as Rng(Rng const&), we don't allow assignment. If you need a new Rng create one with the default constructor Rng().
|
|
*/
|
|
Rng& operator=(Rng const&) = delete;
|
|
|
|
// moving is ok
|
|
Rng(Rng&&) noexcept = default;
|
|
Rng& operator=(Rng&&) noexcept = default;
|
|
~Rng() noexcept = default;
|
|
|
|
/**
|
|
* @brief Creates a new Random generator with random seed.
|
|
*
|
|
* Instead of a default seed (as the random generators from the STD), this properly seeds the random generator from
|
|
* `std::random_device`. It guarantees correct seeding. Note that seeding can be relatively slow, depending on the source of
|
|
* randomness used. So it is best to create a Rng once and use it for all your randomness purposes.
|
|
*/
|
|
Rng();
|
|
|
|
/*!
|
|
Creates a new Rng that is seeded with a specific seed. Each Rng created from the same seed will produce the same randomness
|
|
sequence. This can be useful for deterministic behavior.
|
|
|
|
@verbatim embed:rst
|
|
.. note::
|
|
|
|
The random algorithm might change between nanobench releases. Whenever a faster and/or better random
|
|
generator becomes available, I will switch the implementation.
|
|
@endverbatim
|
|
|
|
As per the Romu paper, this seeds the Rng with splitMix64 algorithm and performs 10 initial rounds for further mixing up of the
|
|
internal state.
|
|
|
|
@param seed The 64bit seed. All values are allowed, even 0.
|
|
*/
|
|
explicit Rng(uint64_t seed) noexcept;
|
|
Rng(uint64_t x, uint64_t y) noexcept;
|
|
explicit Rng(std::vector<uint64_t> const& data);
|
|
|
|
/**
|
|
* Creates a copy of the Rng, thus the copy provides exactly the same random sequence as the original.
|
|
*/
|
|
ANKERL_NANOBENCH(NODISCARD) Rng copy() const noexcept;
|
|
|
|
/**
|
|
* @brief Produces a 64bit random value. This should be very fast, thus it is marked as inline. In my benchmark, this is ~46 times
|
|
* faster than `std::default_random_engine` for producing 64bit random values. It seems that the fastest std contender is
|
|
* `std::mt19937_64`. Still, this RNG is 2-3 times as fast.
|
|
*
|
|
* @return uint64_t The next 64 bit random value.
|
|
*/
|
|
inline uint64_t operator()() noexcept;
|
|
|
|
/**
|
|
* Generates a random number between 0 and range (excluding range).
|
|
*
|
|
* The algorithm only produces 32bit numbers, and is slightly biased. The effect is quite small unless your range is close to the
|
|
* maximum value of an integer. It is possible to correct the bias with rejection sampling (see
|
|
* [here](https://lemire.me/blog/2016/06/30/fast-random-shuffling/), but this is most likely irrelevant in practices for the
|
|
* purposes of this Rng.
|
|
*
|
|
* See Daniel Lemire's blog post [A fast alternative to the modulo
|
|
* reduction](https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/)
|
|
*
|
|
* @param range Upper exclusive range. E.g a value of 3 will generate random numbers 0, 1, 2.
|
|
* @return uint32_t Generated random values in range [0, range(.
|
|
*/
|
|
inline uint32_t bounded(uint32_t range) noexcept;
|
|
|
|
/**
|
|
* Provides a random uniform double value between 0 and 1. This uses the method described in [Generating uniform doubles in the
|
|
* unit interval](http://prng.di.unimi.it/), and is extremely fast.
|
|
*
|
|
* @return double Uniformly distributed double value in range [0,1(, excluding 1.
|
|
*/
|
|
inline double uniform01() noexcept;
|
|
|
|
/**
|
|
* Shuffles all entries in the given container. Although this has a slight bias due to the implementation of bounded(), this is
|
|
* preferable to `std::shuffle` because it is over 5 times faster. See Daniel Lemire's blog post [Fast random
|
|
* shuffling](https://lemire.me/blog/2016/06/30/fast-random-shuffling/).
|
|
*
|
|
* @param container The whole container will be shuffled.
|
|
*/
|
|
template <typename Container>
|
|
void shuffle(Container& container) noexcept;
|
|
|
|
/**
|
|
* Extracts the full state of the generator, e.g. for serialization. For this RNG this is just 2 values, but to stay API compatible
|
|
* with future implementations that potentially use more state, we use a vector.
|
|
*
|
|
* @return Vector containing the full state:
|
|
*/
|
|
ANKERL_NANOBENCH(NODISCARD) std::vector<uint64_t> state() const;
|
|
|
|
private:
|
|
static constexpr uint64_t rotl(uint64_t x, unsigned k) noexcept;
|
|
|
|
uint64_t mX;
|
|
uint64_t mY;
|
|
};
|
|
|
|
/**
|
|
* @brief A column of the markdown table, for Bench::hideColumn() and Bench::showColumn().
|
|
*
|
|
* The names in brackets are the headers the column is printed with; `op` follows Bench::unit(), and the
|
|
* time unit follows Bench::timeUnit().
|
|
*
|
|
* @see ankerl::nanobench::Bench::hideColumn()
|
|
*/
|
|
enum class Column : size_t {
|
|
relative, ///< `relative` - only shown when Bench::relative() is set.
|
|
complexityN, ///< `complexityN` - only shown when Bench::complexityN() is set.
|
|
timePerUnit, ///< `ns/op` - time for one unit.
|
|
unitPerSecond, ///< `op/s` - units per second.
|
|
error, ///< `err%` - median absolute percentage error over the epochs.
|
|
instructions, ///< `ins/op` - retired instructions, Linux only.
|
|
cycles, ///< `cyc/op` - CPU cycles, Linux only.
|
|
ipc, ///< `IPC` - instructions per cycle, Linux only.
|
|
branches, ///< `bra/op` - retired branch instructions, Linux only.
|
|
branchMisses, ///< `miss%` - percentage of branches mispredicted, Linux only.
|
|
total, ///< `total` - wall clock time spent measuring this row.
|
|
_size ///< Not a column; the number of them.
|
|
};
|
|
|
|
/**
|
|
* @brief Main entry point to nanobench's benchmarking facility.
|
|
*
|
|
* It holds configuration and results from one or more benchmark runs. Usually it is used in a single line, where the object is
|
|
* constructed, configured, and then a benchmark is run. E.g. like this:
|
|
*
|
|
* ankerl::nanobench::Bench().unit("byte").batch(1000).run("random fluctuations", [&] {
|
|
* // here be the benchmark code
|
|
* });
|
|
*
|
|
* In that example Bench() constructs the benchmark, it is then configured with unit() and batch(), and after configuration a
|
|
* benchmark is executed with run(). Once run() has finished, it prints the result to `std::cout`. It would also store the results
|
|
* in the Bench instance, but in this case the object is immediately destroyed so it's not available any more.
|
|
*/
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_PUSH)
|
|
class Bench {
|
|
public:
|
|
/**
|
|
* @brief Creates a new benchmark for configuration and running of benchmarks.
|
|
*/
|
|
Bench();
|
|
|
|
Bench(Bench&& other) noexcept;
|
|
Bench& operator=(Bench&& other) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE));
|
|
Bench(Bench const& other);
|
|
Bench& operator=(Bench const& other);
|
|
~Bench() noexcept;
|
|
|
|
/*!
|
|
@brief Repeatedly calls `op()` based on the configuration, and performs measurements.
|
|
|
|
This call is marked with `noinline` to prevent the compiler to optimize beyond different benchmarks. This can have quite a big
|
|
effect on benchmark accuracy.
|
|
|
|
@verbatim embed:rst
|
|
.. note::
|
|
|
|
Each call to your lambda must have a side effect that the compiler can't possibly optimize it away. E.g. add a result to an
|
|
externally defined number (like `x` in the above example), and finally call `doNotOptimizeAway` on the variables the compiler
|
|
must not remove. You can also use :cpp:func:`ankerl::nanobench::doNotOptimizeAway` directly in the lambda, but be aware that
|
|
this has a small overhead.
|
|
|
|
@endverbatim
|
|
|
|
@tparam Op The code to benchmark.
|
|
*/
|
|
template <typename Op>
|
|
ANKERL_NANOBENCH(NOINLINE)
|
|
Bench& run(char const* benchmarkName, Op&& op);
|
|
|
|
template <typename Op>
|
|
ANKERL_NANOBENCH(NOINLINE)
|
|
Bench& run(std::string const& benchmarkName, Op&& op);
|
|
|
|
#if ANKERL_NANOBENCH(HAS_STRING_VIEW)
|
|
/// Same as run(char const* benchmarkName, Op op), for a std::string_view name. Only available
|
|
/// when compiling as C++17 or newer.
|
|
template <typename Op>
|
|
ANKERL_NANOBENCH(NOINLINE)
|
|
Bench& run(std::string_view benchmarkName, Op&& op);
|
|
#endif
|
|
|
|
/**
|
|
* @brief Same as run(char const* benchmarkName, Op op), but instead uses the previously set name.
|
|
* @tparam Op The code to benchmark.
|
|
*/
|
|
template <typename Op>
|
|
ANKERL_NANOBENCH(NOINLINE)
|
|
Bench& run(Op&& op);
|
|
|
|
/**
|
|
* @brief Compares alternatives against each other, paired and interleaved.
|
|
*
|
|
* Takes `name, op` pairs - two or more of them. The first is the baseline everything else is
|
|
* measured against, exactly as with relative(), but measured against it *at the same time*
|
|
* rather than one after the other. Drift the machine introduces - a frequency ramp, a noisy
|
|
* neighbour, thermal throttling - then hits every alternative alike and cancels out of the
|
|
* ratios.
|
|
*
|
|
* @code
|
|
* ankerl::nanobench::Bench().epochs(52).compare(
|
|
* "std::mt19937", [&] { ... }, // the baseline
|
|
* "sfc4", [&] { ... },
|
|
* "romu", [&] { ... });
|
|
* @endcode
|
|
*
|
|
* Prints an ordinary nanobench table with two extra columns - the ratio to the baseline and a
|
|
* confidence interval for it - and a summary line underneath.
|
|
*
|
|
@verbatim embed:rst
|
|
A worked example with output, and the reasoning behind each choice below, is in the tutorial at
|
|
:ref:`ab-comparison`.
|
|
|
|
What it does, in order:
|
|
|
|
#. Calibrates an iteration count once, up front, and uses the **same** count for every
|
|
alternative for the whole run. An epoch's fixed overhead is divided by that count, so two
|
|
different counts would amortize it differently and bias the ratio - 1.2% on 200us epochs, and
|
|
no amount of pairing removes it. The shared count is the slowest alternative's, raised where
|
|
that would leave the fastest one's epoch too short for the clock to resolve, and capped so
|
|
the slowest one still fits in maxEpochTime().
|
|
#. Runs one epoch of each per round, interleaved, so drift is common to the round.
|
|
#. Orders each block of N rounds as a randomly chosen cyclic Latin square, so every alternative
|
|
occupies every position exactly once per block and their mean positions are equal - which is
|
|
what cancels a drift that is linear over the block. For two alternatives this is exactly
|
|
ABBA/BAAB.
|
|
#. Reduces each round to ``ln(t_baseline) - ln(t_alternative)``, dropping rounds where either
|
|
side measured zero.
|
|
#. Reports the median of those as the ratio, with a sign-test interval around it, widened to
|
|
keep the whole table at 95% rather than each row separately.
|
|
|
|
.. note::
|
|
|
|
Use more rounds than the default 11. An epoch is about a millisecond, so ``epochs(52)`` costs
|
|
a tenth of a second per alternative and buys an interval narrow enough to act on. The count is
|
|
rounded up to a whole number of blocks, and raised if it is too small to support an interval
|
|
at all.
|
|
|
|
.. warning::
|
|
|
|
Interleaving means each alternative runs with the others' cache and branch predictor state.
|
|
That is usually the more honest measurement for "which should I ship", but it is a different
|
|
measurement from running one alone - use :cpp:func:`run() <ankerl::nanobench::Bench::run()>`
|
|
for that.
|
|
|
|
@endverbatim
|
|
*
|
|
* @tparam Args Alternating names and operations.
|
|
* @param args `name, op` pairs; at least two.
|
|
* @return The ratios and their intervals; also written to output() unless that is nullptr.
|
|
*/
|
|
template <typename... Args>
|
|
ANKERL_NANOBENCH(NOINLINE)
|
|
CompareResult compare(Args&&... args);
|
|
|
|
/**
|
|
* @brief Title of the benchmark, will be shown in the table header. Changing the title will start a new markdown table.
|
|
*
|
|
* @param benchmarkTitle The title of the benchmark.
|
|
*/
|
|
Bench& title(char const* benchmarkTitle);
|
|
Bench& title(std::string const& benchmarkTitle);
|
|
|
|
/**
|
|
* @brief Gets the title of the benchmark
|
|
*/
|
|
ANKERL_NANOBENCH(NODISCARD) std::string const& title() const noexcept;
|
|
|
|
/// Name of the benchmark, will be shown in the table row.
|
|
Bench& name(char const* benchmarkName);
|
|
Bench& name(std::string const& benchmarkName);
|
|
#if ANKERL_NANOBENCH(HAS_STRING_VIEW)
|
|
/// Only available when compiling as C++17 or newer. @see name()
|
|
Bench& name(std::string_view benchmarkName);
|
|
#endif
|
|
ANKERL_NANOBENCH(NODISCARD) std::string const& name() const noexcept;
|
|
|
|
/**
|
|
* @brief Set context information.
|
|
*
|
|
* The information can be accessed using custom render templates via `{{context(variableName)}}`.
|
|
* Trying to render a variable that hasn't been set before raises an exception.
|
|
* Not included in (default) markdown table.
|
|
*
|
|
* @see clearContext, render
|
|
*
|
|
* @param variableName The name of the context variable.
|
|
* @param variableValue The value of the context variable.
|
|
*/
|
|
Bench& context(char const* variableName, char const* variableValue);
|
|
Bench& context(std::string const& variableName, std::string const& variableValue);
|
|
|
|
/**
|
|
* @brief Reset context information.
|
|
*
|
|
* This may improve efficiency when using many context entries,
|
|
* or improve robustness by removing spurious context entries.
|
|
*
|
|
* @see context
|
|
*/
|
|
Bench& clearContext();
|
|
|
|
/**
|
|
* @brief Sets the batch size.
|
|
*
|
|
* E.g. number of processed byte, or some other metric for the size of the processed data in each iteration. If you benchmark
|
|
* hashing of a 1000 byte long string and want byte/sec as a result, you can specify 1000 as the batch size.
|
|
*
|
|
* @tparam T Any input type is internally cast to `double`.
|
|
* @param b batch size
|
|
*/
|
|
template <typename T>
|
|
Bench& batch(T b) noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) double batch() const noexcept;
|
|
|
|
/**
|
|
* @brief Sets the operation unit.
|
|
*
|
|
* Defaults to "op". Could be e.g. "byte" for string processing. This is used for the table header, e.g. to show `ns/byte`. Use
|
|
* singular (*byte*, not *bytes*). A change clears the currently collected results.
|
|
*
|
|
* @param unit The unit name.
|
|
*/
|
|
Bench& unit(char const* unit);
|
|
Bench& unit(std::string const& unit);
|
|
ANKERL_NANOBENCH(NODISCARD) std::string const& unit() const noexcept;
|
|
|
|
/**
|
|
* @brief Sets the time unit to be used for the default output.
|
|
*
|
|
* Nanobench defaults to using ns (nanoseconds) as output in the markdown. For some benchmarks this is too coarse, so it is
|
|
* possible to configure this. E.g. use `timeUnit(1ms, "ms")` to show `ms/op` instead of `ns/op`.
|
|
*
|
|
* @param tu Time unit to display the results in, default is 1ns.
|
|
* @param tuName Name for the time unit, default is "ns"
|
|
*/
|
|
Bench& timeUnit(std::chrono::duration<double> const& tu, std::string const& tuName);
|
|
ANKERL_NANOBENCH(NODISCARD) std::string const& timeUnitName() const noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) std::chrono::duration<double> const& timeUnit() const noexcept;
|
|
|
|
/**
|
|
* @brief Set the output stream where the resulting markdown table will be printed to.
|
|
*
|
|
* The default is `&std::cout`. You can disable all output by setting `nullptr`.
|
|
*
|
|
* @param outstream Pointer to output stream, can be `nullptr`.
|
|
*/
|
|
Bench& output(std::ostream* outstream) noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) std::ostream* output() const noexcept;
|
|
|
|
/**
|
|
* Modern processors have a very accurate clock, being able to measure as low as 20 nanoseconds. This is the main trick nanobech to
|
|
* be so fast: we find out how accurate the clock is, then run the benchmark only so often that the clock's accuracy is good enough
|
|
* for accurate measurements.
|
|
*
|
|
* The default is to run one epoch for 1000 times the clock resolution. So for 20ns resolution and 11 epochs, this gives a total
|
|
* runtime of
|
|
*
|
|
* @f[
|
|
* 20ns * 1000 * 11 \approx 0.2ms
|
|
* @f]
|
|
*
|
|
* To be precise, nanobench adds a 0-20% random noise to each evaluation. This is to prevent any aliasing effects, and further
|
|
* improves accuracy.
|
|
*
|
|
* Total runtime will be higher though: Some initial time is needed to find out the target number of iterations for each epoch, and
|
|
* there is some overhead involved to start & stop timers and calculate resulting statistics and writing the output.
|
|
*
|
|
* @param multiple Target number of times of clock resolution. Usually 1000 is a good compromise between runtime and accuracy.
|
|
*/
|
|
Bench& clockResolutionMultiple(size_t multiple) noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) size_t clockResolutionMultiple() const noexcept;
|
|
|
|
/**
|
|
* @brief Controls number of epochs, the number of measurements to perform.
|
|
*
|
|
* The reported result will be the median of evaluation of each epoch. The higher you choose this, the more
|
|
* deterministic the result be and outliers will be more easily removed. Also the `err%` will be more accurate the higher this
|
|
* number is. Note that the `err%` will not necessarily decrease when number of epochs is increased. But it will be a more accurate
|
|
* representation of the benchmarked code's runtime stability.
|
|
*
|
|
* Choose the value wisely. In practice, 11 has been shown to be a reasonable choice between runtime performance and accuracy.
|
|
* This setting goes hand in hand with minEpochIterations() (or minEpochTime()). If you are more interested in *median* runtime,
|
|
* you might want to increase epochs(). If you are more interested in *mean* runtime, you might want to increase
|
|
* minEpochIterations() instead.
|
|
*
|
|
* @param numEpochs Number of epochs.
|
|
*/
|
|
Bench& epochs(size_t numEpochs) noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) size_t epochs() const noexcept;
|
|
|
|
/**
|
|
* @brief Asks Bench::compare() to keep measuring until the ratio is known this precisely.
|
|
*
|
|
@verbatim embed:rst
|
|
The width is in log space, which for the small widths worth asking for is the width of the
|
|
interval as a fraction of the ratio: ``0.05`` means the ratio is pinned to about ±2.5%. Zero,
|
|
the default, disables it and :cpp:func:`epochs() <ankerl::nanobench::Bench::epochs()>` alone
|
|
decides the count, as before.
|
|
|
|
How many rounds a target takes cannot be known in advance -- it depends on how noisy the
|
|
machine is -- and the default 11 is often far too few: it leaves the interval on a ratio
|
|
about 25% wide, which cannot tell a 10% difference from nothing. Guessing the number by hand
|
|
means guessing again on the next machine.
|
|
|
|
So compare() runs **two stages**. The first is ``epochs()`` rounds and is used only to see how
|
|
noisy the measurement is; the second runs as many rounds as that implies, and **the reported
|
|
result comes from the second stage alone**. Splitting them is what keeps the statistics
|
|
honest: a rule that watched one growing sample and stopped as soon as it looked tight enough
|
|
would stop preferentially on the samples that happen to look tight, and the interval would
|
|
then cover less often than it claims. Here the stopping decision is made from data that is
|
|
then discarded, so the second stage's interval is exactly as good as its confidence says. The
|
|
price of that is the pilot, a fixed ``epochs()`` rounds.
|
|
|
|
Simulated over 4000 trials against skewed noise, asking for a width of 0.2 at a nominal 95%:
|
|
two stages cover **96.0%** of the time, and the same rule watching one growing sample covers
|
|
**92.9%** while using half the rounds. The first is the sign test's usual slight
|
|
over-coverage from discreteness; the second is the interval quietly meaning less than it
|
|
says.
|
|
|
|
A machine too noisy to reach the target within
|
|
:cpp:func:`maxEpochs() <ankerl::nanobench::Bench::maxEpochs()>` stops there and reports the
|
|
width it did reach, rather than running forever or pretending.
|
|
@endverbatim
|
|
*
|
|
* @param width Interval width to aim for, in log space. 0 disables.
|
|
*/
|
|
Bench& targetIntervalWidth(double width) noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) double targetIntervalWidth() const noexcept;
|
|
|
|
/**
|
|
* @brief Upper limit on the rounds Bench::compare() runs when targetIntervalWidth() is set.
|
|
* Default 1000.
|
|
*/
|
|
Bench& maxEpochs(size_t numEpochs) noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) size_t maxEpochs() const noexcept;
|
|
|
|
/**
|
|
* @brief Upper limit for the runtime of each epoch.
|
|
*
|
|
* As a safety precaution if the clock is not very accurate, we can set an upper limit for the maximum evaluation time per
|
|
* epoch. Default is 100ms. At least a single evaluation of the benchmark is performed.
|
|
*
|
|
* @see minEpochTime, minEpochIterations
|
|
*
|
|
* @param t Maximum target runtime for a single epoch.
|
|
*/
|
|
Bench& maxEpochTime(std::chrono::nanoseconds t) noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) std::chrono::nanoseconds maxEpochTime() const noexcept;
|
|
|
|
/**
|
|
* @brief Minimum time each epoch should take.
|
|
*
|
|
* Default is 1ms, so we are mostly relying on clockResolutionMultiple(). In most cases this is exactly what you want. If you see
|
|
* that the evaluation is unreliable with a high `err%`, you can increase either minEpochTime() or minEpochIterations().
|
|
*
|
|
* @see maxEpochTime, minEpochIterations
|
|
*
|
|
* @param t Minimum time each epoch should take.
|
|
*/
|
|
Bench& minEpochTime(std::chrono::nanoseconds t) noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) std::chrono::nanoseconds minEpochTime() const noexcept;
|
|
|
|
/**
|
|
* @brief Sets the minimum number of iterations each epoch should take.
|
|
*
|
|
* Default is 1, and we rely on clockResolutionMultiple(). If the `err%` is high and you want a more smooth result, you might want
|
|
* to increase the minimum number of iterations, or increase the minEpochTime().
|
|
*
|
|
* @see minEpochTime, maxEpochTime, minEpochIterations
|
|
*
|
|
* @param numIters Minimum number of iterations per epoch.
|
|
*/
|
|
Bench& minEpochIterations(uint64_t numIters) noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) uint64_t minEpochIterations() const noexcept;
|
|
|
|
/**
|
|
* Sets exactly the number of iterations for each epoch. Ignores all other epoch limits. This forces nanobench to use exactly
|
|
* the given number of iterations for each epoch, not more and not less. Default is 0 (disabled).
|
|
*
|
|
* @param numIters Exact number of iterations to use. Set to 0 to disable.
|
|
*/
|
|
Bench& epochIterations(uint64_t numIters) noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) uint64_t epochIterations() const noexcept;
|
|
|
|
/**
|
|
* @brief Sets a number of iterations that are initially performed without any measurements.
|
|
*
|
|
* Some benchmarks need a few evaluations to warm up caches / database / whatever access. Normally this should not be needed, since
|
|
* we show the median result so initial outliers will be filtered away automatically. If the warmup effect is large though, you
|
|
* might want to set it. Default is 0.
|
|
*
|
|
* @param numWarmupIters Number of warmup iterations.
|
|
*/
|
|
Bench& warmup(uint64_t numWarmupIters) noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) uint64_t warmup() const noexcept;
|
|
|
|
/**
|
|
* @brief Marks the next run as the baseline.
|
|
*
|
|
* Call `relative(true)` to mark the run as the baseline. Successive runs will be compared to this run. Just like all the
|
|
* other columns, the comparison is per unit, so it stays meaningful when the runs use a different Bench::batch:
|
|
*
|
|
* @f[
|
|
* 100\% * \frac{baseline / baselineBatch}{runtime / batch}
|
|
* @f]
|
|
*
|
|
* * 100% means it is exactly as fast as the baseline
|
|
* * >100% means it is faster than the baseline. E.g. 200% means the current run is twice as fast as the baseline.
|
|
* * <100% means it is slower than the baseline. E.g. 50% means it is twice as slow as the baseline.
|
|
*
|
|
* See the tutorial section "Comparing Results" for example usage.
|
|
*
|
|
* @param isRelativeEnabled True to enable processing
|
|
*/
|
|
Bench& relative(bool isRelativeEnabled) noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) bool relative() const noexcept;
|
|
|
|
/**
|
|
* @brief Enables/disables performance counters.
|
|
*
|
|
* On Linux nanobench has a powerful feature to use performance counters. This enables counting of retired instructions, count
|
|
* number of branches, missed branches, etc. On default this is enabled, but you can disable it if you don't need that feature.
|
|
*
|
|
* @param showPerformanceCounters True to enable, false to disable.
|
|
*/
|
|
Bench& performanceCounters(bool showPerformanceCounters) noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) bool performanceCounters() const noexcept;
|
|
|
|
/**
|
|
* @brief Removes a column from the table.
|
|
*
|
|
* The full table is over 150 characters wide, which wraps in most terminals. Hide what you are not
|
|
* reading:
|
|
*
|
|
* bench.hideColumn(Column::instructions)
|
|
* .hideColumn(Column::cycles)
|
|
* .hideColumn(Column::ipc);
|
|
*
|
|
* Hiding a column only changes what is printed - the measurement still happens, and results() and
|
|
* the render templates are unaffected. To stop *measuring* the performance counters as well, use
|
|
* performanceCounters(false), which hides all five of their columns at once.
|
|
*
|
|
* @param column The column to hide.
|
|
*/
|
|
Bench& hideColumn(Column column) noexcept;
|
|
|
|
/// Shows a column hidden with hideColumn() again. @see hideColumn()
|
|
Bench& showColumn(Column column) noexcept;
|
|
|
|
/// True when @p column is not hidden. A column can still be absent from the table for other
|
|
/// reasons - `relative` without relative(), or the counter columns off Linux. @see hideColumn()
|
|
ANKERL_NANOBENCH(NODISCARD) bool isColumnVisible(Column column) const noexcept;
|
|
|
|
/**
|
|
* @brief Adds a column showing the value of a context variable.
|
|
*
|
|
* Context variables set with context() are otherwise only reachable from a render template, which
|
|
* makes parameterised benchmarks awkward to read on the console. This puts one in the table:
|
|
*
|
|
* bench.context("threads", std::to_string(n))
|
|
* .contextColumn("threads")
|
|
* .run(...);
|
|
*
|
|
* The column is added once per name, in the order given, before the benchmark name. A row whose
|
|
* context does not have the variable is left blank.
|
|
*
|
|
* @param variableName Name of the context variable to show. @see context()
|
|
*/
|
|
Bench& contextColumn(std::string const& variableName);
|
|
|
|
/// Removes all columns added with contextColumn(). @see contextColumn()
|
|
Bench& clearContextColumns() noexcept;
|
|
|
|
/**
|
|
* @brief Retrieves all benchmark results collected by the bench object so far.
|
|
*
|
|
* Each call to run() generates a Result that is stored within the Bench instance. This is mostly for advanced users who want to
|
|
* see all the nitty gritty details.
|
|
*
|
|
* @return All results collected so far.
|
|
*/
|
|
ANKERL_NANOBENCH(NODISCARD) std::vector<Result> const& results() const noexcept;
|
|
|
|
/*!
|
|
@verbatim embed:rst
|
|
|
|
Convenience shortcut to :cpp:func:`ankerl::nanobench::doNotOptimizeAway`.
|
|
|
|
@endverbatim
|
|
*/
|
|
template <typename Arg>
|
|
Bench& doNotOptimizeAway(Arg&& arg);
|
|
|
|
/*!
|
|
@verbatim embed:rst
|
|
|
|
Sets N for asymptotic complexity calculation, so it becomes possible to calculate `Big O
|
|
<https://en.wikipedia.org/wiki/Big_O_notation>`_ from multiple benchmark evaluations.
|
|
|
|
Use :cpp:func:`ankerl::nanobench::Bench::complexityBigO` when the evaluation has finished. See the tutorial
|
|
:ref:`asymptotic-complexity` for details.
|
|
|
|
@endverbatim
|
|
|
|
@tparam T Any type is cast to `double`.
|
|
@param n Length of N for the next benchmark run, so it is possible to calculate `bigO`.
|
|
*/
|
|
template <typename T>
|
|
Bench& complexityN(T n) noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) double complexityN() const noexcept;
|
|
|
|
/*!
|
|
Calculates [Big O](https://en.wikipedia.org/wiki/Big_O_notation>) of the results with all preconfigured complexity functions.
|
|
Currently these complexity functions are fitted into the benchmark results:
|
|
|
|
@f$ \mathcal{O}(1) @f$,
|
|
@f$ \mathcal{O}(n) @f$,
|
|
@f$ \mathcal{O}(\log{}n) @f$,
|
|
@f$ \mathcal{O}(n\log{}n) @f$,
|
|
@f$ \mathcal{O}(n^2) @f$,
|
|
@f$ \mathcal{O}(n^3) @f$.
|
|
|
|
If we e.g. evaluate the complexity of `std::sort`, this is the result of `std::cout << bench.complexityBigO()`:
|
|
|
|
```
|
|
| coefficient | err% | complexity
|
|
|--------------:|-------:|------------
|
|
| 5.08935e-09 | 2.6% | O(n log n)
|
|
| 6.10608e-08 | 8.0% | O(n)
|
|
| 1.29307e-11 | 47.2% | O(n^2)
|
|
| 2.48677e-15 | 69.6% | O(n^3)
|
|
| 9.88133e-06 | 132.3% | O(log n)
|
|
| 5.98793e-05 | 162.5% | O(1)
|
|
```
|
|
|
|
So in this case @f$ \mathcal{O}(n\log{}n) @f$ provides the best approximation.
|
|
|
|
@verbatim embed:rst
|
|
See the tutorial :ref:`asymptotic-complexity` for details.
|
|
@endverbatim
|
|
@return Evaluation results, which can be printed or otherwise inspected.
|
|
*/
|
|
std::vector<BigO> complexityBigO() const;
|
|
|
|
/**
|
|
* @brief Calculates bigO for a custom function.
|
|
*
|
|
* E.g. to calculate the mean squared error for @f$ \mathcal{O}(\log{}\log{}n) @f$, which is not part of the default set of
|
|
* complexityBigO(), you can do this:
|
|
*
|
|
* ```
|
|
* auto logLogN = bench.complexityBigO("O(log log n)", [](double n) {
|
|
* return std::log2(std::log2(n));
|
|
* });
|
|
* ```
|
|
*
|
|
* The resulting mean squared error can be printed with `std::cout << logLogN`. E.g. it prints something like this:
|
|
*
|
|
* ```text
|
|
* 2.46985e-05 * O(log log n), rms=1.48121
|
|
* ```
|
|
*
|
|
* @tparam Op Type of mapping operation.
|
|
* @param name Name for the function, e.g. "O(log log n)"
|
|
* @param op Op's operator() maps a `double` with the desired complexity function, e.g. `log2(log2(n))`.
|
|
* @return BigO Error calculation, which is streamable to std::cout.
|
|
*/
|
|
template <typename Op>
|
|
BigO complexityBigO(char const* name, Op op) const;
|
|
|
|
template <typename Op>
|
|
BigO complexityBigO(std::string const& name, Op op) const;
|
|
|
|
/*!
|
|
@verbatim embed:rst
|
|
|
|
Convenience shortcut to :cpp:func:`ankerl::nanobench::render`.
|
|
|
|
@endverbatim
|
|
*/
|
|
Bench& render(char const* templateContent, std::ostream& os);
|
|
Bench& render(std::string const& templateContent, std::ostream& os);
|
|
|
|
Bench& config(Config const& benchmarkConfig);
|
|
ANKERL_NANOBENCH(NODISCARD) Config const& config() const noexcept;
|
|
|
|
/*!
|
|
@brief Runs `setupOp()` once before each epoch, without measuring it.
|
|
|
|
Use this to restore whatever state your benchmark consumes, when that restoration would otherwise
|
|
pollute the measurement:
|
|
|
|
@code
|
|
bench.setup([&] { data = pristine; })
|
|
.run("consume data", [&] { consume(data); });
|
|
@endcode
|
|
|
|
@verbatim embed:rst
|
|
.. important::
|
|
|
|
The setup runs **once per epoch, not once per iteration**. An epoch calls ``op()`` many times
|
|
(see :cpp:func:`epochIterations() <ankerl::nanobench::Bench::epochIterations()>`), and the setup
|
|
does not run again in between. So this helps when your operation can be repeated as-is and only
|
|
the *starting* state has to be established - and it does **not** help when every single call
|
|
mutates the data such that the next call would measure something different.
|
|
|
|
For that second case, set :cpp:func:`epochIterations(1) <ankerl::nanobench::Bench::epochIterations()>`
|
|
so an epoch is a single call, which makes setup effectively per-iteration - at the cost of much
|
|
noisier results, since one call is then timed against the clock's resolution. The alternative,
|
|
and usually the better measurement, is to time the setup separately and subtract it.
|
|
|
|
Timers are deliberately not started and stopped around each iteration: for anything fast that
|
|
costs more than the thing being measured, and the performance counters would have to be
|
|
restarted too.
|
|
|
|
.. note::
|
|
|
|
The returned object keeps a reference to this ``Bench``, so don't let it outlive it - call
|
|
``run()`` on the same expression, as above.
|
|
|
|
@endverbatim
|
|
|
|
@tparam SetupOp The untimed code to run before each epoch.
|
|
@param setupOp The setup to run.
|
|
*/
|
|
template <typename SetupOp>
|
|
detail::SetupRunner<SetupOp> setup(SetupOp setupOp);
|
|
|
|
private:
|
|
// Collects the `name, op` pack into two flat vectors, once. Everything after this point is
|
|
// ordinary index-based code in the implementation block rather than more template recursion.
|
|
static void compareCollect(std::vector<std::string>& /*names*/, std::vector<detail::ErasedOp>& /*ops*/) {}
|
|
|
|
template <typename Name, typename Op, typename... Rest>
|
|
static void compareCollect(std::vector<std::string>& names, std::vector<detail::ErasedOp>& ops, Name&& name, Op&& op,
|
|
Rest&&... rest) {
|
|
names.emplace_back(std::forward<Name>(name));
|
|
ops.push_back(detail::eraseOp(op));
|
|
compareCollect(names, ops, std::forward<Rest>(rest)...);
|
|
}
|
|
|
|
// The measuring half, which no longer needs to know the operations' types.
|
|
ANKERL_NANOBENCH(NODISCARD)
|
|
CompareResult compareImpl(std::vector<std::string> const& names, std::vector<detail::ErasedOp> const& ops);
|
|
|
|
// Number of iterations of `op` that makes one epoch last at least `target`.
|
|
ANKERL_NANOBENCH(NODISCARD)
|
|
uint64_t compareCalibrate(detail::ErasedOp const& op, Clock::duration target) const;
|
|
|
|
// The one iteration count the whole experiment runs at. Fixed for every alternative and every
|
|
// round: a count that drifted between them would be a second thing changing while the comparison
|
|
// is being made, and an epoch's fixed overhead is divided by it, so two different counts amortize
|
|
// that overhead differently and bias the ratio.
|
|
ANKERL_NANOBENCH(NODISCARD)
|
|
uint64_t compareIterations(std::vector<detail::ErasedOp> const& ops) const;
|
|
|
|
// Runs `numIters` iterations of `op` as one epoch, appending the measurement to `result`.
|
|
static void compareEpoch(detail::ErasedOp const& op, uint64_t numIters, Result& result);
|
|
|
|
template <typename SetupOp, typename Op>
|
|
Bench& runImpl(SetupOp& setupOp, Op&& op);
|
|
|
|
template <typename SetupOp>
|
|
friend class detail::SetupRunner;
|
|
|
|
Config mConfig{};
|
|
std::vector<Result> mResults{};
|
|
};
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_POP)
|
|
|
|
/**
|
|
* @brief Makes sure none of the given arguments are optimized away by the compiler.
|
|
*
|
|
* @tparam Arg Type of the argument that shouldn't be optimized away.
|
|
* @param arg The input that we mark as being used, even though we don't do anything with it.
|
|
*/
|
|
template <typename Arg>
|
|
void doNotOptimizeAway(Arg&& arg);
|
|
|
|
namespace detail {
|
|
|
|
#if !ANKERL_NANOBENCH(ASM_DONT_OPTIMIZE_AWAY)
|
|
void doNotOptimizeAwaySink(void const*);
|
|
|
|
template <typename T>
|
|
void doNotOptimizeAway(T const& val);
|
|
|
|
#else
|
|
|
|
// These assembly magic is directly from what Google Benchmark is doing. I have previously used what facebook's folly was doing, but
|
|
// this seemed to have compilation problems in some cases. Google Benchmark seemed to be the most well tested anyways.
|
|
// see https://github.com/google/benchmark/blob/v1.7.1/include/benchmark/benchmark.h#L443-L446
|
|
template <typename T>
|
|
void doNotOptimizeAway(T const& val) {
|
|
// NOLINTNEXTLINE(hicpp-no-assembler)
|
|
asm volatile("" : : "r,m"(val) : "memory");
|
|
}
|
|
|
|
template <typename T>
|
|
void doNotOptimizeAway(T& val) {
|
|
# if defined(__clang__)
|
|
// NOLINTNEXTLINE(hicpp-no-assembler)
|
|
asm volatile("" : "+r,m"(val) : : "memory");
|
|
# else
|
|
// NOLINTNEXTLINE(hicpp-no-assembler)
|
|
asm volatile("" : "+m,r"(val) : : "memory");
|
|
# endif
|
|
}
|
|
#endif
|
|
|
|
// internally used, but visible because run() is templated.
|
|
// Not movable/copy-able, so we simply use a pointer instead of unique_ptr. This saves us from
|
|
// having to include <memory>, and the template instantiation overhead of unique_ptr which is unfortunately quite significant.
|
|
ANKERL_NANOBENCH(IGNORE_EFFCPP_PUSH)
|
|
class IterationLogic {
|
|
public:
|
|
explicit IterationLogic(Bench const& bench);
|
|
IterationLogic(IterationLogic&&) = delete;
|
|
IterationLogic& operator=(IterationLogic&&) = delete;
|
|
IterationLogic(IterationLogic const&) = delete;
|
|
IterationLogic& operator=(IterationLogic const&) = delete;
|
|
~IterationLogic();
|
|
|
|
ANKERL_NANOBENCH(NODISCARD) uint64_t numIters() const noexcept;
|
|
void add(std::chrono::nanoseconds elapsed, PerformanceCounters const& pc) noexcept;
|
|
void moveResultTo(std::vector<Result>& results) noexcept;
|
|
|
|
private:
|
|
struct Impl;
|
|
Impl* mPimpl;
|
|
};
|
|
ANKERL_NANOBENCH(IGNORE_EFFCPP_POP)
|
|
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_PUSH)
|
|
class PerformanceCounters {
|
|
public:
|
|
PerformanceCounters(PerformanceCounters const&) = delete;
|
|
PerformanceCounters(PerformanceCounters&&) = delete;
|
|
PerformanceCounters& operator=(PerformanceCounters const&) = delete;
|
|
PerformanceCounters& operator=(PerformanceCounters&&) = delete;
|
|
|
|
PerformanceCounters();
|
|
~PerformanceCounters();
|
|
|
|
void beginMeasure();
|
|
void endMeasure();
|
|
void updateResults(uint64_t numIters);
|
|
|
|
ANKERL_NANOBENCH(NODISCARD) PerfCountSet<uint64_t> const& val() const noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) PerfCountSet<bool> const& has() const noexcept;
|
|
|
|
private:
|
|
#if ANKERL_NANOBENCH(PERF_COUNTERS)
|
|
LinuxPerformanceCounters* mPc = nullptr;
|
|
#endif
|
|
PerfCountSet<uint64_t> mVal{};
|
|
PerfCountSet<bool> mHas{};
|
|
};
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_POP)
|
|
|
|
// Gets the singleton
|
|
PerformanceCounters& performanceCounters();
|
|
|
|
// How long one epoch should take: the clock's resolution multiplied up until the measurement sits
|
|
// well above the clock's own noise, then clamped into [minEpochTime, maxEpochTime]. The minimum is
|
|
// applied last, so it wins when the two bounds conflict.
|
|
//
|
|
// Both run() and compare() size their epochs with this. It is a function rather than a line in each
|
|
// of them because it is a rule, and a rule with two copies is a rule with two behaviours.
|
|
Clock::duration targetRuntimePerEpoch(Bench const& bench);
|
|
|
|
// Whether hideColumn() left this column visible. Takes the Config rather than the Bench because the
|
|
// table writers only have the Config - every Result carries a copy of the one it was measured under.
|
|
ANKERL_NANOBENCH(NODISCARD) bool isColumnVisible(Config const& config, Column column) noexcept;
|
|
|
|
// The frequency-scaling and pyperf warnings, and the note that the kernel refused the performance
|
|
// counters. Declared up here because compare() is a template defined above the implementation block
|
|
// and has to be able to call them - a program whose only nanobench call is compare() needs the
|
|
// warning at least as much as one that calls run().
|
|
void printStabilityInformationOnce(std::ostream* outStream);
|
|
void printPerformanceCounterHintOnce(std::ostream* outStream, bool wantsPerformanceCounters);
|
|
|
|
// The last table settings written to this stream. When they change, a new header is written.
|
|
uint64_t& streamHeaderHash(std::ostream& os);
|
|
|
|
// The arithmetic that turns what the kernel hands back into the numbers of the ins/op, cyc/op, IPC,
|
|
// bra/op and miss% columns. It lives out here, rather than inside the Linux-only counter class,
|
|
// because it is the part that can be silently wrong: a counter that is quietly 5% low looks exactly
|
|
// like a correct one. Out here it is compiled and testable on every platform, while
|
|
// perf_event_open() - which cannot be exercised in a container at all - stays where it is.
|
|
//
|
|
// All of these saturate at 0 rather than wrapping: a correction larger than the measurement means
|
|
// the measurement was noise, not that the value is enormous.
|
|
|
|
// a - b, or 0 when b is the larger. The one place that saturation is written down: hand-rolled, it
|
|
// was six ternaries that had already drifted between '>' and '>=' without anyone noticing, because at
|
|
// a == b the two spellings agree.
|
|
uint64_t saturatingSub(uint64_t a, uint64_t b) noexcept;
|
|
|
|
// Rounded integer division, for spreading a measured overhead over the iterations it covers.
|
|
uint64_t divRounded(uint64_t a, uint64_t divisor) noexcept;
|
|
|
|
// How many uint64_t a PERF_FORMAT_GROUP | PERF_FORMAT_ID read needs for `numEvents` events: three
|
|
// header words (nr, time_enabled, time_running), then a value and an id per event. That layout used
|
|
// to be written down separately at the buffer sizing, at the buffer initialization and here - three
|
|
// places to keep in step on a code path most containers and VMs cannot run at all.
|
|
size_t perfReadFormatSize(size_t numEvents) noexcept;
|
|
|
|
// Index of the i-th event's value in such a read, which is exactly where the i events before it end.
|
|
size_t perfValueIndex(uint64_t i) noexcept;
|
|
|
|
// Extrapolates a counter that the kernel could only keep active for part of the measurement, the way
|
|
// `perf stat` does. Returns the value unchanged when it was active throughout.
|
|
uint64_t scaleMultiplexed(uint64_t value, uint64_t timeEnabled, uint64_t timeRunning) noexcept;
|
|
|
|
// Subtracts the calibrated overhead of measuring, plus the per-iteration overhead of the loop doing
|
|
// the measuring, from a raw counter value. Pass 0 for a correction that does not apply.
|
|
uint64_t correctOverhead(uint64_t value, uint64_t measuringOverhead, uint64_t loopOverheadPerIteration, uint64_t numIters) noexcept;
|
|
|
|
// The loop's own per-iteration cost, from a calibration run of `numIters` operations and one of
|
|
// twice as many: whatever the second run did not cost twice over is the loop rather than the work.
|
|
uint64_t loopOverheadPerIteration(uint64_t single, uint64_t doubled, uint64_t measuringOverhead, uint64_t numIters) noexcept;
|
|
|
|
// Removes the branch that the measuring loop itself takes once per iteration, plus the one that ends
|
|
// it, from a raw branch count.
|
|
uint64_t correctBranchInstructions(uint64_t rawBranchInstructions, uint64_t numIters) noexcept;
|
|
|
|
// Paired statistics, for Bench::compare(). Pure functions over the per-round measurements, so they are
|
|
// testable without running a benchmark at all - which matters, because the alternative is judging a
|
|
// confidence interval by looking at it.
|
|
|
|
// ln(a[i]) - ln(b[i]) for each round. Working in log space turns a speedup, which is multiplicative,
|
|
// into a difference, which is what every statistic below assumes. Rounds where either side measured
|
|
// 0 - a clock too coarse for the operation - carry no ratio and are dropped.
|
|
std::vector<double> pairedLogRatios(std::vector<double> const& a, std::vector<double> const& b);
|
|
|
|
// The same, over two Results' per-round elapsed times. compare() runs one epoch of every alternative
|
|
// per round, so their measurements line up round for round and can be paired directly - not only
|
|
// against the baseline, but between any two of them.
|
|
std::vector<double> pairedLogRatios(Result const& a, Result const& b);
|
|
|
|
// Median of a copy of the values. 0.0 when there are none.
|
|
double medianOf(std::vector<double> values);
|
|
|
|
// The pair of 0-based order statistics that bracket the median with at least `confidence`
|
|
// probability: the largest k with P(Bin(n, 1/2) < k) <= (1 - confidence) / 2 gives the interval
|
|
// [x_(k), x_(n+1-k)]. Returns {1, 0} - an empty range - when no such interval exists, which is the
|
|
// case for fewer than six observations at 95%.
|
|
std::pair<size_t, size_t> medianIntervalIndices(size_t n, double confidence);
|
|
|
|
// Rounds whose two sides measured the same time to the last tick the clock could report, i.e. a log
|
|
// ratio of exactly 0. Not evidence of equality - evidence that the clock ran out of resolution.
|
|
size_t countTiedRounds(std::vector<double> const& logRatios);
|
|
|
|
// Confidence each individual interval is built at, so that all `numComparisons` of them together
|
|
// hold at 95%. Bonferroni: simple, assumption-free, and conservative here because the comparisons
|
|
// share a baseline and are therefore correlated. One function, because the table and the summary
|
|
// line underneath it must not be able to report intervals at two different confidences.
|
|
double bonferroniConfidence(size_t numComparisons) noexcept;
|
|
|
|
// Distribution-free confidence interval for the median, from those order statistics.
|
|
//
|
|
// The sign test, and it is chosen for what it does *not* assume. It needs the observations to be
|
|
// independent and nothing else - no shape, no symmetry, no finite variance, no asymptotics - and it
|
|
// is exact at every n rather than approximately right for large ones. The alternatives all want
|
|
// more: a t-interval wants normality, a bootstrap wants its own asymptotics and converges slowly
|
|
// for a median in particular, and Wilcoxon wants the differences to be symmetric about their
|
|
// median. That last one is exactly what paired timings do not give you, because an operation can be
|
|
// arbitrarily slower but never faster than its floor, so the differences are skewed whenever one
|
|
// side has the heavier tail. Measured on such data: sign 96.7% coverage of a nominal 95%, Wilcoxon
|
|
// 91.5%.
|
|
//
|
|
// The price is width - roughly 10% wider than a bootstrap - and slight over-coverage from the
|
|
// discreteness. Both err towards saying "no difference resolved", which is the right direction for
|
|
// a tool whose output ends up in a pull request.
|
|
std::pair<double, double> medianInterval(std::vector<double> values, double confidence);
|
|
|
|
// How many rounds a second stage needs so that its interval is no wider than `targetWidth`.
|
|
//
|
|
// The width of a sign-test interval for the median shrinks as 1/sqrt(n) -- the two order statistics
|
|
// that bracket it sit about z*sqrt(n)/2 ranks either side of the middle, so the distance between
|
|
// them falls off with the square root of the count. So the count that reaches a target width scales
|
|
// with the square of the ratio of widths, which is all this computes, plus the rounding the caller
|
|
// needs: up to a whole block of `numOps`, never below what was already run, never above `maxRounds`.
|
|
//
|
|
// `observedWidth` and `targetWidth` are both in log space, the space the interval is computed in.
|
|
// A degenerate pilot -- every round identical, so a width of zero -- asks for the minimum rather
|
|
// than dividing by zero.
|
|
size_t roundsForTargetWidth(size_t pilotRounds, double observedWidth, double targetWidth, size_t numOps, size_t maxRounds) noexcept;
|
|
|
|
// Branch misses cannot exceed the branches they were taken from, and the loop is assumed to mispredict
|
|
// its own exit once - so at least one miss is always attributed to it.
|
|
double correctBranchMisses(uint64_t rawBranchMisses, double correctedBranchInstructions) noexcept;
|
|
|
|
// Applies a "key=value,key=value" string - the contents of NANOBENCH_CONFIG - to bench, by calling
|
|
// the setter each key is named after. Separate from reading the environment so that it can be tested
|
|
// by handing it a string, rather than through a variable that is spelled putenv on one platform and
|
|
// _putenv_s on another.
|
|
//
|
|
// An entry that cannot be applied appends one message to errors and is skipped; every other entry
|
|
// still applies. Nothing throws, so a typo behaves the same for a consumer built -fno-exceptions.
|
|
void applyConfigString(Bench& bench, std::string const& configStr, std::vector<std::string>& errors);
|
|
|
|
} // namespace detail
|
|
|
|
class BigO {
|
|
public:
|
|
using RangeMeasure = std::vector<std::pair<double, double>>;
|
|
|
|
template <typename Op>
|
|
static RangeMeasure mapRangeMeasure(RangeMeasure data, Op op) {
|
|
for (auto& rangeMeasure : data) {
|
|
rangeMeasure.first = op(rangeMeasure.first);
|
|
}
|
|
return data;
|
|
}
|
|
|
|
static RangeMeasure collectRangeMeasure(std::vector<Result> const& results);
|
|
|
|
template <typename Op>
|
|
BigO(char const* bigOName, RangeMeasure const& rangeMeasure, Op rangeToN)
|
|
: BigO(bigOName, mapRangeMeasure(rangeMeasure, rangeToN)) {}
|
|
|
|
template <typename Op>
|
|
BigO(std::string bigOName, RangeMeasure const& rangeMeasure, Op rangeToN)
|
|
: BigO(std::move(bigOName), mapRangeMeasure(rangeMeasure, rangeToN)) {}
|
|
|
|
BigO(char const* bigOName, RangeMeasure const& scaledRangeMeasure);
|
|
BigO(std::string bigOName, RangeMeasure const& scaledRangeMeasure);
|
|
ANKERL_NANOBENCH(NODISCARD) std::string const& name() const noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) double constant() const noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) double normalizedRootMeanSquare() const noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) bool operator<(BigO const& other) const noexcept;
|
|
|
|
private:
|
|
std::string mName{};
|
|
double mConstant{};
|
|
double mNormalizedRootMeanSquare{};
|
|
};
|
|
std::ostream& operator<<(std::ostream& os, BigO const& bigO);
|
|
std::ostream& operator<<(std::ostream& os, std::vector<ankerl::nanobench::BigO> const& bigOs);
|
|
|
|
} // namespace nanobench
|
|
} // namespace ankerl
|
|
|
|
// implementation /////////////////////////////////////////////////////////////////////////////////
|
|
|
|
namespace ankerl {
|
|
namespace nanobench {
|
|
|
|
constexpr uint64_t(Rng::min)() {
|
|
return 0;
|
|
}
|
|
|
|
constexpr uint64_t(Rng::max)() {
|
|
return (std::numeric_limits<uint64_t>::max)();
|
|
}
|
|
|
|
ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined")
|
|
uint64_t Rng::operator()() noexcept {
|
|
auto x = mX;
|
|
|
|
mX = UINT64_C(15241094284759029579) * mY;
|
|
mY = rotl(mY - x, 27);
|
|
|
|
return x;
|
|
}
|
|
|
|
ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined")
|
|
uint32_t Rng::bounded(uint32_t range) noexcept {
|
|
uint64_t const r32 = static_cast<uint32_t>(operator()());
|
|
auto multiresult = r32 * range;
|
|
return static_cast<uint32_t>(multiresult >> 32U);
|
|
}
|
|
|
|
double Rng::uniform01() noexcept {
|
|
auto i = (UINT64_C(0x3ff) << 52U) | (operator()() >> 12U);
|
|
// can't use union in c++ here for type puning, it's undefined behavior.
|
|
// std::memcpy is optimized anyways.
|
|
double d{};
|
|
std::memcpy(&d, &i, sizeof(double));
|
|
return d - 1.0;
|
|
}
|
|
|
|
template <typename Container>
|
|
void Rng::shuffle(Container& container) noexcept {
|
|
auto i = container.size();
|
|
while (i > 1U) {
|
|
using std::swap;
|
|
auto n = operator()();
|
|
// using decltype(i) instead of size_t to be compatible to containers with 32bit index (see #80)
|
|
auto b1 = static_cast<decltype(i)>((static_cast<uint32_t>(n) * static_cast<uint64_t>(i)) >> 32U);
|
|
swap(container[--i], container[b1]);
|
|
|
|
auto b2 = static_cast<decltype(i)>(((n >> 32U) * static_cast<uint64_t>(i)) >> 32U);
|
|
swap(container[--i], container[b2]);
|
|
}
|
|
}
|
|
|
|
ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined")
|
|
constexpr uint64_t Rng::rotl(uint64_t x, unsigned k) noexcept {
|
|
return (x << k) | (x >> (64U - k));
|
|
}
|
|
|
|
namespace detail {
|
|
|
|
// A setup lambda that captures little - or nothing - is smaller than the alignment of the Bench
|
|
// reference next to it, so this pads. That is unavoidable here and harmless, but a consumer building
|
|
// with -Weverything sees it as an error in nanobench's own header.
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_PUSH)
|
|
template <typename SetupOp>
|
|
class SetupRunner {
|
|
public:
|
|
explicit SetupRunner(SetupOp setupOp, Bench& bench)
|
|
: mSetupOp(std::move(setupOp))
|
|
, mBench(bench) {}
|
|
|
|
template <typename Op>
|
|
ANKERL_NANOBENCH_NO_SANITIZE("integer")
|
|
Bench& run(Op&& op) {
|
|
return mBench.runImpl(mSetupOp, std::forward<Op>(op));
|
|
}
|
|
|
|
// Bench::run() takes a name, so setup().run() has to as well - otherwise adding a setup to an
|
|
// existing benchmark means rewriting its call site to use name() separately.
|
|
template <typename Op>
|
|
Bench& run(char const* benchmarkName, Op&& op) {
|
|
mBench.name(benchmarkName);
|
|
return run(std::forward<Op>(op));
|
|
}
|
|
|
|
template <typename Op>
|
|
Bench& run(std::string const& benchmarkName, Op&& op) {
|
|
mBench.name(benchmarkName);
|
|
return run(std::forward<Op>(op));
|
|
}
|
|
|
|
private:
|
|
SetupOp mSetupOp;
|
|
Bench& mBench;
|
|
};
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_POP)
|
|
} // namespace detail
|
|
|
|
template <typename Op>
|
|
ANKERL_NANOBENCH_NO_SANITIZE("integer")
|
|
Bench& Bench::run(Op&& op) {
|
|
auto setupOp = [] {};
|
|
return runImpl(setupOp, std::forward<Op>(op));
|
|
}
|
|
|
|
template <typename SetupOp, typename Op>
|
|
ANKERL_NANOBENCH_NO_SANITIZE("integer")
|
|
Bench& Bench::runImpl(SetupOp& setupOp, Op&& op) {
|
|
// It is important that this method is kept short so the compiler can do better optimizations/ inlining of op()
|
|
detail::IterationLogic iterationLogic(*this);
|
|
auto& pc = detail::performanceCounters();
|
|
|
|
while (auto n = iterationLogic.numIters()) {
|
|
setupOp();
|
|
|
|
pc.beginMeasure();
|
|
Clock::time_point const before = Clock::now();
|
|
while (n-- > 0) {
|
|
op();
|
|
}
|
|
Clock::time_point const after = Clock::now();
|
|
pc.endMeasure();
|
|
pc.updateResults(iterationLogic.numIters());
|
|
iterationLogic.add(after - before, pc);
|
|
}
|
|
iterationLogic.moveResultTo(mResults);
|
|
return *this;
|
|
}
|
|
|
|
template <typename... Args>
|
|
CompareResult Bench::compare(Args&&... args) {
|
|
static_assert(sizeof...(args) % 2 == 0, "compare() takes `name, op` pairs");
|
|
static_assert(sizeof...(args) >= 4, "compare() needs at least two alternatives to compare");
|
|
|
|
std::vector<std::string> names;
|
|
std::vector<detail::ErasedOp> ops;
|
|
names.reserve(sizeof...(args) / 2);
|
|
ops.reserve(sizeof...(args) / 2);
|
|
compareCollect(names, ops, std::forward<Args>(args)...);
|
|
return compareImpl(names, ops);
|
|
}
|
|
|
|
template <typename SetupOp>
|
|
detail::SetupRunner<SetupOp> Bench::setup(SetupOp setupOp) {
|
|
return detail::SetupRunner<SetupOp>(std::move(setupOp), *this);
|
|
}
|
|
|
|
// Performs all evaluations.
|
|
template <typename Op>
|
|
Bench& Bench::run(char const* benchmarkName, Op&& op) {
|
|
name(benchmarkName);
|
|
return run(std::forward<Op>(op));
|
|
}
|
|
|
|
template <typename Op>
|
|
Bench& Bench::run(std::string const& benchmarkName, Op&& op) {
|
|
name(benchmarkName);
|
|
return run(std::forward<Op>(op));
|
|
}
|
|
|
|
#if ANKERL_NANOBENCH(HAS_STRING_VIEW)
|
|
template <typename Op>
|
|
Bench& Bench::run(std::string_view benchmarkName, Op&& op) {
|
|
name(benchmarkName);
|
|
return run(std::forward<Op>(op));
|
|
}
|
|
#endif
|
|
|
|
template <typename Op>
|
|
BigO Bench::complexityBigO(char const* benchmarkName, Op op) const {
|
|
return BigO(benchmarkName, BigO::collectRangeMeasure(mResults), op);
|
|
}
|
|
|
|
template <typename Op>
|
|
BigO Bench::complexityBigO(std::string const& benchmarkName, Op op) const {
|
|
return BigO(benchmarkName, BigO::collectRangeMeasure(mResults), op);
|
|
}
|
|
|
|
// Set the batch size, e.g. number of processed bytes, or some other metric for the size of the processed data in each iteration.
|
|
// Any argument is cast to double.
|
|
template <typename T>
|
|
Bench& Bench::batch(T b) noexcept {
|
|
mConfig.mBatch = static_cast<double>(b);
|
|
return *this;
|
|
}
|
|
|
|
// Sets the computation complexity of the next run. Any argument is cast to double.
|
|
template <typename T>
|
|
Bench& Bench::complexityN(T n) noexcept {
|
|
mConfig.mComplexityN = static_cast<double>(n);
|
|
return *this;
|
|
}
|
|
|
|
// Convenience: makes sure none of the given arguments are optimized away by the compiler.
|
|
template <typename Arg>
|
|
Bench& Bench::doNotOptimizeAway(Arg&& arg) {
|
|
detail::doNotOptimizeAway(std::forward<Arg>(arg));
|
|
return *this;
|
|
}
|
|
|
|
// Makes sure none of the given arguments are optimized away by the compiler.
|
|
template <typename Arg>
|
|
void doNotOptimizeAway(Arg&& arg) {
|
|
detail::doNotOptimizeAway(std::forward<Arg>(arg));
|
|
}
|
|
|
|
namespace detail {
|
|
|
|
#if !ANKERL_NANOBENCH(ASM_DONT_OPTIMIZE_AWAY)
|
|
template <typename T>
|
|
void doNotOptimizeAway(T const& val) {
|
|
doNotOptimizeAwaySink(&val);
|
|
}
|
|
|
|
#endif
|
|
|
|
} // namespace detail
|
|
} // namespace nanobench
|
|
} // namespace ankerl
|
|
|
|
#if defined(ANKERL_NANOBENCH_IMPLEMENT)
|
|
|
|
///////////////////////////////////////////////////////////////////////////////////////////////////
|
|
// implementation part - only visible in .cpp
|
|
///////////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
# include <algorithm> // sort, reverse
|
|
# include <atomic> // compare_exchange_strong in loop overhead
|
|
# include <cstdlib> // getenv
|
|
# include <cstring> // strstr, strncmp
|
|
# include <fstream> // ifstream to parse proc files
|
|
# include <iomanip> // setw, setprecision
|
|
# include <iostream> // cout
|
|
# include <limits> // numeric_limits, to parse NANOBENCH_CONFIG without overflowing
|
|
# include <numeric> // accumulate
|
|
# include <random> // random_device
|
|
# include <sstream> // to_s in Number
|
|
# include <stdexcept> // throw for rendering templates
|
|
# if defined(__linux__)
|
|
# include <unistd.h> //sysconf
|
|
# endif
|
|
# if ANKERL_NANOBENCH(PERF_COUNTERS)
|
|
# include <map> // map
|
|
|
|
# include <linux/perf_event.h>
|
|
# include <sys/ioctl.h>
|
|
# include <sys/syscall.h>
|
|
# endif
|
|
|
|
// declarations ///////////////////////////////////////////////////////////////////////////////////
|
|
|
|
namespace ankerl {
|
|
namespace nanobench {
|
|
|
|
// helper stuff that is only intended to be used internally
|
|
namespace detail {
|
|
|
|
// formatting utilities
|
|
namespace fmt {
|
|
|
|
// groups the integer digits of an already formatted number in threes
|
|
std::string addThousandsSeparators(std::string str, char sep);
|
|
|
|
class StreamStateRestorer;
|
|
class Number;
|
|
class MarkDownColumn;
|
|
class MarkDownCode;
|
|
|
|
} // namespace fmt
|
|
} // namespace detail
|
|
} // namespace nanobench
|
|
} // namespace ankerl
|
|
|
|
// definitions ////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
namespace ankerl {
|
|
namespace nanobench {
|
|
|
|
uint64_t splitMix64(uint64_t& state) noexcept;
|
|
|
|
namespace detail {
|
|
|
|
// helpers to get double values
|
|
template <typename T>
|
|
inline double d(T t) noexcept {
|
|
return static_cast<double>(t);
|
|
}
|
|
inline double d(Clock::duration duration) noexcept {
|
|
return std::chrono::duration_cast<std::chrono::duration<double>>(duration).count();
|
|
}
|
|
|
|
// Rounds to the nearest uint64_t. Casting a double that doesn't fit is undefined behavior, so the value is clamped
|
|
// into a safe range first. The comparisons are written so that a NaN takes the same path as a negative value.
|
|
inline uint64_t u64(double val) noexcept {
|
|
auto const maxVal = d((std::numeric_limits<uint64_t>::max)() / 2U);
|
|
if (!(val > 0.0)) {
|
|
return 0;
|
|
}
|
|
// +0.5 for correct rounding when casting
|
|
// NOLINTNEXTLINE(bugprone-incorrect-roundings)
|
|
return static_cast<uint64_t>((val < maxVal ? val : maxVal) + 0.5);
|
|
}
|
|
|
|
// Calculates clock resolution once, and remembers the result
|
|
inline Clock::duration clockResolution() noexcept;
|
|
|
|
} // namespace detail
|
|
|
|
namespace templates {
|
|
|
|
char const* csv() noexcept {
|
|
return R"DELIM("title";"name";"unit";"batch";"elapsed";"error %";"instructions";"branches";"branch misses";"total"
|
|
{{#result}}"{{title}}";"{{name}}";"{{unit}}";{{batch}};{{median(elapsed)}};{{medianAbsolutePercentError(elapsed)}};{{median(instructions)}};{{median(branchinstructions)}};{{median(branchmisses)}};{{sumProduct(iterations, elapsed)}}
|
|
{{/result}})DELIM";
|
|
}
|
|
|
|
char const* compareCsv() noexcept {
|
|
return R"DELIM("title";"name";"relative";"relative low";"relative high";"significant";"tied rounds";"rounds";"elapsed";"error %"
|
|
{{#alternative}}"{{title}}";"{{name}}";{{relative}};{{relativeLow}};{{relativeHigh}};{{significant}};{{tiedRounds}};{{rounds}};{{median(elapsed)}};{{medianAbsolutePercentError(elapsed)}}
|
|
{{/alternative}})DELIM";
|
|
}
|
|
|
|
char const* htmlBoxplot() noexcept {
|
|
return R"DELIM(<html>
|
|
|
|
<head>
|
|
<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
|
|
</head>
|
|
|
|
<body>
|
|
<div id="myDiv"></div>
|
|
<script>
|
|
var data = [
|
|
{{#result}}{
|
|
name: '{{name}}',
|
|
y: [{{#measurement}}{{elapsed}}{{^-last}}, {{/last}}{{/measurement}}],
|
|
},
|
|
{{/result}}
|
|
];
|
|
var title = '{{title}}';
|
|
|
|
data = data.map(a => Object.assign(a, { boxpoints: 'all', pointpos: 0, type: 'box' }));
|
|
var layout = { title: { text: title }, showlegend: false, yaxis: { title: 'time per unit', rangemode: 'tozero', autorange: true } }; Plotly.newPlot('myDiv', data, layout, {responsive: true});
|
|
</script>
|
|
</body>
|
|
|
|
</html>)DELIM";
|
|
}
|
|
|
|
char const* pyperf() noexcept {
|
|
return R"DELIM({
|
|
"benchmarks": [
|
|
{
|
|
"runs": [
|
|
{
|
|
"values": [
|
|
{{#measurement}} {{elapsed}}{{^-last}},
|
|
{{/last}}{{/measurement}}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"loops": {{sum(iterations)}},
|
|
"inner_loops": {{batch}},
|
|
"name": "{{title}}",
|
|
"unit": "second"
|
|
},
|
|
"version": "1.0"
|
|
})DELIM";
|
|
}
|
|
|
|
char const* json() noexcept {
|
|
return R"DELIM({
|
|
"results": [
|
|
{{#result}} {
|
|
"title": "{{title}}",
|
|
"name": "{{name}}",
|
|
"unit": "{{unit}}",
|
|
"batch": {{batch}},
|
|
"complexityN": {{complexityN}},
|
|
"epochs": {{epochs}},
|
|
"clockResolution": {{clockResolution}},
|
|
"clockResolutionMultiple": {{clockResolutionMultiple}},
|
|
"maxEpochTime": {{maxEpochTime}},
|
|
"minEpochTime": {{minEpochTime}},
|
|
"minEpochIterations": {{minEpochIterations}},
|
|
"epochIterations": {{epochIterations}},
|
|
"warmup": {{warmup}},
|
|
"relative": {{relative}},
|
|
"median(elapsed)": {{median(elapsed)}},
|
|
"medianAbsolutePercentError(elapsed)": {{medianAbsolutePercentError(elapsed)}},
|
|
"median(instructions)": {{median(instructions)}},
|
|
"medianAbsolutePercentError(instructions)": {{medianAbsolutePercentError(instructions)}},
|
|
"median(cpucycles)": {{median(cpucycles)}},
|
|
"median(contextswitches)": {{median(contextswitches)}},
|
|
"median(pagefaults)": {{median(pagefaults)}},
|
|
"median(branchinstructions)": {{median(branchinstructions)}},
|
|
"median(branchmisses)": {{median(branchmisses)}},
|
|
"totalTime": {{sumProduct(iterations, elapsed)}},
|
|
"measurements": [
|
|
{{#measurement}} {
|
|
"iterations": {{iterations}},
|
|
"elapsed": {{elapsed}},
|
|
"pagefaults": {{pagefaults}},
|
|
"cpucycles": {{cpucycles}},
|
|
"contextswitches": {{contextswitches}},
|
|
"instructions": {{instructions}},
|
|
"branchinstructions": {{branchinstructions}},
|
|
"branchmisses": {{branchmisses}}
|
|
}{{^-last}},{{/-last}}
|
|
{{/measurement}} ]
|
|
}{{^-last}},{{/-last}}
|
|
{{/result}} ]
|
|
})DELIM";
|
|
}
|
|
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_PUSH)
|
|
struct Node {
|
|
enum class Type { tag, content, section, inverted_section };
|
|
|
|
char const* begin;
|
|
char const* end;
|
|
std::vector<Node> children;
|
|
Type type;
|
|
|
|
template <size_t N>
|
|
// NOLINTNEXTLINE(hicpp-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays)
|
|
bool operator==(char const (&str)[N]) const noexcept {
|
|
// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay)
|
|
return static_cast<size_t>(std::distance(begin, end) + 1) == N && 0 == strncmp(str, begin, N - 1);
|
|
}
|
|
};
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_POP)
|
|
|
|
// A node points into the template rather than owning its text, and these two are the only places that
|
|
// have to know it: everywhere else asks for the text or writes it out. Free functions rather than
|
|
// members, so Node stays the plain aggregate that parseMustacheTemplate() brace-initializes.
|
|
static std::string text(Node const& n) {
|
|
return {n.begin, n.end};
|
|
}
|
|
|
|
static void writeTo(Node const& n, std::ostream& out) {
|
|
out.write(n.begin, std::distance(n.begin, n.end));
|
|
}
|
|
|
|
// NOLINTNEXTLINE(misc-no-recursion)
|
|
static std::vector<Node> parseMustacheTemplate(char const** tpl) {
|
|
std::vector<Node> nodes;
|
|
|
|
while (true) {
|
|
auto const* begin = std::strstr(*tpl, "{{");
|
|
auto const* end = begin;
|
|
if (begin != nullptr) {
|
|
// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
|
|
begin += 2;
|
|
end = std::strstr(begin, "}}");
|
|
}
|
|
|
|
if (begin == nullptr || end == nullptr) {
|
|
// nothing found, finish node
|
|
// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
|
|
nodes.emplace_back(Node{*tpl, *tpl + std::strlen(*tpl), std::vector<Node>{}, Node::Type::content});
|
|
return nodes;
|
|
}
|
|
|
|
// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
|
|
nodes.emplace_back(Node{*tpl, begin - 2, std::vector<Node>{}, Node::Type::content});
|
|
|
|
// we found a tag
|
|
// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
|
|
*tpl = end + 2;
|
|
switch (*begin) {
|
|
case '/':
|
|
// finished! bail out
|
|
return nodes;
|
|
|
|
case '#':
|
|
// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
|
|
nodes.emplace_back(Node{begin + 1, end, parseMustacheTemplate(tpl), Node::Type::section});
|
|
break;
|
|
|
|
case '^':
|
|
// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
|
|
nodes.emplace_back(Node{begin + 1, end, parseMustacheTemplate(tpl), Node::Type::inverted_section});
|
|
break;
|
|
|
|
default:
|
|
nodes.emplace_back(Node{begin, end, std::vector<Node>{}, Node::Type::tag});
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
static bool generateFirstLast(Node const& n, size_t idx, size_t size, std::ostream& out) {
|
|
ANKERL_NANOBENCH_LOG("n.type=" << static_cast<int>(n.type));
|
|
bool const matchFirst = n == "-first";
|
|
bool const matchLast = n == "-last";
|
|
if (!matchFirst && !matchLast) {
|
|
return false;
|
|
}
|
|
|
|
bool doWrite = false;
|
|
if (n.type == Node::Type::section) {
|
|
doWrite = (matchFirst && idx == 0) || (matchLast && idx == size - 1);
|
|
} else if (n.type == Node::Type::inverted_section) {
|
|
doWrite = (matchFirst && idx != 0) || (matchLast && idx != size - 1);
|
|
}
|
|
|
|
if (doWrite) {
|
|
for (auto const& child : n.children) {
|
|
if (child.type == Node::Type::content) {
|
|
writeTo(child, out);
|
|
}
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static bool matchCmdArgs(std::string const& str, std::vector<std::string>& matchResult) {
|
|
matchResult.clear();
|
|
auto idxOpen = str.find('(');
|
|
auto idxClose = str.find(')', idxOpen);
|
|
if (idxClose == std::string::npos) {
|
|
return false;
|
|
}
|
|
|
|
matchResult.emplace_back(str.substr(0, idxOpen));
|
|
|
|
// split by comma
|
|
matchResult.emplace_back();
|
|
for (size_t i = idxOpen + 1; i != idxClose; ++i) {
|
|
if (str[i] == ' ' || str[i] == '\t') {
|
|
// skip whitespace
|
|
continue;
|
|
}
|
|
if (str[i] == ',') {
|
|
// got a comma => new string
|
|
matchResult.emplace_back();
|
|
continue;
|
|
}
|
|
// no whitespace no comma, append
|
|
matchResult.back() += str[i];
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// One tag: writes `value` when the node names it, and reports whether it did.
|
|
template <size_t N, typename T>
|
|
// NOLINTNEXTLINE(hicpp-avoid-c-arrays,modernize-avoid-c-arrays,cppcoreguidelines-avoid-c-arrays)
|
|
static bool writeTag(Node const& n, char const (&tagName)[N], T const& value, std::ostream& out) {
|
|
if (!(n == tagName)) {
|
|
return false;
|
|
}
|
|
out << value;
|
|
return true;
|
|
}
|
|
|
|
static bool generateConfigTag(Node const& n, Config const& config, std::ostream& out) {
|
|
using detail::d;
|
|
|
|
// Not part of the chain below: the chain evaluates the arguments of every tag up to the one that
|
|
// matches, and measuring the clock's resolution the first time is the one that is not free.
|
|
if (n == "clockResolution") {
|
|
out << d(detail::clockResolution());
|
|
return true;
|
|
}
|
|
|
|
// '||' short-circuits, so at most one of these writes anything and nothing past the match is even
|
|
// evaluated. Written out as fourteen copies of writeTag()'s body, this was a four-line stanza per
|
|
// tag whose only compiler-visible mistake - forgetting the `return true` - fell through to
|
|
// "command not understood" at runtime.
|
|
return writeTag(n, "title", config.mBenchmarkTitle, out) || writeTag(n, "name", config.mBenchmarkName, out) ||
|
|
writeTag(n, "unit", config.mUnit, out) || writeTag(n, "batch", config.mBatch, out) ||
|
|
writeTag(n, "complexityN", config.mComplexityN, out) || writeTag(n, "epochs", config.mNumEpochs, out) ||
|
|
writeTag(n, "clockResolutionMultiple", config.mClockResolutionMultiple, out) ||
|
|
writeTag(n, "maxEpochTime", d(config.mMaxEpochTime), out) || writeTag(n, "minEpochTime", d(config.mMinEpochTime), out) ||
|
|
writeTag(n, "minEpochIterations", config.mMinEpochIterations, out) ||
|
|
writeTag(n, "epochIterations", config.mEpochIterations, out) || writeTag(n, "warmup", config.mWarmup, out) ||
|
|
writeTag(n, "relative", config.mIsRelative, out);
|
|
}
|
|
|
|
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
|
|
// `elapsed` is seconds, which is rarely the unit anyone wants in a report, and the template language
|
|
// has no arithmetic to fix that afterwards (issue #107). So a time measure may carry a unit suffix -
|
|
// `elapsedms`, `elapsedus`, `elapsedns` - and this resolves it to the measure plus the factor to
|
|
// multiply by. Plain `elapsed` keeps its meaning, so existing templates render byte for byte as
|
|
// before.
|
|
static Result::Measure measureFromString(std::string const& str, double& scale) {
|
|
scale = 1.0;
|
|
auto m = Result::fromString(str);
|
|
if (Result::Measure::_size != m) {
|
|
return m;
|
|
}
|
|
|
|
if (str == "elapsedms") {
|
|
scale = 1e3;
|
|
return Result::Measure::elapsed;
|
|
}
|
|
if (str == "elapsedus") {
|
|
scale = 1e6;
|
|
return Result::Measure::elapsed;
|
|
}
|
|
if (str == "elapsedns") {
|
|
scale = 1e9;
|
|
return Result::Measure::elapsed;
|
|
}
|
|
return Result::Measure::_size;
|
|
}
|
|
|
|
// The one-argument measure commands - {{median(elapsed)}} and friends. Split out of
|
|
// generateResultTag so that adding the unit-scaled measures did not push its cognitive complexity
|
|
// past what clang-tidy accepts.
|
|
static bool generateMeasureTag(std::string const& command, Result const& r, Result::Measure m, double scale, std::ostream& out) {
|
|
if (command == "median") {
|
|
out << r.median(m) * scale;
|
|
return true;
|
|
}
|
|
if (command == "average") {
|
|
out << r.average(m) * scale;
|
|
return true;
|
|
}
|
|
if (command == "medianAbsolutePercentError") {
|
|
// a relative error is the same number whatever the unit, so this one is not scaled
|
|
out << r.medianAbsolutePercentError(m);
|
|
return true;
|
|
}
|
|
if (command == "sum") {
|
|
out << r.sum(m) * scale;
|
|
return true;
|
|
}
|
|
if (command == "minimum") {
|
|
out << r.minimum(m) * scale;
|
|
return true;
|
|
}
|
|
if (command == "maximum") {
|
|
out << r.maximum(m) * scale;
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static std::ostream& generateResultTag(Node const& n, Result const& r, std::ostream& out) {
|
|
if (generateConfigTag(n, r.config(), out)) {
|
|
return out;
|
|
}
|
|
// match e.g. "median(elapsed)"
|
|
// g++ 4.8 doesn't implement std::regex :(
|
|
// static std::regex const regOpArg1("^([a-zA-Z]+)\\(([a-zA-Z]*)\\)$");
|
|
// std::cmatch matchResult;
|
|
// if (std::regex_match(n.begin, n.end, matchResult, regOpArg1)) {
|
|
std::vector<std::string> matchResult;
|
|
if (matchCmdArgs(text(n), matchResult)) {
|
|
if (matchResult.size() == 2) {
|
|
if (matchResult[0] == "context") {
|
|
return out << r.context(matchResult[1]);
|
|
}
|
|
|
|
double scale = 1.0;
|
|
auto m = measureFromString(matchResult[1], scale);
|
|
if (m == Result::Measure::_size) {
|
|
return out << 0.0;
|
|
}
|
|
if (generateMeasureTag(matchResult[0], r, m, scale, out)) {
|
|
return out;
|
|
}
|
|
} else if (matchResult.size() == 3) {
|
|
double scale1 = 1.0;
|
|
double scale2 = 1.0;
|
|
auto m1 = measureFromString(matchResult[1], scale1);
|
|
auto m2 = measureFromString(matchResult[2], scale2);
|
|
if (m1 == Result::Measure::_size || m2 == Result::Measure::_size) {
|
|
return out << 0.0;
|
|
}
|
|
|
|
if (matchResult[0] == "sumProduct") {
|
|
return out << r.sumProduct(m1, m2) * scale1 * scale2;
|
|
}
|
|
}
|
|
}
|
|
|
|
// match e.g. "sumProduct(elapsed, iterations)"
|
|
// static std::regex const regOpArg2("^([a-zA-Z]+)\\(([a-zA-Z]*)\\s*,\\s+([a-zA-Z]*)\\)$");
|
|
|
|
// nothing matches :(
|
|
ANKERL_NANOBENCH_THROW(std::runtime_error("command '" + text(n) + "' not understood"));
|
|
}
|
|
|
|
static void generateResultMeasurement(std::vector<Node> const& nodes, size_t idx, Result const& r, std::ostream& out) {
|
|
for (auto const& n : nodes) {
|
|
if (!generateFirstLast(n, idx, r.size(), out)) {
|
|
ANKERL_NANOBENCH_LOG("n.type=" << static_cast<int>(n.type));
|
|
switch (n.type) {
|
|
case Node::Type::content:
|
|
writeTo(n, out);
|
|
break;
|
|
|
|
case Node::Type::inverted_section:
|
|
ANKERL_NANOBENCH_THROW(std::runtime_error("got a inverted section inside measurement"));
|
|
|
|
case Node::Type::section:
|
|
ANKERL_NANOBENCH_THROW(std::runtime_error("got a section inside measurement"));
|
|
|
|
case Node::Type::tag: {
|
|
double scale = 1.0;
|
|
auto m = measureFromString(text(n), scale);
|
|
if (m == Result::Measure::_size || !r.has(m)) {
|
|
out << 0.0;
|
|
} else {
|
|
out << r.get(idx, m) * scale;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// The tags a comparison adds on top of a result's. Returns false for anything it does not know, so
|
|
// the caller can fall back to the result tags and a template keeps working whichever it names.
|
|
static bool generateCompareTag(Node const& n, CompareResult const& cmp, size_t idx, std::ostream& out) {
|
|
if (n == "relative") {
|
|
out << cmp[idx].relative;
|
|
} else if (n == "relativeLow") {
|
|
out << cmp[idx].relativeLow;
|
|
} else if (n == "relativeHigh") {
|
|
out << cmp[idx].relativeHigh;
|
|
} else if (n == "tiedRounds") {
|
|
out << cmp[idx].tiedRounds;
|
|
} else if (n == "significant") {
|
|
out << (cmp.isSignificant(idx) ? 1 : 0);
|
|
} else if (n == "rounds") {
|
|
out << cmp.rounds();
|
|
} else if (n == "comparisons") {
|
|
out << cmp.comparisons();
|
|
} else {
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static void generateCompareEntry(std::vector<Node> const& nodes, size_t idx, CompareResult const& cmp, std::ostream& out) {
|
|
auto const& r = cmp[idx].result;
|
|
for (auto const& n : nodes) {
|
|
if (!generateFirstLast(n, idx, cmp.size(), out)) {
|
|
ANKERL_NANOBENCH_LOG("n.type=" << static_cast<int>(n.type));
|
|
switch (n.type) {
|
|
case Node::Type::content:
|
|
writeTo(n, out);
|
|
break;
|
|
|
|
case Node::Type::inverted_section:
|
|
ANKERL_NANOBENCH_THROW(std::runtime_error("got a inverted section inside alternative"));
|
|
|
|
case Node::Type::section:
|
|
if (n == "measurement") {
|
|
for (size_t i = 0; i < r.size(); ++i) {
|
|
generateResultMeasurement(n.children, i, r, out);
|
|
}
|
|
} else {
|
|
ANKERL_NANOBENCH_THROW(std::runtime_error("got a section inside alternative"));
|
|
}
|
|
break;
|
|
|
|
case Node::Type::tag:
|
|
if (!generateCompareTag(n, cmp, idx, out)) {
|
|
generateResultTag(n, r, out);
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
static void generateResult(std::vector<Node> const& nodes, size_t idx, std::vector<Result> const& results, std::ostream& out) {
|
|
auto const& r = results[idx];
|
|
for (auto const& n : nodes) {
|
|
if (!generateFirstLast(n, idx, results.size(), out)) {
|
|
ANKERL_NANOBENCH_LOG("n.type=" << static_cast<int>(n.type));
|
|
switch (n.type) {
|
|
case Node::Type::content:
|
|
writeTo(n, out);
|
|
break;
|
|
|
|
case Node::Type::inverted_section:
|
|
ANKERL_NANOBENCH_THROW(std::runtime_error("got a inverted section inside result"));
|
|
|
|
case Node::Type::section:
|
|
if (n == "measurement") {
|
|
for (size_t i = 0; i < r.size(); ++i) {
|
|
generateResultMeasurement(n.children, i, r, out);
|
|
}
|
|
} else {
|
|
ANKERL_NANOBENCH_THROW(std::runtime_error("got a section inside result"));
|
|
}
|
|
break;
|
|
|
|
case Node::Type::tag:
|
|
generateResultTag(n, r, out);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
} // namespace templates
|
|
|
|
// helper stuff that only intended to be used internally
|
|
namespace detail {
|
|
|
|
char const* getEnv(char const* name);
|
|
bool isEndlessRunning(std::string const& name);
|
|
bool isWarningsEnabled();
|
|
|
|
// Applies NANOBENCH_CONFIG to a freshly built Bench, printing whatever it could not use. Every Bench
|
|
// re-reads and re-parses the variable; only the complaining is once per process.
|
|
void applyEnvConfig(Bench& bench);
|
|
|
|
template <typename T>
|
|
T parseFile(std::string const& filename, bool* fail);
|
|
|
|
void gatherStabilityInformation(std::vector<std::string>& warnings, std::vector<std::string>& recommendations);
|
|
|
|
// determines resolution of the given clock. This is done by measuring multiple times and returning the minimum time difference.
|
|
Clock::duration calcClockResolution(size_t numEvaluations) noexcept;
|
|
|
|
// formatting utilities
|
|
namespace fmt {
|
|
|
|
// RAII to save & restore a stream's state
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_PUSH)
|
|
class StreamStateRestorer {
|
|
public:
|
|
explicit StreamStateRestorer(std::ostream& s);
|
|
~StreamStateRestorer();
|
|
|
|
// sets back all stream info that we remembered at construction
|
|
void restore();
|
|
|
|
// don't allow copying / moving
|
|
StreamStateRestorer(StreamStateRestorer const&) = delete;
|
|
StreamStateRestorer& operator=(StreamStateRestorer const&) = delete;
|
|
StreamStateRestorer(StreamStateRestorer&&) = delete;
|
|
StreamStateRestorer& operator=(StreamStateRestorer&&) = delete;
|
|
|
|
private:
|
|
std::ostream& mStream;
|
|
std::locale mLocale;
|
|
std::streamsize const mPrecision;
|
|
std::streamsize const mWidth;
|
|
std::ostream::char_type const mFill;
|
|
std::ostream::fmtflags const mFmtFlags;
|
|
};
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_POP)
|
|
|
|
// Number formatter
|
|
class Number {
|
|
public:
|
|
Number(int width, int precision, double value);
|
|
ANKERL_NANOBENCH(NODISCARD) std::string to_s() const;
|
|
|
|
private:
|
|
friend std::ostream& operator<<(std::ostream& os, Number const& n);
|
|
std::ostream& write(std::ostream& os) const;
|
|
|
|
int mWidth;
|
|
int mPrecision;
|
|
double mValue;
|
|
};
|
|
|
|
// helper replacement for std::to_string of signed/unsigned numbers so we are locale independent
|
|
std::string to_s(uint64_t n);
|
|
|
|
std::ostream& operator<<(std::ostream& os, Number const& n);
|
|
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_PUSH)
|
|
class MarkDownColumn {
|
|
public:
|
|
MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val) noexcept;
|
|
// a column holding text instead of a number, for the context columns
|
|
MarkDownColumn(int w, std::string tit, std::string text) noexcept;
|
|
ANKERL_NANOBENCH(NODISCARD) std::string title() const;
|
|
ANKERL_NANOBENCH(NODISCARD) std::string separator() const;
|
|
ANKERL_NANOBENCH(NODISCARD) std::string invalid() const;
|
|
ANKERL_NANOBENCH(NODISCARD) std::string value() const;
|
|
|
|
private:
|
|
// the column's padding convention, stated once: right aligned in mWidth, one trailing space
|
|
ANKERL_NANOBENCH(NODISCARD) std::string padded(std::string const& text) const;
|
|
|
|
int mWidth;
|
|
int mPrecision;
|
|
std::string mTitle;
|
|
std::string mSuffix;
|
|
double mValue;
|
|
std::string mText{};
|
|
bool mIsText{false};
|
|
};
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_POP)
|
|
|
|
// Formats any text as markdown code, escaping backticks.
|
|
class MarkDownCode {
|
|
public:
|
|
explicit MarkDownCode(std::string const& what);
|
|
|
|
private:
|
|
friend std::ostream& operator<<(std::ostream& os, MarkDownCode const& mdCode);
|
|
std::ostream& write(std::ostream& os) const;
|
|
|
|
std::string mWhat{};
|
|
};
|
|
|
|
std::ostream& operator<<(std::ostream& os, MarkDownCode const& mdCode);
|
|
|
|
} // namespace fmt
|
|
} // namespace detail
|
|
} // namespace nanobench
|
|
} // namespace ankerl
|
|
|
|
// implementation /////////////////////////////////////////////////////////////////////////////////
|
|
|
|
namespace ankerl {
|
|
namespace nanobench {
|
|
|
|
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
|
|
void render(char const* mustacheTemplate, std::vector<Result> const& results, std::ostream& out) {
|
|
detail::fmt::StreamStateRestorer const restorer(out);
|
|
|
|
out.precision(std::numeric_limits<double>::digits10);
|
|
auto nodes = templates::parseMustacheTemplate(&mustacheTemplate);
|
|
|
|
for (auto const& n : nodes) {
|
|
ANKERL_NANOBENCH_LOG("n.type=" << static_cast<int>(n.type));
|
|
switch (n.type) {
|
|
case templates::Node::Type::content:
|
|
templates::writeTo(n, out);
|
|
break;
|
|
|
|
case templates::Node::Type::inverted_section:
|
|
ANKERL_NANOBENCH_THROW(std::runtime_error("unknown list '" + templates::text(n) + "'"));
|
|
|
|
case templates::Node::Type::section:
|
|
if (n == "result") {
|
|
const size_t nbResults = results.size();
|
|
for (size_t i = 0; i < nbResults; ++i) {
|
|
generateResult(n.children, i, results, out);
|
|
}
|
|
} else if (n == "measurement") {
|
|
if (results.size() != 1) {
|
|
ANKERL_NANOBENCH_THROW(std::runtime_error(
|
|
"render: can only use section 'measurement' here if there is a single result, but there are " +
|
|
detail::fmt::to_s(results.size())));
|
|
}
|
|
// when we only have a single result, we can immediately go into its measurement.
|
|
auto const& r = results.front();
|
|
for (size_t i = 0; i < r.size(); ++i) {
|
|
generateResultMeasurement(n.children, i, r, out);
|
|
}
|
|
} else {
|
|
ANKERL_NANOBENCH_THROW(std::runtime_error("render: unknown section '" + templates::text(n) + "'"));
|
|
}
|
|
break;
|
|
|
|
case templates::Node::Type::tag:
|
|
if (results.size() == 1) {
|
|
// result & config are both supported there
|
|
generateResultTag(n, results.front(), out);
|
|
} else {
|
|
// This just uses the last result's config.
|
|
if (!generateConfigTag(n, results.back().config(), out)) {
|
|
ANKERL_NANOBENCH_THROW(std::runtime_error("unknown tag '" + templates::text(n) + "'"));
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
void render(char const* mustacheTemplate, CompareResult const& cmp, std::ostream& out) {
|
|
detail::fmt::StreamStateRestorer const restorer(out);
|
|
|
|
out.precision(std::numeric_limits<double>::digits10);
|
|
auto nodes = templates::parseMustacheTemplate(&mustacheTemplate);
|
|
|
|
for (auto const& n : nodes) {
|
|
ANKERL_NANOBENCH_LOG("n.type=" << static_cast<int>(n.type));
|
|
switch (n.type) {
|
|
case templates::Node::Type::content:
|
|
templates::writeTo(n, out);
|
|
break;
|
|
|
|
case templates::Node::Type::inverted_section:
|
|
ANKERL_NANOBENCH_THROW(std::runtime_error("unknown list '" + templates::text(n) + "'"));
|
|
|
|
case templates::Node::Type::section:
|
|
if (n == "alternative") {
|
|
for (size_t i = 0; i < cmp.size(); ++i) {
|
|
templates::generateCompareEntry(n.children, i, cmp, out);
|
|
}
|
|
} else {
|
|
ANKERL_NANOBENCH_THROW(std::runtime_error("render: unknown section '" + templates::text(n) + "'"));
|
|
}
|
|
break;
|
|
|
|
case templates::Node::Type::tag:
|
|
// Outside the section only the experiment-wide tags mean anything; the baseline's config
|
|
// answers for the rest, which is what makes a bare {{title}} in a header line work.
|
|
if (!templates::generateCompareTag(n, cmp, 0, out) && !templates::generateConfigTag(n, cmp[0].result.config(), out)) {
|
|
ANKERL_NANOBENCH_THROW(std::runtime_error("unknown tag '" + templates::text(n) + "'"));
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
void render(std::string const& mustacheTemplate, CompareResult const& cmp, std::ostream& out) {
|
|
render(mustacheTemplate.c_str(), cmp, out);
|
|
}
|
|
|
|
void render(std::string const& mustacheTemplate, std::vector<Result> const& results, std::ostream& out) {
|
|
render(mustacheTemplate.c_str(), results, out);
|
|
}
|
|
|
|
void render(char const* mustacheTemplate, const Bench& bench, std::ostream& out) {
|
|
render(mustacheTemplate, bench.results(), out);
|
|
}
|
|
|
|
void render(std::string const& mustacheTemplate, const Bench& bench, std::ostream& out) {
|
|
render(mustacheTemplate.c_str(), bench.results(), out);
|
|
}
|
|
|
|
namespace detail {
|
|
|
|
PerformanceCounters& performanceCounters() {
|
|
# if defined(__clang__)
|
|
# pragma clang diagnostic push
|
|
# pragma clang diagnostic ignored "-Wexit-time-destructors"
|
|
# endif
|
|
static PerformanceCounters pc;
|
|
# if defined(__clang__)
|
|
# pragma clang diagnostic pop
|
|
# endif
|
|
return pc;
|
|
}
|
|
|
|
Clock::duration targetRuntimePerEpoch(Bench const& bench) {
|
|
auto target = detail::clockResolution() * bench.clockResolutionMultiple();
|
|
if (target > bench.maxEpochTime()) {
|
|
target = bench.maxEpochTime();
|
|
}
|
|
if (target < bench.minEpochTime()) {
|
|
target = bench.minEpochTime();
|
|
}
|
|
return target;
|
|
}
|
|
|
|
// Windows version of doNotOptimizeAway
|
|
// see https://github.com/google/benchmark/blob/v1.7.1/include/benchmark/benchmark.h#L514
|
|
// see https://github.com/facebook/folly/blob/v2023.01.30.00/folly/lang/Hint-inl.h#L54-L58
|
|
// see https://learn.microsoft.com/en-us/cpp/preprocessor/optimize
|
|
# if !ANKERL_NANOBENCH(ASM_DONT_OPTIMIZE_AWAY)
|
|
# if defined(_MSC_VER)
|
|
# pragma optimize("", off)
|
|
# endif
|
|
void doNotOptimizeAwaySink(void const*) {}
|
|
# if defined(_MSC_VER)
|
|
# pragma optimize("", on)
|
|
# endif
|
|
# endif
|
|
|
|
template <typename T>
|
|
T parseFile(std::string const& filename, bool* fail) {
|
|
std::ifstream fin(filename); // NOLINT(misc-const-correctness)
|
|
T num{};
|
|
fin >> num;
|
|
if (fail != nullptr) {
|
|
*fail = fin.fail();
|
|
}
|
|
return num;
|
|
}
|
|
|
|
char const* getEnv(char const* name) {
|
|
# if defined(_MSC_VER)
|
|
# pragma warning(push)
|
|
# pragma warning(disable : 4996) // getenv': This function or variable may be unsafe.
|
|
# endif
|
|
return std::getenv(name); // NOLINT(concurrency-mt-unsafe)
|
|
# if defined(_MSC_VER)
|
|
# pragma warning(pop)
|
|
# endif
|
|
}
|
|
|
|
bool isEndlessRunning(std::string const& name) {
|
|
auto const* const endless = getEnv("NANOBENCH_ENDLESS");
|
|
return nullptr != endless && endless == name;
|
|
}
|
|
|
|
// True when environment variable NANOBENCH_SUPPRESS_WARNINGS is either not set at all, or set to "0"
|
|
bool isWarningsEnabled() {
|
|
auto const* const suppression = getEnv("NANOBENCH_SUPPRESS_WARNINGS");
|
|
return nullptr == suppression || suppression == std::string("0");
|
|
}
|
|
|
|
// NANOBENCH_CONFIG ///////////////////////////////////////////////////////////////////////////////
|
|
//
|
|
// The parsing below reports a bad value by returning the reason for it as a string, empty when the
|
|
// value was fine. That is what keeps this usable from a Bench constructor: throwing would be worse
|
|
// than the mistake it reports, and impossible for a consumer built with -fno-exceptions.
|
|
//
|
|
// The numbers are parsed by hand rather than through a stream or std::stoull. Both of those accept
|
|
// "-1" as a uint64_t - the standard has them read a signed value and negate it - which would turn a
|
|
// typo into 18446744073709551615 epochs. Doing it here also keeps the decimal point out of the
|
|
// hands of the global locale, and lets a duration be scaled by an exact power of ten instead of a
|
|
// double multiplication that turns 1.5s into 1499999999ns.
|
|
|
|
// Everything before the unit of a duration, and every count: "1.5" gives digits 15 and one fraction
|
|
// digit. A sign, an exponent, a second dot and anything that does not fit are all rejected - these
|
|
// are epoch times and iteration counts, not a general purpose number format.
|
|
static std::string parseDecimal(std::string const& str, uint64_t& digits, int& fractionDigits) {
|
|
digits = 0;
|
|
fractionDigits = 0;
|
|
bool seenDot = false;
|
|
bool seenDigit = false;
|
|
for (auto ch : str) {
|
|
if ('.' == ch) {
|
|
if (seenDot) {
|
|
return "is not a number";
|
|
}
|
|
seenDot = true;
|
|
continue;
|
|
}
|
|
if (ch < '0' || ch > '9') {
|
|
return "is not a number";
|
|
}
|
|
auto const digit = static_cast<uint64_t>(ch - '0');
|
|
if (digits > (std::numeric_limits<uint64_t>::max() - digit) / 10U) {
|
|
return "is too large";
|
|
}
|
|
digits = digits * 10U + digit;
|
|
seenDigit = true;
|
|
if (seenDot) {
|
|
++fractionDigits;
|
|
}
|
|
}
|
|
return seenDigit ? std::string() : std::string("is not a number");
|
|
}
|
|
|
|
// Multiplies by ten `exponent` times, or divides that often when it is negative. Only the last
|
|
// division rounds, so that the digits thrown away before it cannot round up twice.
|
|
static bool scaleByPowerOfTen(uint64_t& value, int exponent) {
|
|
while (exponent > 0) {
|
|
if (value > std::numeric_limits<uint64_t>::max() / 10U) {
|
|
return false;
|
|
}
|
|
value *= 10U;
|
|
--exponent;
|
|
}
|
|
while (exponent < 0) {
|
|
auto const remainder = value % 10U;
|
|
value /= 10U;
|
|
if (-1 == exponent && remainder >= 5U) {
|
|
++value;
|
|
}
|
|
++exponent;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// The trailing ns/us/ms/s of a duration. exponent is then the power of ten that turns that unit into
|
|
// nanoseconds, and numberLen how much of value is left in front of it.
|
|
//
|
|
// Every unit is an "s" with an optional metric prefix, which is worth saying that way rather than as
|
|
// a list of four suffixes to try: a list has to put the two character units first, since every one
|
|
// of them also ends in a match for "s", and nothing about the list says so.
|
|
static bool splitTimeUnit(std::string const& value, int& exponent, size_t& numberLen) {
|
|
if (value.empty() || 's' != value[value.size() - 1]) {
|
|
return false;
|
|
}
|
|
numberLen = value.size() - 1;
|
|
exponent = 9;
|
|
if (numberLen > 0) {
|
|
auto const prefix = value[numberLen - 1];
|
|
if ('n' == prefix) {
|
|
exponent = 0;
|
|
--numberLen;
|
|
} else if ('u' == prefix) {
|
|
exponent = 3;
|
|
--numberLen;
|
|
} else if ('m' == prefix) {
|
|
exponent = 6;
|
|
--numberLen;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// A whole number of iterations or epochs.
|
|
static std::string parseCount(std::string const& value, uint64_t& out) {
|
|
int fractionDigits = 0;
|
|
std::string reason = parseDecimal(value, out, fractionDigits);
|
|
if (reason.empty() && fractionDigits > 0) {
|
|
reason = "is not a whole number";
|
|
}
|
|
return reason;
|
|
}
|
|
|
|
// A number with a required unit, e.g. "1.5s". The unit is required because a bare number reads as
|
|
// whatever the writer had in mind and would be taken as nanoseconds, so "minEpochTime=5" would
|
|
// quietly become a 5ns floor - which is the trap that made one variable per setting, in nanoseconds,
|
|
// the wrong shape for this.
|
|
static std::string parseDuration(std::string const& value, std::chrono::nanoseconds& out) {
|
|
int unitExponent = 0;
|
|
size_t numberLen = 0;
|
|
if (!splitTimeUnit(value, unitExponent, numberLen)) {
|
|
return "is missing a time unit - use ns, us, ms or s";
|
|
}
|
|
|
|
// Nested, rather than the flatter chain of early returns this wants to be. `return reason;` is
|
|
// only elided when it is the *only* return of a named local, and clang's -Wnrvo is an error
|
|
// here; the early return above is fine because it returns a temporary built from a literal.
|
|
uint64_t digits = 0;
|
|
int fractionDigits = 0;
|
|
std::string reason = parseDecimal(value.substr(0, numberLen), digits, fractionDigits);
|
|
if (reason.empty()) {
|
|
using Rep = std::chrono::nanoseconds::rep;
|
|
if (scaleByPowerOfTen(digits, unitExponent - fractionDigits) &&
|
|
digits <= static_cast<uint64_t>(std::numeric_limits<Rep>::max())) {
|
|
out = std::chrono::nanoseconds(static_cast<Rep>(digits));
|
|
} else {
|
|
reason = "is too large";
|
|
}
|
|
}
|
|
return reason;
|
|
}
|
|
|
|
// One key, applied by calling the Bench setter it is named after. Going through the setter is what
|
|
// "the keys are the Bench setter names" has to mean to be worth writing down: minEpochIterations
|
|
// turns 0 into 1, and a setter that grows a rule later gets it for a value from the environment too.
|
|
//
|
|
// Arg is spelled at the call site rather than deduced. Every one of these names a setter *and* a
|
|
// getter, and naming the parameter type outright makes picking between them ordinary overload
|
|
// resolution rather than deduction against an overload set.
|
|
template <typename Arg>
|
|
static bool setCount(Bench& bench, std::string const& key, char const* name, Bench& (Bench::*setter)(Arg), std::string const& value,
|
|
std::string& reason) {
|
|
if (key != name) {
|
|
return false;
|
|
}
|
|
uint64_t parsed = 0;
|
|
reason = parseCount(value, parsed);
|
|
// size_t is 32 bits on a third of the matrix, and a count that does not survive the trip would
|
|
// otherwise run a quietly different benchmark than the one that was asked for
|
|
if (reason.empty() && static_cast<uint64_t>(static_cast<Arg>(parsed)) != parsed) {
|
|
reason = "is too large";
|
|
}
|
|
if (reason.empty()) {
|
|
(bench.*setter)(static_cast<Arg>(parsed));
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static bool setDuration(Bench& bench, std::string const& key, char const* name, Bench& (Bench::*setter)(std::chrono::nanoseconds),
|
|
std::string const& value, std::string& reason) {
|
|
if (key != name) {
|
|
return false;
|
|
}
|
|
std::chrono::nanoseconds parsed{};
|
|
reason = parseDuration(value, parsed);
|
|
if (reason.empty()) {
|
|
(bench.*setter)(parsed);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// Everything NANOBENCH_CONFIG understands, each key named exactly once. '||' short-circuits, so at
|
|
// most one of these runs and nothing past the match is even evaluated.
|
|
//
|
|
// This is the shape generateConfigTag() uses for the same seven names in the other direction, and
|
|
// for the reason its comment gives: as a stanza per key, the one mistake the compiler cannot see is
|
|
// a key you classified and then forgot to assign, which falls through to whichever branch was last.
|
|
static bool applyKnownKey(Bench& bench, std::string const& key, std::string const& value, std::string& reason) {
|
|
return setDuration(bench, key, "minEpochTime", &Bench::minEpochTime, value, reason) ||
|
|
setDuration(bench, key, "maxEpochTime", &Bench::maxEpochTime, value, reason) ||
|
|
setCount<size_t>(bench, key, "epochs", &Bench::epochs, value, reason) ||
|
|
setCount<uint64_t>(bench, key, "warmup", &Bench::warmup, value, reason) ||
|
|
setCount<uint64_t>(bench, key, "minEpochIterations", &Bench::minEpochIterations, value, reason) ||
|
|
setCount<uint64_t>(bench, key, "epochIterations", &Bench::epochIterations, value, reason) ||
|
|
setCount<size_t>(bench, key, "clockResolutionMultiple", &Bench::clockResolutionMultiple, value, reason);
|
|
}
|
|
|
|
// Sorted, and next to the chain above so that a key added to one and not the other is visible on one
|
|
// screen. Only tuning knobs are there at all: name, title, unit, batch and timeUnit say what a
|
|
// benchmark *is*, and letting a shell variable change those for a whole process would change what
|
|
// the numbers mean rather than how long they take.
|
|
static char const* configKeyList() {
|
|
return "clockResolutionMultiple, epochIterations, epochs, maxEpochTime, minEpochIterations, minEpochTime, warmup";
|
|
}
|
|
|
|
static void applyConfigEntry(Bench& bench, std::string const& key, std::string const& value, std::vector<std::string>& errors) {
|
|
std::string reason;
|
|
if (!applyKnownKey(bench, key, value, reason)) {
|
|
errors.push_back("NANOBENCH_CONFIG: unknown key '" + key + "' - valid keys are " + configKeyList());
|
|
} else if (!reason.empty()) {
|
|
errors.push_back("NANOBENCH_CONFIG: '" + key + "=" + value + "' " + reason);
|
|
}
|
|
}
|
|
|
|
// Whitespace around a key or a value is somebody laying the variable out to be read, not part of
|
|
// what they typed.
|
|
static std::string trimWhitespace(std::string const& str) {
|
|
char const* const spaces = " \t\n\r\f\v";
|
|
auto const first = str.find_first_not_of(spaces);
|
|
if (std::string::npos == first) {
|
|
return {};
|
|
}
|
|
return str.substr(first, str.find_last_not_of(spaces) - first + 1);
|
|
}
|
|
|
|
void applyConfigString(Bench& bench, std::string const& configStr, std::vector<std::string>& errors) {
|
|
size_t pos = 0;
|
|
while (pos < configStr.size()) {
|
|
auto const comma = configStr.find(',', pos);
|
|
auto const end = std::string::npos == comma ? configStr.size() : comma;
|
|
auto const entry = trimWhitespace(configStr.substr(pos, end - pos));
|
|
pos = end + 1;
|
|
|
|
// an empty variable, or a trailing comma, is nothing to complain about
|
|
if (entry.empty()) {
|
|
continue;
|
|
}
|
|
auto const equals = entry.find('=');
|
|
if (std::string::npos == equals) {
|
|
errors.push_back("NANOBENCH_CONFIG: '" + entry + "' is not a key=value pair");
|
|
continue;
|
|
}
|
|
applyConfigEntry(bench, trimWhitespace(entry.substr(0, equals)), trimWhitespace(entry.substr(equals + 1)), errors);
|
|
}
|
|
}
|
|
|
|
// Applies NANOBENCH_CONFIG to a freshly built Bench, so that anything the benchmark sets afterwards
|
|
// still wins: the environment moves the default, it does not overrule a benchmark that needs a
|
|
// particular setting to mean anything.
|
|
void applyEnvConfig(Bench& bench) {
|
|
auto const* const configStr = getEnv("NANOBENCH_CONFIG");
|
|
if (nullptr == configStr) {
|
|
return;
|
|
}
|
|
|
|
std::vector<std::string> errors;
|
|
applyConfigString(bench, configStr, errors);
|
|
|
|
// Every Bench parses the variable, but a typo in it is one mistake and gets one message. This is
|
|
// deliberately not behind NANOBENCH_SUPPRESS_WARNINGS: that hides statements about the machine
|
|
// being unsuitable, whereas this is a mistake in something the user typed moments ago, and
|
|
// swallowing it is how somebody ends up believing a sweep ran when it did not.
|
|
static bool isFirstBench = true;
|
|
if (isFirstBench) {
|
|
isFirstBench = false;
|
|
for (auto const& error : errors) {
|
|
std::cerr << error << std::endl;
|
|
}
|
|
}
|
|
}
|
|
|
|
void gatherStabilityInformation(std::vector<std::string>& warnings, std::vector<std::string>& recommendations) {
|
|
warnings.clear();
|
|
recommendations.clear();
|
|
|
|
# if defined(DEBUG)
|
|
warnings.emplace_back("DEBUG defined");
|
|
bool const recommendCheckFlags = true;
|
|
# else
|
|
bool const recommendCheckFlags = false;
|
|
# endif
|
|
|
|
bool recommendPyPerf = false;
|
|
# if defined(__linux__)
|
|
auto nprocs = sysconf(_SC_NPROCESSORS_CONF);
|
|
if (nprocs <= 0) {
|
|
warnings.emplace_back("couldn't figure out number of processors - no governor, turbo check possible");
|
|
} else {
|
|
// check frequency scaling
|
|
for (long id = 0; id < nprocs; ++id) {
|
|
auto idStr = detail::fmt::to_s(static_cast<uint64_t>(id));
|
|
auto sysCpu = "/sys/devices/system/cpu/cpu" + idStr;
|
|
auto minFreq = parseFile<int64_t>(sysCpu + "/cpufreq/scaling_min_freq", nullptr);
|
|
auto maxFreq = parseFile<int64_t>(sysCpu + "/cpufreq/scaling_max_freq", nullptr);
|
|
if (minFreq != maxFreq) {
|
|
auto minMHz = d(minFreq) / 1000.0;
|
|
auto maxMHz = d(maxFreq) / 1000.0;
|
|
warnings.emplace_back("CPU frequency scaling enabled: CPU " + idStr + " between " +
|
|
detail::fmt::Number(1, 1, minMHz).to_s() + " and " + detail::fmt::Number(1, 1, maxMHz).to_s() +
|
|
" MHz");
|
|
recommendPyPerf = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
auto fail = false;
|
|
auto currentGovernor = parseFile<std::string>("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor", &fail);
|
|
if (!fail && "performance" != currentGovernor) {
|
|
warnings.emplace_back("CPU governor is '" + currentGovernor + "' but should be 'performance'");
|
|
recommendPyPerf = true;
|
|
}
|
|
|
|
auto noTurbo = parseFile<int>("/sys/devices/system/cpu/intel_pstate/no_turbo", &fail);
|
|
if (!fail && noTurbo == 0) {
|
|
warnings.emplace_back("Turbo is enabled, CPU frequency will fluctuate");
|
|
recommendPyPerf = true;
|
|
}
|
|
}
|
|
# endif
|
|
|
|
if (recommendCheckFlags) {
|
|
recommendations.emplace_back("Make sure you compile for Release");
|
|
}
|
|
if (recommendPyPerf) {
|
|
recommendations.emplace_back("Use 'pyperf system tune' before benchmarking. See https://github.com/psf/pyperf");
|
|
}
|
|
}
|
|
|
|
void printStabilityInformationOnce(std::ostream* outStream) {
|
|
static bool shouldPrint = true;
|
|
if (shouldPrint && (nullptr != outStream) && isWarningsEnabled()) {
|
|
auto& os = *outStream;
|
|
shouldPrint = false;
|
|
std::vector<std::string> warnings;
|
|
std::vector<std::string> recommendations;
|
|
gatherStabilityInformation(warnings, recommendations);
|
|
if (warnings.empty()) {
|
|
return;
|
|
}
|
|
|
|
os << "Warning, results might be unstable:" << std::endl;
|
|
for (auto const& w : warnings) {
|
|
os << "* " << w << std::endl;
|
|
}
|
|
|
|
os << std::endl << "Recommendations" << std::endl;
|
|
for (auto const& r : recommendations) {
|
|
os << "* " << r << std::endl;
|
|
}
|
|
}
|
|
}
|
|
|
|
// When perf_event_open is refused, the table simply comes out five columns narrower. That reads like a
|
|
// nanobench bug rather than a setting of the machine, and people have gone looking for it more than once
|
|
// (issues #106, #123). So say what happened - but only where those columns could have appeared at all:
|
|
// on a platform that has no perf events in the first place nothing is wrong, and a note on every single
|
|
// run would be pure noise.
|
|
void printPerformanceCounterHintOnce(std::ostream* outStream, bool wantsPerformanceCounters) {
|
|
# if ANKERL_NANOBENCH(PERF_COUNTERS)
|
|
static bool shouldPrint = true;
|
|
if (!shouldPrint || !wantsPerformanceCounters || (nullptr == outStream) || !isWarningsEnabled()) {
|
|
return;
|
|
}
|
|
shouldPrint = false;
|
|
|
|
auto const& has = performanceCounters().has();
|
|
if (has.instructions || has.cpuCycles || has.branchInstructions || has.branchMisses) {
|
|
return;
|
|
}
|
|
|
|
*outStream << "Note: perf_event_open failed, so the ins/op, cyc/op, IPC, bra/op and miss% columns are missing." << std::endl
|
|
<< "This is usually a container or VM without virtualized performance counters, or" << std::endl
|
|
<< "/proc/sys/kernel/perf_event_paranoid being too restrictive." << std::endl
|
|
<< std::endl;
|
|
# else
|
|
(void)outStream;
|
|
(void)wantsPerformanceCounters;
|
|
# endif
|
|
}
|
|
|
|
// Which index of the pword/iword array every stream carries is ours. Allocated once per process.
|
|
static int streamHeaderHashIndex() {
|
|
static int const index = std::ios_base::xalloc();
|
|
return index;
|
|
}
|
|
|
|
// pword holds a raw void*, so what it points at has to be freed when the stream goes away.
|
|
static void streamHeaderHashCallback(std::ios_base::event ev, std::ios_base& ios, int index) {
|
|
if (std::ios_base::erase_event == ev) {
|
|
// NOLINTNEXTLINE(cppcoreguidelines-owning-memory)
|
|
delete static_cast<uint64_t*>(ios.pword(index));
|
|
ios.pword(index) = nullptr;
|
|
} else if (std::ios_base::copyfmt_event == ev) {
|
|
// copyfmt just copied our pointer into this stream; only one of the two may own it
|
|
ios.pword(index) = nullptr;
|
|
}
|
|
}
|
|
|
|
// This used to be a single process-wide value, so two Bench objects writing to different streams
|
|
// overwrote each other's settings and the second table came out with no header at all (issue #112).
|
|
//
|
|
// The obvious fix - a static map keyed on the ostream* - grows for the lifetime of the process and,
|
|
// worse, hands a freshly constructed stream the state of a destroyed one that happened to be
|
|
// allocated at the same address, which shows up as a header going missing at random. iostreams
|
|
// already provide per-stream storage whose lifetime is exactly the stream's, which is what this
|
|
// wants.
|
|
uint64_t& streamHeaderHash(std::ostream& os) {
|
|
auto const index = streamHeaderHashIndex();
|
|
void*& slot = os.pword(index);
|
|
if (nullptr == slot) {
|
|
// iword is a separate array at the same index; it records that the callback is registered so
|
|
// a stream never registers it twice. copyfmt copies iword and the callbacks together, so a
|
|
// stream copied from another is already covered.
|
|
if (0 == os.iword(index)) {
|
|
os.iword(index) = 1;
|
|
os.register_callback(streamHeaderHashCallback, index);
|
|
}
|
|
// NOLINTNEXTLINE(cppcoreguidelines-owning-memory)
|
|
slot = new uint64_t{};
|
|
}
|
|
return *static_cast<uint64_t*>(slot);
|
|
}
|
|
|
|
ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined")
|
|
inline uint64_t hash_combine(uint64_t seed, uint64_t val) {
|
|
return seed ^ (val + UINT64_C(0x9e3779b9) + (seed << 6U) + (seed >> 2U));
|
|
}
|
|
|
|
// determines resolution of the given clock. This is done by measuring multiple times and returning the minimum time difference.
|
|
Clock::duration calcClockResolution(size_t numEvaluations) noexcept {
|
|
auto bestDuration = Clock::duration::max();
|
|
Clock::time_point tBegin;
|
|
Clock::time_point tEnd;
|
|
for (size_t i = 0; i < numEvaluations; ++i) {
|
|
tBegin = Clock::now();
|
|
do {
|
|
tEnd = Clock::now();
|
|
} while (tBegin == tEnd);
|
|
bestDuration = (std::min)(bestDuration, tEnd - tBegin);
|
|
}
|
|
return bestDuration;
|
|
}
|
|
|
|
// Calculates clock resolution once, and remembers the result
|
|
Clock::duration clockResolution() noexcept {
|
|
static Clock::duration const sResolution = calcClockResolution(20);
|
|
return sResolution;
|
|
}
|
|
|
|
// Appends a column unless hideColumn() switched it off.
|
|
//
|
|
// Whether a column *can* be shown - is the counter available at all - and whether the caller wants it
|
|
// are different questions, and interleaving them at every site made it easy to answer one where the
|
|
// other was meant. The availability checks stay next to the values; the visibility policy is here,
|
|
// for every builder below, so a change to how a column is gated is one edit rather than three.
|
|
static void addColumn(std::vector<fmt::MarkDownColumn>& columns, Config const& config, Column column, int width, int precision,
|
|
std::string title, std::string suffix, double value) {
|
|
if (isColumnVisible(config, column)) {
|
|
columns.emplace_back(width, precision, std::move(title), std::move(suffix), value);
|
|
}
|
|
}
|
|
|
|
// The ratio-to-baseline column both table writers put in front of the measurements. Its geometry is
|
|
// here rather than at the two call sites: a header written at one width above rows written at
|
|
// another lines up nowhere.
|
|
static fmt::MarkDownColumn relativeColumn(double percent) {
|
|
return {11, 1, "relative", "%", percent};
|
|
}
|
|
|
|
// The performance counter columns of a row, empty when the counters are switched off or the kernel
|
|
// refused them. Split out of measurementColumns() below only to keep both of them under clang-tidy's
|
|
// cognitive complexity limit - the two are read as one list.
|
|
static std::vector<fmt::MarkDownColumn> counterColumns(Config const& config, Result const& result) {
|
|
std::vector<fmt::MarkDownColumn> columns;
|
|
if (!config.mShowPerformanceCounters) {
|
|
return columns;
|
|
}
|
|
|
|
auto const hasIns = result.has(Result::Measure::instructions);
|
|
auto const hasCyc = result.has(Result::Measure::cpucycles);
|
|
double const rInsMedian = hasIns ? result.median(Result::Measure::instructions) : 0.0;
|
|
double const rCycMedian = hasCyc ? result.median(Result::Measure::cpucycles) : 0.0;
|
|
|
|
if (hasIns) {
|
|
addColumn(columns, config, Column::instructions, 18, 2, "ins/" + config.mUnit, "", rInsMedian / config.mBatch);
|
|
}
|
|
if (hasCyc) {
|
|
addColumn(columns, config, Column::cycles, 18, 2, "cyc/" + config.mUnit, "", rCycMedian / config.mBatch);
|
|
}
|
|
|
|
// Which columns a row has must depend only on what the machine can measure, never on what it
|
|
// measured: keying IPC off the values instead meant a row whose op reported zero instructions
|
|
// came out one cell short of the header it sits under, and the rest of the table with it.
|
|
if (hasIns && hasCyc) {
|
|
addColumn(columns, config, Column::ipc, 9, 3, "IPC", "", rCycMedian <= 0.0 ? 0.0 : rInsMedian / rCycMedian);
|
|
}
|
|
|
|
if (result.has(Result::Measure::branchinstructions)) {
|
|
double const rBraMedian = result.median(Result::Measure::branchinstructions);
|
|
addColumn(columns, config, Column::branches, 17, 2, "bra/" + config.mUnit, "", rBraMedian / config.mBatch);
|
|
if (result.has(Result::Measure::branchmisses)) {
|
|
double const misses = result.median(Result::Measure::branchmisses);
|
|
addColumn(columns, config, Column::branchMisses, 10, 1, "miss%", "%",
|
|
rBraMedian < 1e-9 ? 0.0 : 100.0 * misses / rBraMedian);
|
|
}
|
|
}
|
|
return columns;
|
|
}
|
|
|
|
// The columns that describe what was measured, as opposed to how one row relates to another. Both
|
|
// table writers are built out of these: IterationLogic prepends `relative`, a comparison prepends
|
|
// `relative` and `95% CI`, and everything from here on is the same table in both.
|
|
//
|
|
// It is one function because it used to be two copies, and the copies drifted exactly the way copies
|
|
// do: the comparison table silently lost the performance counters it collects anyway and the context
|
|
// columns, and hideColumn() went inert there until it was patched separately. Adding a column here
|
|
// adds it to both.
|
|
//
|
|
// Takes the Config rather than the Bench because a comparison has no Bench to ask - every Result
|
|
// carries a copy of the config it was measured under, which is the same object either way.
|
|
static std::vector<fmt::MarkDownColumn> measurementColumns(Config const& config, Result const& result) {
|
|
std::vector<fmt::MarkDownColumn> columns;
|
|
|
|
auto const median = result.median(Result::Measure::elapsed);
|
|
|
|
if (config.mComplexityN > 0.0) {
|
|
addColumn(columns, config, Column::complexityN, 14, 0, "complexityN", "", config.mComplexityN);
|
|
}
|
|
|
|
// context columns come before the measurements: they say which benchmark this row is, and that
|
|
// reads better on the left. A row without the variable gets a blank cell rather than a missing
|
|
// column, so the table stays rectangular.
|
|
for (auto const& variableName : config.mContextColumns) {
|
|
auto it = config.mContext.find(variableName);
|
|
auto const width = static_cast<int>((std::max)(variableName.size() + 3, static_cast<size_t>(11)));
|
|
columns.emplace_back(width, variableName, it == config.mContext.end() ? std::string() : it->second);
|
|
}
|
|
|
|
addColumn(columns, config, Column::timePerUnit, 22, 2, config.mTimeUnitName + "/" + config.mUnit, "",
|
|
median / (config.mTimeUnit.count() * config.mBatch));
|
|
addColumn(columns, config, Column::unitPerSecond, 22, 2, config.mUnit + "/s", "", median <= 0.0 ? 0.0 : config.mBatch / median);
|
|
addColumn(columns, config, Column::error, 10, 1, "err%", "%", result.medianAbsolutePercentError(Result::Measure::elapsed) * 100.0);
|
|
|
|
auto const counters = counterColumns(config, result);
|
|
columns.insert(columns.end(), counters.begin(), counters.end());
|
|
|
|
addColumn(columns, config, Column::total, 12, 2, "total", "",
|
|
result.sumProduct(Result::Measure::iterations, Result::Measure::elapsed));
|
|
return columns;
|
|
}
|
|
|
|
// The markdown table's header and separator lines.
|
|
//
|
|
// Both table writers go through this, because the table is not rectangular in the obvious sense: the
|
|
// last cell holds the table's title on the header line and a benchmark's name on a data line, and the
|
|
// separator's last cell is one dash per title character, so it is one character wider than the header
|
|
// cell above it. What lines up - and what the tests assert on - is the offset of the last '|'. Two
|
|
// spellings of that convention agree only until one of them changes.
|
|
static void writeTableHeaderLines(std::ostream& os, std::vector<fmt::MarkDownColumn> const& columns, std::string const& title) {
|
|
os << std::endl;
|
|
for (auto const& col : columns) {
|
|
os << col.title();
|
|
}
|
|
os << "| " << title << std::endl;
|
|
|
|
for (auto const& col : columns) {
|
|
os << col.separator();
|
|
}
|
|
os << "|:" << std::string(title.size() + 1U, '-') << std::endl;
|
|
}
|
|
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_PUSH)
|
|
struct IterationLogic::Impl {
|
|
enum class State { warmup, upscaling_runtime, measuring, endless };
|
|
|
|
explicit Impl(Bench const& bench)
|
|
: mBench(bench)
|
|
, mResult(bench.config()) {
|
|
printStabilityInformationOnce(mBench.output());
|
|
printPerformanceCounterHintOnce(mBench.output(), mBench.performanceCounters());
|
|
|
|
// determine target runtime per epoch
|
|
mTargetRuntimePerEpoch = detail::targetRuntimePerEpoch(mBench);
|
|
|
|
if (isEndlessRunning(mBench.name())) {
|
|
std::cerr << "NANOBENCH_ENDLESS set: running '" << mBench.name() << "' endlessly" << std::endl;
|
|
mNumIters = (std::numeric_limits<uint64_t>::max)();
|
|
mState = State::endless;
|
|
} else if (0 != mBench.warmup()) {
|
|
mNumIters = mBench.warmup();
|
|
mState = State::warmup;
|
|
} else if (hasExactNumIters()) {
|
|
mNumIters = mBench.epochIterations();
|
|
mState = State::measuring;
|
|
} else {
|
|
mNumIters = mBench.minEpochIterations();
|
|
mState = State::upscaling_runtime;
|
|
}
|
|
}
|
|
|
|
// True when an exact number of iterations per epoch was requested, which overrides any calculated one.
|
|
ANKERL_NANOBENCH(NODISCARD) bool hasExactNumIters() const noexcept {
|
|
return 0 != mBench.epochIterations();
|
|
}
|
|
|
|
// The number of iterations the next epoch should run, honoring an exact count if one was requested.
|
|
ANKERL_NANOBENCH(NODISCARD) uint64_t calcNextNumIters(std::chrono::nanoseconds elapsed, uint64_t iters) noexcept {
|
|
return hasExactNumIters() ? mBench.epochIterations() : calcBestNumIters(elapsed, iters);
|
|
}
|
|
|
|
// directly calculates new iters based on elapsed&iters, and stretches the epoch by 0-20% of random noise so epochs
|
|
// don't get into lockstep with periodic disturbances. Makes sure we don't under- or overflow.
|
|
ANKERL_NANOBENCH(NODISCARD) uint64_t calcBestNumIters(std::chrono::nanoseconds elapsed, uint64_t iters) noexcept {
|
|
auto doubleElapsed = d(elapsed);
|
|
auto doubleTargetRuntimePerEpoch = d(mTargetRuntimePerEpoch);
|
|
auto doubleNewIters = doubleTargetRuntimePerEpoch / doubleElapsed * d(iters);
|
|
|
|
// An elapsed time of 0 (possible with a coarse clock) makes the division above produce infinity, or even NaN
|
|
// when the target runtime is 0 as well. Comparing with '!(a >= b)' instead of 'a < b' replaces NaN with the
|
|
// minimum too, so an epoch never ends up with a nonsensical number of iterations.
|
|
auto doubleMinEpochIters = d(mBench.minEpochIterations());
|
|
if (!(doubleNewIters >= doubleMinEpochIters)) {
|
|
doubleNewIters = doubleMinEpochIters;
|
|
}
|
|
doubleNewIters *= 1.0 + 0.2 * mRng.uniform01();
|
|
|
|
return u64(doubleNewIters);
|
|
}
|
|
|
|
ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") void upscale(std::chrono::nanoseconds elapsed) {
|
|
if (elapsed * 10 < mTargetRuntimePerEpoch) {
|
|
// we are far below the target runtime. Multiply iterations by 10 (with overflow check)
|
|
if (mNumIters > (std::numeric_limits<uint64_t>::max)() / 10) {
|
|
// overflow :-(
|
|
showResult("iterations overflow. Maybe your code got optimized away?");
|
|
mNumIters = 0;
|
|
return;
|
|
}
|
|
mNumIters *= 10;
|
|
} else {
|
|
mNumIters = calcBestNumIters(elapsed, mNumIters);
|
|
}
|
|
}
|
|
|
|
void add(std::chrono::nanoseconds elapsed, PerformanceCounters const& pc) noexcept {
|
|
# if defined(ANKERL_NANOBENCH_LOG_ENABLED)
|
|
auto oldIters = mNumIters;
|
|
# endif
|
|
|
|
switch (mState) {
|
|
case State::warmup:
|
|
// an exact number of iterations makes upscaling pointless, so warmup is over either way
|
|
if (hasExactNumIters() || isCloseEnoughForMeasurements(elapsed)) {
|
|
// if elapsed is close enough, we can skip upscaling and go right to measurements
|
|
// still, we don't add the result to the measurements.
|
|
mState = State::measuring;
|
|
mNumIters = calcNextNumIters(elapsed, mNumIters);
|
|
} else {
|
|
// not close enough: switch to upscaling
|
|
mState = State::upscaling_runtime;
|
|
upscale(elapsed);
|
|
}
|
|
break;
|
|
|
|
case State::upscaling_runtime:
|
|
if (isCloseEnoughForMeasurements(elapsed)) {
|
|
// if we are close enough, add measurement and switch to always measuring
|
|
mState = State::measuring;
|
|
mTotalElapsed += elapsed;
|
|
mTotalNumIters += mNumIters;
|
|
mResult.add(elapsed, mNumIters, pc);
|
|
mNumIters = calcNextNumIters(mTotalElapsed, mTotalNumIters);
|
|
} else {
|
|
upscale(elapsed);
|
|
}
|
|
break;
|
|
|
|
case State::measuring:
|
|
// just add measurements - no questions asked. Even when runtime is low. But we can't ignore
|
|
// that fluctuation, or else we would bias the result
|
|
mTotalElapsed += elapsed;
|
|
mTotalNumIters += mNumIters;
|
|
mResult.add(elapsed, mNumIters, pc);
|
|
mNumIters = calcNextNumIters(mTotalElapsed, mTotalNumIters);
|
|
break;
|
|
|
|
case State::endless:
|
|
mNumIters = (std::numeric_limits<uint64_t>::max)();
|
|
break;
|
|
}
|
|
|
|
if (static_cast<uint64_t>(mResult.size()) == mBench.epochs()) {
|
|
// we got all the results that we need, finish it
|
|
showResult("");
|
|
mNumIters = 0;
|
|
}
|
|
|
|
ANKERL_NANOBENCH_LOG(mBench.name() << ": " << detail::fmt::Number(20, 3, d(elapsed.count())) << " elapsed, "
|
|
<< detail::fmt::Number(20, 3, d(mTargetRuntimePerEpoch.count())) << " target. oldIters="
|
|
<< oldIters << ", mNumIters=" << mNumIters << ", mState=" << static_cast<int>(mState));
|
|
}
|
|
|
|
void showResult(std::string const& errorMessage) const {
|
|
ANKERL_NANOBENCH_LOG(errorMessage);
|
|
|
|
if (nullptr == mBench.output()) {
|
|
return;
|
|
}
|
|
|
|
// prepare column data ///////
|
|
std::vector<fmt::MarkDownColumn> columns;
|
|
|
|
if (mBench.relative()) {
|
|
double relativePercent = 100.0;
|
|
if (!mBench.results().empty()) {
|
|
// Every other column is per unit, so this one has to be as well. Comparing the raw epoch times would
|
|
// give a skewed percentage as soon as the baseline was run with a different batch size.
|
|
// See https://github.com/martinus/nanobench/issues/131
|
|
// This is (baseline / baselineBatch) / (rMedian / batch) as a single fraction, so one guard does.
|
|
auto const rMedian = mResult.median(Result::Measure::elapsed);
|
|
auto const& baseline = mBench.results().front();
|
|
auto const num = baseline.median(Result::Measure::elapsed) * mBench.batch();
|
|
auto const den = rMedian * baseline.config().mBatch;
|
|
relativePercent = den <= 0.0 ? 0.0 : num / den * 100.0;
|
|
}
|
|
if (isColumnVisible(mBench.config(), Column::relative)) {
|
|
columns.push_back(relativeColumn(relativePercent));
|
|
}
|
|
}
|
|
|
|
// everything a comparison table shows too
|
|
auto const measurements = measurementColumns(mResult.config(), mResult);
|
|
columns.insert(columns.end(), measurements.begin(), measurements.end());
|
|
|
|
double const rErrorMedian = mResult.medianAbsolutePercentError(Result::Measure::elapsed);
|
|
|
|
// write everything
|
|
auto& os = *mBench.output();
|
|
|
|
// The shape of the table *is* the columns it has, so that is what decides whether the header
|
|
// above the previous row still fits this one. Hashing a hand-kept list of the config fields
|
|
// that influence the columns meant every conditional column had a second place to be
|
|
// remembered, two hundred lines from the one that adds it - and complexityN, which
|
|
// measurementColumns() adds only when it is set, had already been forgotten there.
|
|
uint64_t hash = 0;
|
|
hash = hash_combine(std::hash<std::string>{}(mBench.title()), hash);
|
|
for (auto const& col : columns) {
|
|
hash = hash_combine(std::hash<std::string>{}(col.title()), hash);
|
|
}
|
|
|
|
auto& lastHeaderHash = streamHeaderHash(os);
|
|
if (hash != lastHeaderHash) {
|
|
lastHeaderHash = hash;
|
|
writeTableHeaderLines(os, columns, mBench.title());
|
|
}
|
|
|
|
if (!errorMessage.empty()) {
|
|
for (auto const& col : columns) {
|
|
os << col.invalid();
|
|
}
|
|
os << "| :boom: " << fmt::MarkDownCode(mBench.name()) << " (" << errorMessage << ')' << std::endl;
|
|
return;
|
|
}
|
|
|
|
for (auto const& col : columns) {
|
|
os << col.value();
|
|
}
|
|
os << "| ";
|
|
auto showUnstable = isWarningsEnabled() && rErrorMedian >= 0.05;
|
|
if (showUnstable) {
|
|
os << ":wavy_dash: ";
|
|
}
|
|
os << fmt::MarkDownCode(mBench.name());
|
|
if (showUnstable) {
|
|
auto avgIters = d(mTotalNumIters) / d(mBench.epochs());
|
|
auto suggestedIters = u64(avgIters * 10);
|
|
|
|
os << " (Unstable with ~" << detail::fmt::Number(1, 1, avgIters) << " iters. Increase `minEpochIterations` to e.g. "
|
|
<< suggestedIters << ")";
|
|
}
|
|
os << std::endl;
|
|
}
|
|
|
|
ANKERL_NANOBENCH(NODISCARD) bool isCloseEnoughForMeasurements(std::chrono::nanoseconds elapsed) const noexcept {
|
|
return elapsed * 3 >= mTargetRuntimePerEpoch * 2;
|
|
}
|
|
|
|
uint64_t mNumIters = 1; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
Bench const& mBench; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
std::chrono::nanoseconds mTargetRuntimePerEpoch{}; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
Result mResult; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
Rng mRng{123}; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
std::chrono::nanoseconds mTotalElapsed{}; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
uint64_t mTotalNumIters = 0; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
State mState = State::upscaling_runtime; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
};
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_POP)
|
|
|
|
IterationLogic::IterationLogic(Bench const& bench)
|
|
: mPimpl(new Impl(bench)) {}
|
|
|
|
IterationLogic::~IterationLogic() {
|
|
delete mPimpl;
|
|
}
|
|
|
|
uint64_t IterationLogic::numIters() const noexcept {
|
|
ANKERL_NANOBENCH_LOG(mPimpl->mBench.name() << ": mNumIters=" << mPimpl->mNumIters);
|
|
return mPimpl->mNumIters;
|
|
}
|
|
|
|
void IterationLogic::add(std::chrono::nanoseconds elapsed, PerformanceCounters const& pc) noexcept {
|
|
mPimpl->add(elapsed, pc);
|
|
}
|
|
|
|
void IterationLogic::moveResultTo(std::vector<Result>& results) noexcept {
|
|
results.emplace_back(std::move(mPimpl->mResult));
|
|
}
|
|
|
|
// The counter arithmetic, deliberately outside the PERF_COUNTERS guard - see the declarations.
|
|
|
|
uint64_t saturatingSub(uint64_t a, uint64_t b) noexcept {
|
|
return a >= b ? a - b : UINT64_C(0);
|
|
}
|
|
|
|
uint64_t divRounded(uint64_t a, uint64_t divisor) noexcept {
|
|
return (a + divisor / 2) / divisor;
|
|
}
|
|
|
|
size_t perfReadFormatSize(size_t numEvents) noexcept {
|
|
return 3U + numEvents * 2U;
|
|
}
|
|
|
|
size_t perfValueIndex(uint64_t i) noexcept {
|
|
return perfReadFormatSize(static_cast<size_t>(i));
|
|
}
|
|
|
|
uint64_t scaleMultiplexed(uint64_t value, uint64_t timeEnabled, uint64_t timeRunning) noexcept {
|
|
if (0U == timeRunning || timeEnabled <= timeRunning) {
|
|
// the event was active the whole time, nothing to extrapolate
|
|
return value;
|
|
}
|
|
// counts stay far below 2^53, so double is exact enough here and cannot overflow the way the
|
|
// integer multiplication would
|
|
return u64(d(value) * (d(timeEnabled) / d(timeRunning)));
|
|
}
|
|
|
|
ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined")
|
|
uint64_t correctOverhead(uint64_t value, uint64_t measuringOverhead, uint64_t loopOverheadPerIter, uint64_t numIters) noexcept {
|
|
return saturatingSub(saturatingSub(value, measuringOverhead), loopOverheadPerIter * numIters);
|
|
}
|
|
|
|
ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined")
|
|
uint64_t loopOverheadPerIteration(uint64_t single, uint64_t doubled, uint64_t measuringOverhead, uint64_t numIters) noexcept {
|
|
auto const m1 = saturatingSub(single, measuringOverhead);
|
|
auto const m2 = saturatingSub(doubled, measuringOverhead);
|
|
// the second run does the work twice, so twice the first run minus the second is what the loop
|
|
// costs on its own
|
|
return divRounded(saturatingSub(m1 * 2, m2), numIters);
|
|
}
|
|
|
|
ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined")
|
|
uint64_t correctBranchInstructions(uint64_t rawBranchInstructions, uint64_t numIters) noexcept {
|
|
// one branch per iteration for the loop, plus the one that ends it
|
|
return saturatingSub(rawBranchInstructions, numIters + 1U);
|
|
}
|
|
|
|
size_t roundsForTargetWidth(size_t pilotRounds, double observedWidth, double targetWidth, size_t numOps, size_t maxRounds) noexcept {
|
|
auto rounds = pilotRounds;
|
|
if (targetWidth > 0.0 && observedWidth > targetWidth) {
|
|
auto const factor = (observedWidth / targetWidth) * (observedWidth / targetWidth);
|
|
// d() is only defined further down; this stays in double throughout and clamps before the
|
|
// cast back, so a pilot that measured almost nothing cannot overflow the count.
|
|
auto const wanted = static_cast<double>(pilotRounds) * factor;
|
|
auto const capped = wanted > static_cast<double>(maxRounds) ? static_cast<double>(maxRounds) : wanted;
|
|
rounds = static_cast<size_t>(capped);
|
|
if (rounds < pilotRounds) {
|
|
rounds = pilotRounds;
|
|
}
|
|
}
|
|
if (rounds > maxRounds) {
|
|
rounds = maxRounds;
|
|
}
|
|
// whole blocks, so every alternative is measured the same number of times
|
|
if (0U != numOps && 0U != rounds % numOps) {
|
|
rounds += numOps - (rounds % numOps);
|
|
}
|
|
return rounds;
|
|
}
|
|
|
|
double correctBranchMisses(uint64_t rawBranchMisses, double correctedBranchInstructions) noexcept {
|
|
auto branchMisses = d(rawBranchMisses);
|
|
if (branchMisses > correctedBranchInstructions) {
|
|
// can't have more branch misses than there were branches
|
|
branchMisses = correctedBranchInstructions;
|
|
}
|
|
|
|
// assuming at least one missed branch for the loop
|
|
branchMisses -= 1.0;
|
|
if (branchMisses < 1.0) {
|
|
branchMisses = 1.0;
|
|
}
|
|
return branchMisses;
|
|
}
|
|
|
|
# if ANKERL_NANOBENCH(PERF_COUNTERS)
|
|
|
|
// glibc declares ioctl()'s request parameter as unsigned long, musl as int. PERF_EVENT_IOC_ID and
|
|
// friends carry the direction bits in the high end, so they don't fit into an int and passing one
|
|
// straight through is a value-changing conversion that -Werror rejects on musl (issue #92). Deduce
|
|
// the declared parameter type from ioctl() itself, so the same cast is right for either libc.
|
|
# if defined(__BIONIC__)
|
|
// Except on bionic, which declares ioctl() *twice* - once taking int and once taking unsigned - for
|
|
// exactly the reason above, so that either signedness compiles without a cast. Its header says out
|
|
// loud that the overload breaks anyone taking ioctl()'s address, and points at naming a concrete
|
|
// type instead: &::ioctl is an overload set, and nothing can be deduced from one. That deduction
|
|
// failure is the whole of issue microsoft/vcpkg#53422 - nanobench built for no Android ABI at all.
|
|
// unsigned is the overload to name, since it is the one PERF_EVENT_IOC_ID fits into.
|
|
using IoctlRequest = unsigned;
|
|
# else
|
|
// Never defined - only ever asked for its return type.
|
|
template <typename Ret, typename Fd, typename Request>
|
|
Request ioctlRequestType(Ret (*)(Fd, Request, ...));
|
|
|
|
# if defined(__cpp_noexcept_function_type)
|
|
// Since C++17 noexcept is part of a function's type, and glibc declares ioctl() with __THROW - so
|
|
// without this second overload the deduction above stops matching at -std=c++17.
|
|
template <typename Ret, typename Fd, typename Request>
|
|
Request ioctlRequestType(Ret (*)(Fd, Request, ...) noexcept);
|
|
# endif
|
|
|
|
using IoctlRequest = decltype(ioctlRequestType(&::ioctl));
|
|
# endif
|
|
|
|
template <typename Arg>
|
|
int perfIoctl(int fd, unsigned long request, Arg arg) {
|
|
// NOLINTNEXTLINE(hicpp-signed-bitwise,cppcoreguidelines-pro-type-vararg)
|
|
return ioctl(fd, static_cast<IoctlRequest>(request), arg);
|
|
}
|
|
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_PUSH)
|
|
class LinuxPerformanceCounters {
|
|
public:
|
|
struct Target {
|
|
Target(uint64_t* targetValue_, bool correctMeasuringOverhead_, bool correctLoopOverhead_)
|
|
: targetValue(targetValue_)
|
|
, correctMeasuringOverhead(correctMeasuringOverhead_)
|
|
, correctLoopOverhead(correctLoopOverhead_) {}
|
|
|
|
uint64_t* targetValue{}; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
bool correctMeasuringOverhead{}; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
bool correctLoopOverhead{}; // NOLINT(misc-non-private-member-variables-in-classes)
|
|
};
|
|
|
|
LinuxPerformanceCounters() = default;
|
|
LinuxPerformanceCounters(LinuxPerformanceCounters const&) = delete;
|
|
LinuxPerformanceCounters(LinuxPerformanceCounters&&) = delete;
|
|
LinuxPerformanceCounters& operator=(LinuxPerformanceCounters const&) = delete;
|
|
LinuxPerformanceCounters& operator=(LinuxPerformanceCounters&&) = delete;
|
|
~LinuxPerformanceCounters();
|
|
|
|
bool monitor(perf_sw_ids swId, Target target);
|
|
bool monitor(perf_hw_id hwId, Target target);
|
|
|
|
ANKERL_NANOBENCH(NODISCARD) bool hasError() const noexcept {
|
|
return mHasError;
|
|
}
|
|
|
|
// Just reading data is faster than enable & disabling.
|
|
// we subtract data ourselves.
|
|
inline void beginMeasure() {
|
|
if (mHasError) {
|
|
return;
|
|
}
|
|
|
|
mHasError = -1 == perfIoctl(mFd, PERF_EVENT_IOC_RESET, PERF_IOC_FLAG_GROUP);
|
|
if (mHasError) {
|
|
return;
|
|
}
|
|
|
|
mHasError = -1 == perfIoctl(mFd, PERF_EVENT_IOC_ENABLE, PERF_IOC_FLAG_GROUP);
|
|
}
|
|
|
|
inline void endMeasure() {
|
|
if (mHasError) {
|
|
return;
|
|
}
|
|
|
|
mHasError = (-1 == perfIoctl(mFd, PERF_EVENT_IOC_DISABLE, PERF_IOC_FLAG_GROUP));
|
|
if (mHasError) {
|
|
return;
|
|
}
|
|
|
|
auto const numBytes = sizeof(uint64_t) * mCounters.size();
|
|
auto ret = read(mFd, mCounters.data(), numBytes);
|
|
mHasError = ret != static_cast<ssize_t>(numBytes);
|
|
if (!mHasError) {
|
|
scaleCounters();
|
|
}
|
|
}
|
|
|
|
void updateResults(uint64_t numIters);
|
|
|
|
// Compensates the counters that were just read for multiplexing.
|
|
//
|
|
// When more events are monitored than the hardware can count at the same time, the kernel time-shares the counters
|
|
// between them. An event is then only active for a fraction of the measurement, and the value read back is just what
|
|
// happened while it was active. Scaling it up by enabled/running extrapolates to the whole measurement, which is
|
|
// exactly what `perf stat` does. Without this, cycles/instructions/branches are silently underreported - e.g. when
|
|
// the NMI watchdog occupies one of the counters, or in a VM that exposes a restricted PMU.
|
|
inline void scaleCounters() noexcept {
|
|
auto const enabled = timeEnabled() - mTotalTimeEnabledNanos;
|
|
auto const running = timeRunning() - mTotalTimeRunningNanos;
|
|
mTotalTimeEnabledNanos = timeEnabled();
|
|
mTotalTimeRunningNanos = timeRunning();
|
|
|
|
for (uint64_t i = 0; i < numEvents(); ++i) {
|
|
auto const idx = perfValueIndex(i);
|
|
mCounters[idx] = scaleMultiplexed(mCounters[idx], enabled, running);
|
|
}
|
|
}
|
|
|
|
ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined")
|
|
static inline uint32_t mix(uint32_t x) noexcept {
|
|
x ^= x << 13U;
|
|
x ^= x >> 17U;
|
|
x ^= x << 5U;
|
|
return x;
|
|
}
|
|
|
|
template <typename Op>
|
|
ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined")
|
|
void calibrate(Op&& op) {
|
|
// clear current calibration data, so a measurement that fails below subtracts nothing rather
|
|
// than something stale
|
|
for (auto& v : mCalibratedOverhead) {
|
|
v = UINT64_C(0);
|
|
}
|
|
|
|
// the new data is a minimum over 100 runs, so it starts at the maximum
|
|
std::vector<uint64_t> newCalibration(mCalibratedOverhead.size(), (std::numeric_limits<uint64_t>::max)());
|
|
for (size_t iter = 0; iter < 100; ++iter) {
|
|
beginMeasure();
|
|
op();
|
|
endMeasure();
|
|
if (mHasError) {
|
|
return;
|
|
}
|
|
|
|
for (size_t i = 0; i < newCalibration.size(); ++i) {
|
|
newCalibration[i] = (std::min)(newCalibration[i], mCounters[i]);
|
|
}
|
|
}
|
|
|
|
mCalibratedOverhead = std::move(newCalibration);
|
|
|
|
{
|
|
// calibrate loop overhead. For branches & instructions this makes sense, not so much for everything else like cycles.
|
|
// marsaglia's xorshift: mov, sal/shr, xor. Times 3.
|
|
// This has the nice property that the compiler doesn't seem to be able to optimize multiple calls any further.
|
|
// see https://godbolt.org/z/49RVQ5
|
|
uint64_t const numIters = 100000U + (std::random_device{}() & 3U);
|
|
uint64_t n = numIters;
|
|
uint32_t x = 1234567;
|
|
|
|
beginMeasure();
|
|
while (n-- > 0) {
|
|
x = mix(x);
|
|
}
|
|
endMeasure();
|
|
detail::doNotOptimizeAway(x);
|
|
auto measure1 = mCounters;
|
|
|
|
n = numIters;
|
|
beginMeasure();
|
|
while (n-- > 0) {
|
|
// we now run *twice* so we can easily calculate the overhead
|
|
x = mix(x);
|
|
x = mix(x);
|
|
}
|
|
endMeasure();
|
|
detail::doNotOptimizeAway(x);
|
|
auto measure2 = mCounters;
|
|
|
|
for (size_t i = 0; i < mCounters.size(); ++i) {
|
|
mLoopOverhead[i] = loopOverheadPerIteration(measure1[i], measure2[i], mCalibratedOverhead[i], numIters);
|
|
}
|
|
}
|
|
}
|
|
|
|
private:
|
|
bool monitor(uint32_t type, uint64_t eventid, Target target);
|
|
|
|
// The three header words of the last read, by name rather than by index.
|
|
ANKERL_NANOBENCH(NODISCARD) uint64_t numEvents() const noexcept {
|
|
return mCounters[0];
|
|
}
|
|
ANKERL_NANOBENCH(NODISCARD) uint64_t timeEnabled() const noexcept {
|
|
return mCounters[1];
|
|
}
|
|
ANKERL_NANOBENCH(NODISCARD) uint64_t timeRunning() const noexcept {
|
|
return mCounters[2];
|
|
}
|
|
|
|
std::map<uint64_t, Target> mIdToTarget{};
|
|
|
|
// no events monitored yet, so just the read_format header
|
|
std::vector<uint64_t> mCounters = std::vector<uint64_t>(perfReadFormatSize(0));
|
|
std::vector<uint64_t> mCalibratedOverhead = std::vector<uint64_t>(perfReadFormatSize(0));
|
|
std::vector<uint64_t> mLoopOverhead = std::vector<uint64_t>(perfReadFormatSize(0));
|
|
|
|
// PERF_EVENT_IOC_RESET resets the counters, but not time_enabled/time_running: those keep accumulating over the
|
|
// whole lifetime of the event. So the times of a single measurement are the difference to the previous read.
|
|
uint64_t mTotalTimeEnabledNanos = 0;
|
|
uint64_t mTotalTimeRunningNanos = 0;
|
|
int mFd = -1;
|
|
bool mHasError = false;
|
|
};
|
|
ANKERL_NANOBENCH(IGNORE_PADDED_POP)
|
|
|
|
LinuxPerformanceCounters::~LinuxPerformanceCounters() {
|
|
if (-1 != mFd) {
|
|
close(mFd);
|
|
}
|
|
}
|
|
|
|
bool LinuxPerformanceCounters::monitor(perf_sw_ids swId, LinuxPerformanceCounters::Target target) {
|
|
return monitor(PERF_TYPE_SOFTWARE, swId, target);
|
|
}
|
|
|
|
bool LinuxPerformanceCounters::monitor(perf_hw_id hwId, LinuxPerformanceCounters::Target target) {
|
|
return monitor(PERF_TYPE_HARDWARE, hwId, target);
|
|
}
|
|
|
|
// overflow is ok, it's checked
|
|
ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined")
|
|
void LinuxPerformanceCounters::updateResults(uint64_t numIters) {
|
|
// clear old data
|
|
for (auto& id_value : mIdToTarget) {
|
|
*id_value.second.targetValue = UINT64_C(0);
|
|
}
|
|
|
|
if (mHasError) {
|
|
return;
|
|
}
|
|
|
|
// mCounters has already been compensated for multiplexing in endMeasure(), and so has mCalibratedOverhead
|
|
for (uint64_t i = 0; i < numEvents(); ++i) {
|
|
auto idx = perfValueIndex(i);
|
|
auto id = mCounters[idx + 1U];
|
|
|
|
auto it = mIdToTarget.find(id);
|
|
if (it != mIdToTarget.end()) {
|
|
auto& tgt = it->second;
|
|
// a correction the target did not ask for is applied as a correction of zero
|
|
*tgt.targetValue = correctOverhead(mCounters[idx], tgt.correctMeasuringOverhead ? mCalibratedOverhead[idx] : UINT64_C(0),
|
|
tgt.correctLoopOverhead ? mLoopOverhead[idx] : UINT64_C(0), numIters);
|
|
}
|
|
}
|
|
}
|
|
|
|
bool LinuxPerformanceCounters::monitor(uint32_t type, uint64_t eventid, Target target) {
|
|
*target.targetValue = (std::numeric_limits<uint64_t>::max)();
|
|
if (mHasError) {
|
|
return false;
|
|
}
|
|
|
|
auto pea = perf_event_attr();
|
|
std::memset(&pea, 0, sizeof(perf_event_attr));
|
|
pea.type = type;
|
|
pea.size = sizeof(perf_event_attr);
|
|
pea.config = eventid;
|
|
pea.disabled = 1; // start counter as disabled
|
|
pea.exclude_kernel = 1;
|
|
pea.exclude_hv = 1;
|
|
|
|
// NOLINTNEXTLINE(hicpp-signed-bitwise)
|
|
pea.read_format = PERF_FORMAT_GROUP | PERF_FORMAT_ID | PERF_FORMAT_TOTAL_TIME_ENABLED | PERF_FORMAT_TOTAL_TIME_RUNNING;
|
|
|
|
const int pid = 0; // the current process
|
|
const int cpu = -1; // all CPUs
|
|
# if defined(PERF_FLAG_FD_CLOEXEC) // since Linux 3.14
|
|
const unsigned long flags = PERF_FLAG_FD_CLOEXEC;
|
|
# else
|
|
const unsigned long flags = 0;
|
|
# endif
|
|
|
|
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
|
|
auto fd = static_cast<int>(syscall(__NR_perf_event_open, &pea, pid, cpu, mFd, flags));
|
|
if (-1 == fd) {
|
|
return false;
|
|
}
|
|
if (-1 == mFd) {
|
|
// first call: set to fd, and use this from now on
|
|
mFd = fd;
|
|
}
|
|
uint64_t id = 0;
|
|
if (-1 == perfIoctl(fd, PERF_EVENT_IOC_ID, &id)) {
|
|
// couldn't get id
|
|
return false;
|
|
}
|
|
|
|
// insert into map, rely on the fact that map's references are constant.
|
|
mIdToTarget.emplace(id, target);
|
|
|
|
// prepare readformat with the correct size (after the insert)
|
|
auto size = perfReadFormatSize(mIdToTarget.size());
|
|
mCounters.resize(size);
|
|
mCalibratedOverhead.resize(size);
|
|
mLoopOverhead.resize(size);
|
|
|
|
return true;
|
|
}
|
|
|
|
PerformanceCounters::PerformanceCounters()
|
|
: mPc(new LinuxPerformanceCounters())
|
|
, mVal()
|
|
, mHas() {
|
|
|
|
// HW events
|
|
mHas.cpuCycles = mPc->monitor(PERF_COUNT_HW_REF_CPU_CYCLES, LinuxPerformanceCounters::Target(&mVal.cpuCycles, true, false));
|
|
if (!mHas.cpuCycles) {
|
|
// Fallback to cycles counter, reference cycles not available in many systems.
|
|
mHas.cpuCycles = mPc->monitor(PERF_COUNT_HW_CPU_CYCLES, LinuxPerformanceCounters::Target(&mVal.cpuCycles, true, false));
|
|
}
|
|
mHas.instructions = mPc->monitor(PERF_COUNT_HW_INSTRUCTIONS, LinuxPerformanceCounters::Target(&mVal.instructions, true, true));
|
|
mHas.branchInstructions =
|
|
mPc->monitor(PERF_COUNT_HW_BRANCH_INSTRUCTIONS, LinuxPerformanceCounters::Target(&mVal.branchInstructions, true, false));
|
|
mHas.branchMisses = mPc->monitor(PERF_COUNT_HW_BRANCH_MISSES, LinuxPerformanceCounters::Target(&mVal.branchMisses, true, false));
|
|
|
|
// SW events
|
|
mHas.pageFaults = mPc->monitor(PERF_COUNT_SW_PAGE_FAULTS, LinuxPerformanceCounters::Target(&mVal.pageFaults, true, false));
|
|
mHas.contextSwitches =
|
|
mPc->monitor(PERF_COUNT_SW_CONTEXT_SWITCHES, LinuxPerformanceCounters::Target(&mVal.contextSwitches, true, false));
|
|
|
|
mPc->calibrate([] {
|
|
auto before = ankerl::nanobench::Clock::now();
|
|
auto after = ankerl::nanobench::Clock::now();
|
|
(void)before;
|
|
(void)after;
|
|
});
|
|
|
|
if (mPc->hasError()) {
|
|
// something failed, don't monitor anything.
|
|
mHas = PerfCountSet<bool>{};
|
|
}
|
|
}
|
|
|
|
PerformanceCounters::~PerformanceCounters() {
|
|
// no need to check for nullptr, delete nullptr has no effect
|
|
delete mPc;
|
|
}
|
|
|
|
void PerformanceCounters::beginMeasure() {
|
|
mPc->beginMeasure();
|
|
}
|
|
|
|
void PerformanceCounters::endMeasure() {
|
|
mPc->endMeasure();
|
|
}
|
|
|
|
void PerformanceCounters::updateResults(uint64_t numIters) {
|
|
mPc->updateResults(numIters);
|
|
}
|
|
|
|
# else
|
|
|
|
PerformanceCounters::PerformanceCounters() = default;
|
|
PerformanceCounters::~PerformanceCounters() = default;
|
|
void PerformanceCounters::beginMeasure() {}
|
|
void PerformanceCounters::endMeasure() {}
|
|
void PerformanceCounters::updateResults(uint64_t) {}
|
|
|
|
# endif
|
|
|
|
ANKERL_NANOBENCH(NODISCARD) PerfCountSet<uint64_t> const& PerformanceCounters::val() const noexcept {
|
|
return mVal;
|
|
}
|
|
ANKERL_NANOBENCH(NODISCARD) PerfCountSet<bool> const& PerformanceCounters::has() const noexcept {
|
|
return mHas;
|
|
}
|
|
|
|
// formatting utilities
|
|
namespace fmt {
|
|
|
|
// RAII to save & restore a stream's state
|
|
StreamStateRestorer::StreamStateRestorer(std::ostream& s)
|
|
: mStream(s)
|
|
, mLocale(s.getloc())
|
|
, mPrecision(s.precision())
|
|
, mWidth(s.width())
|
|
, mFill(s.fill())
|
|
, mFmtFlags(s.flags()) {}
|
|
|
|
StreamStateRestorer::~StreamStateRestorer() {
|
|
restore();
|
|
}
|
|
|
|
// sets back all stream info that we remembered at construction
|
|
void StreamStateRestorer::restore() {
|
|
mStream.imbue(mLocale);
|
|
mStream.precision(mPrecision);
|
|
mStream.width(mWidth);
|
|
mStream.fill(mFill);
|
|
mStream.flags(mFmtFlags);
|
|
}
|
|
|
|
Number::Number(int width, int precision, double value)
|
|
: mWidth(width)
|
|
, mPrecision(precision)
|
|
, mValue(value) {}
|
|
|
|
// Groups the integer digits of an already formatted number in threes: "1234.50" -> "1,234.50".
|
|
//
|
|
// This used to be a std::numpunct facet imbued into the stream, which is the idiomatic way and works
|
|
// right up until someone builds with -fno-rtti: installing a facet goes through __dynamic_cast, which
|
|
// without RTTI reads through a null pointer and takes the process with it (issue #122). Grouping the
|
|
// digits by hand costs a few lines, produces the same output for everyone, and drops an allocation
|
|
// per formatted number along the way.
|
|
//
|
|
// Only a run of digits is touched, so "inf" and "nan" pass through unchanged.
|
|
std::string addThousandsSeparators(std::string str, char sep) {
|
|
size_t const start = (!str.empty() && ('-' == str[0] || '+' == str[0])) ? 1U : 0U;
|
|
size_t end = start;
|
|
while (end < str.size() && str[end] >= '0' && str[end] <= '9') {
|
|
++end;
|
|
}
|
|
|
|
// insert from the right, so the positions still to be visited stay valid
|
|
for (size_t pos = end; pos > start + 3;) {
|
|
pos -= 3;
|
|
str.insert(pos, 1, sep);
|
|
}
|
|
return str;
|
|
}
|
|
|
|
std::ostream& Number::write(std::ostream& os) const {
|
|
// No StreamStateRestorer: the only thing this used to change on os was the imbued locale, and
|
|
// std::setw below is consumed by the very next insertion.
|
|
|
|
// format without the stream's locale, so a global locale that already groups digits cannot group
|
|
// them a second time
|
|
std::stringstream ss;
|
|
ss.imbue(std::locale::classic());
|
|
ss << std::setprecision(mPrecision) << std::fixed << mValue;
|
|
|
|
os << std::setw(mWidth) << addThousandsSeparators(ss.str(), ',');
|
|
return os;
|
|
}
|
|
|
|
std::string Number::to_s() const {
|
|
std::stringstream ss;
|
|
write(ss);
|
|
return ss.str();
|
|
}
|
|
|
|
std::string to_s(uint64_t n) {
|
|
std::string str;
|
|
do {
|
|
str += static_cast<char>('0' + static_cast<char>(n % 10));
|
|
n /= 10;
|
|
} while (n != 0);
|
|
std::reverse(str.begin(), str.end());
|
|
return str;
|
|
}
|
|
|
|
std::ostream& operator<<(std::ostream& os, Number const& n) {
|
|
return n.write(os);
|
|
}
|
|
|
|
MarkDownColumn::MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val) noexcept
|
|
: mWidth(w)
|
|
, mPrecision(prec)
|
|
, mTitle(std::move(tit))
|
|
, mSuffix(std::move(suff))
|
|
, mValue(val) {}
|
|
|
|
MarkDownColumn::MarkDownColumn(int w, std::string tit, std::string text) noexcept
|
|
: mWidth(w)
|
|
, mPrecision(0)
|
|
, mTitle(std::move(tit))
|
|
, mSuffix()
|
|
, mValue(0.0)
|
|
, mText(std::move(text))
|
|
, mIsText(true) {}
|
|
|
|
std::string MarkDownColumn::padded(std::string const& text) const {
|
|
std::stringstream ss;
|
|
ss << '|' << std::setw(mWidth - 2) << std::right << text << ' ';
|
|
return ss.str();
|
|
}
|
|
|
|
std::string MarkDownColumn::title() const {
|
|
return padded(mTitle);
|
|
}
|
|
|
|
std::string MarkDownColumn::separator() const {
|
|
std::string sep(static_cast<size_t>(mWidth), '-');
|
|
sep.front() = '|';
|
|
sep.back() = ':';
|
|
return sep;
|
|
}
|
|
|
|
std::string MarkDownColumn::invalid() const {
|
|
std::string sep(static_cast<size_t>(mWidth), ' ');
|
|
sep.front() = '|';
|
|
sep[sep.size() - 2] = '-';
|
|
return sep;
|
|
}
|
|
|
|
std::string MarkDownColumn::value() const {
|
|
if (mIsText) {
|
|
// aligned like the header, so the ':' the separator puts on the right stays honest
|
|
return padded(mText);
|
|
}
|
|
std::stringstream ss;
|
|
auto width = mWidth - 2 - static_cast<int>(mSuffix.size());
|
|
ss << '|' << Number(width, mPrecision, mValue) << mSuffix << ' ';
|
|
return ss.str();
|
|
}
|
|
|
|
// Formats any text as markdown code, escaping backticks.
|
|
MarkDownCode::MarkDownCode(std::string const& what) {
|
|
mWhat.reserve(what.size() + 2);
|
|
mWhat.push_back('`');
|
|
for (char const c : what) {
|
|
mWhat.push_back(c);
|
|
if ('`' == c) {
|
|
mWhat.push_back('`');
|
|
}
|
|
}
|
|
mWhat.push_back('`');
|
|
}
|
|
|
|
std::ostream& MarkDownCode::write(std::ostream& os) const {
|
|
return os << mWhat;
|
|
}
|
|
|
|
std::ostream& operator<<(std::ostream& os, MarkDownCode const& mdCode) {
|
|
return mdCode.write(os);
|
|
}
|
|
} // namespace fmt
|
|
} // namespace detail
|
|
|
|
// provide implementation here so it's only generated once
|
|
Config::Config() = default;
|
|
Config::~Config() = default;
|
|
Config& Config::operator=(Config const&) = default;
|
|
Config& Config::operator=(Config&&) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)) = default;
|
|
Config::Config(Config const&) = default;
|
|
Config::Config(Config&&) noexcept = default;
|
|
|
|
// provide implementation here so it's only generated once
|
|
Result::~Result() = default;
|
|
Result& Result::operator=(Result const&) = default;
|
|
Result& Result::operator=(Result&&) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)) = default;
|
|
Result::Result(Result const&) = default;
|
|
Result::Result(Result&&) noexcept = default;
|
|
|
|
namespace detail {
|
|
template <typename T>
|
|
inline constexpr typename std::underlying_type<T>::type u(T val) noexcept {
|
|
return static_cast<typename std::underlying_type<T>::type>(val);
|
|
}
|
|
} // namespace detail
|
|
|
|
// Result returned after a benchmark has finished. Can be used as a baseline for relative().
|
|
Result::Result(Config benchmarkConfig)
|
|
: mConfig(std::move(benchmarkConfig))
|
|
// One slot per measure, plus one for _size itself. Result::fromString() returns _size for a name
|
|
// it does not know, so it reaches these accessors whenever a caller resolves a measure from
|
|
// user input - and reading one slot past the end of the storage is not the answer to a typo.
|
|
// With the extra slot it is a permanently empty measurement list, which reads as "nothing was
|
|
// measured": 0.0 from the statistics and false from has(), the same thing the mustache renderer
|
|
// already does with a measure it does not recognise.
|
|
, mNameToMeasurements{detail::u(Result::Measure::_size) + 1U} {}
|
|
|
|
void Result::add(Clock::duration totalElapsed, uint64_t iters, detail::PerformanceCounters const& pc) {
|
|
using detail::d;
|
|
using detail::u;
|
|
|
|
double const dIters = d(iters);
|
|
mNameToMeasurements[u(Result::Measure::iterations)].push_back(dIters);
|
|
|
|
mNameToMeasurements[u(Result::Measure::elapsed)].push_back(d(totalElapsed) / dIters);
|
|
if (pc.has().pageFaults) {
|
|
mNameToMeasurements[u(Result::Measure::pagefaults)].push_back(d(pc.val().pageFaults) / dIters);
|
|
}
|
|
if (pc.has().cpuCycles) {
|
|
mNameToMeasurements[u(Result::Measure::cpucycles)].push_back(d(pc.val().cpuCycles) / dIters);
|
|
}
|
|
if (pc.has().contextSwitches) {
|
|
mNameToMeasurements[u(Result::Measure::contextswitches)].push_back(d(pc.val().contextSwitches) / dIters);
|
|
}
|
|
if (pc.has().instructions) {
|
|
mNameToMeasurements[u(Result::Measure::instructions)].push_back(d(pc.val().instructions) / dIters);
|
|
}
|
|
if (pc.has().branchInstructions) {
|
|
double const branchInstructions = d(detail::correctBranchInstructions(pc.val().branchInstructions, iters));
|
|
mNameToMeasurements[u(Result::Measure::branchinstructions)].push_back(branchInstructions / dIters);
|
|
|
|
if (pc.has().branchMisses) {
|
|
auto const branchMisses = detail::correctBranchMisses(pc.val().branchMisses, branchInstructions);
|
|
mNameToMeasurements[u(Result::Measure::branchmisses)].push_back(branchMisses / dIters);
|
|
}
|
|
}
|
|
}
|
|
|
|
Config const& Result::config() const noexcept {
|
|
return mConfig;
|
|
}
|
|
|
|
inline double calcMedian(std::vector<double>& data) {
|
|
if (data.empty()) {
|
|
return 0.0;
|
|
}
|
|
std::sort(data.begin(), data.end());
|
|
|
|
auto midIdx = data.size() / 2U;
|
|
if (1U == (data.size() & 1U)) {
|
|
return data[midIdx];
|
|
}
|
|
return (data[midIdx - 1U] + data[midIdx]) / 2U;
|
|
}
|
|
|
|
namespace detail {
|
|
|
|
std::vector<double> pairedLogRatios(std::vector<double> const& a, std::vector<double> const& b) {
|
|
std::vector<double> logRatios;
|
|
auto const numPairs = (std::min)(a.size(), b.size());
|
|
logRatios.reserve(numPairs);
|
|
for (size_t i = 0; i < numPairs; ++i) {
|
|
// A round where either side measured 0 carries no ratio at all - the logarithm of it is not a
|
|
// large number, it is negative infinity, and one of those poisons every statistic downstream.
|
|
if (a[i] > 0.0 && b[i] > 0.0) {
|
|
logRatios.push_back(std::log(a[i]) - std::log(b[i]));
|
|
}
|
|
}
|
|
return logRatios;
|
|
}
|
|
|
|
std::vector<double> pairedLogRatios(Result const& a, Result const& b) {
|
|
auto const perRound = [](Result const& result) {
|
|
std::vector<double> values;
|
|
values.reserve(result.size());
|
|
for (size_t r = 0; r < result.size(); ++r) {
|
|
values.push_back(result.get(r, Result::Measure::elapsed));
|
|
}
|
|
return values;
|
|
};
|
|
return pairedLogRatios(perRound(a), perRound(b));
|
|
}
|
|
|
|
double medianOf(std::vector<double> values) {
|
|
return calcMedian(values);
|
|
}
|
|
|
|
double bonferroniConfidence(size_t numComparisons) noexcept {
|
|
return 1.0 - 0.05 / d((std::max)(numComparisons, static_cast<size_t>(1)));
|
|
}
|
|
|
|
std::pair<size_t, size_t> medianIntervalIndices(size_t n, double confidence) {
|
|
auto const alphaHalf = (1.0 - confidence) / 2.0;
|
|
size_t k = 0;
|
|
|
|
if (n <= 1024U) {
|
|
// Exact: walk the binomial tail up from P(X = 0) = 2^-n, each term from the one before it.
|
|
// 2^-1024 is still a representable double, so nothing underflows into a wrong answer.
|
|
double p = std::pow(0.5, d(n));
|
|
double cumulative = 0.0;
|
|
for (size_t i = 0; i < n; ++i) {
|
|
if (i > 0U) {
|
|
p = p * d(n - i + 1U) / d(i);
|
|
}
|
|
if (cumulative + p > alphaHalf) {
|
|
break;
|
|
}
|
|
cumulative += p;
|
|
k = i + 1U;
|
|
}
|
|
} else {
|
|
// well past the point where the normal approximation of a binomial is worth arguing about
|
|
auto const z = 1.959963985;
|
|
auto const approx = d(n) / 2.0 - z * std::sqrt(d(n)) / 2.0;
|
|
k = approx <= 0.0 ? 0U : static_cast<size_t>(approx);
|
|
}
|
|
|
|
if (0U == k || 2U * k > n) {
|
|
// no interval at this confidence for this few observations
|
|
return {1U, 0U};
|
|
}
|
|
return {k - 1U, n - k};
|
|
}
|
|
|
|
size_t countTiedRounds(std::vector<double> const& logRatios) {
|
|
size_t tied = 0;
|
|
for (auto logRatio : logRatios) {
|
|
// written without == so that -Wfloat-equal stays on for everyone else
|
|
if (!(logRatio < 0.0) && !(logRatio > 0.0)) {
|
|
++tied;
|
|
}
|
|
}
|
|
return tied;
|
|
}
|
|
|
|
std::pair<double, double> medianInterval(std::vector<double> values, double confidence) {
|
|
auto const indices = medianIntervalIndices(values.size(), confidence);
|
|
if (indices.first > indices.second) {
|
|
return {0.0, 0.0};
|
|
}
|
|
std::sort(values.begin(), values.end());
|
|
return {values[indices.first], values[indices.second]};
|
|
}
|
|
|
|
} // namespace detail
|
|
|
|
CompareResult::Entry::Entry(std::string entryName, Result entryResult, double entryRelative, double entryRelativeLow,
|
|
double entryRelativeHigh, size_t entryTiedRounds) noexcept
|
|
: name(std::move(entryName))
|
|
, result(std::move(entryResult))
|
|
, relative(entryRelative)
|
|
, relativeLow(entryRelativeLow)
|
|
, relativeHigh(entryRelativeHigh)
|
|
, tiedRounds(entryTiedRounds) {}
|
|
|
|
CompareResult::CompareResult(std::vector<Entry> entries, size_t numRounds)
|
|
: mEntries(std::move(entries))
|
|
, mRounds(numRounds) {}
|
|
|
|
size_t CompareResult::size() const noexcept {
|
|
return mEntries.size();
|
|
}
|
|
|
|
CompareResult::Entry const& CompareResult::operator[](size_t idx) const {
|
|
return mEntries.at(idx);
|
|
}
|
|
|
|
size_t CompareResult::rounds() const noexcept {
|
|
return mRounds;
|
|
}
|
|
|
|
size_t CompareResult::comparisons() const noexcept {
|
|
return mEntries.empty() ? 0U : mEntries.size() - 1U;
|
|
}
|
|
|
|
bool CompareResult::isSignificant(size_t idx) const {
|
|
if (0U == idx) {
|
|
// the baseline is not compared against itself
|
|
return false;
|
|
}
|
|
auto const& entry = mEntries.at(idx);
|
|
return entry.relativeLow > 1.0 || entry.relativeHigh < 1.0;
|
|
}
|
|
|
|
// The entry with the lowest median, ignoring index `skip` - pass size() to ignore nothing. Returns
|
|
// size() when that leaves no entry at all. The winner and the runner-up are the same question asked
|
|
// twice, and asking it twice is how the two answers come to disagree about ties.
|
|
static size_t fastestExcept(CompareResult const& compareResult, size_t skip) {
|
|
size_t best = compareResult.size();
|
|
auto bestMedian = (std::numeric_limits<double>::max)();
|
|
for (size_t i = 0; i < compareResult.size(); ++i) {
|
|
if (i == skip) {
|
|
continue;
|
|
}
|
|
auto const median = compareResult[i].result.median(Result::Measure::elapsed);
|
|
if (median < bestMedian) {
|
|
bestMedian = median;
|
|
best = i;
|
|
}
|
|
}
|
|
return best;
|
|
}
|
|
|
|
size_t CompareResult::fastest() const {
|
|
auto const best = fastestExcept(*this, mEntries.size());
|
|
// with no entries there is nothing to pick, and 0 is what this has always answered
|
|
return best == mEntries.size() ? 0U : best;
|
|
}
|
|
|
|
// The table half of a comparison: an ordinary benchmark table with the ratio to the baseline and an
|
|
// interval for it in front. Split out of operator<< because clang-tidy caps cognitive complexity at
|
|
// 25 and the two halves together sat well over it.
|
|
static void writeCompareTable(std::ostream& os, CompareResult const& compareResult) {
|
|
auto const& config = compareResult[0].result.config();
|
|
auto ratioText = [](double low, double high) {
|
|
std::ostringstream ss;
|
|
ss << detail::fmt::Number(1, 1, low * 100.0) << "% .. " << detail::fmt::Number(1, 1, high * 100.0) << "%";
|
|
return ss.str();
|
|
};
|
|
|
|
// The two extra columns are the comparison itself, so they are not hideable - without them this
|
|
// is just a table. Everything after them is the same set an ordinary table shows, performance
|
|
// counters included: a comparison collects them around every epoch either way, so leaving them
|
|
// out was paying for them and showing nothing.
|
|
auto columnsFor = [&](CompareResult::Entry const& entry, bool isBaseline) {
|
|
std::vector<detail::fmt::MarkDownColumn> columns;
|
|
columns.push_back(detail::relativeColumn(entry.relative * 100.0));
|
|
columns.emplace_back(22, "95% CI", isBaseline ? std::string() : ratioText(entry.relativeLow, entry.relativeHigh));
|
|
auto const measurements = detail::measurementColumns(entry.result.config(), entry.result);
|
|
columns.insert(columns.end(), measurements.begin(), measurements.end());
|
|
return columns;
|
|
};
|
|
|
|
// An ordinary table only reprints its header when the shape changes, and that state lives on the
|
|
// stream. A comparison table always prints its own header, so the remembered shape is now wrong:
|
|
// clear it, or the next run() on this stream matches against a header it never wrote and streams
|
|
// its rows straight under the comparison's summary with no header of their own.
|
|
detail::streamHeaderHash(os) = 0;
|
|
|
|
detail::writeTableHeaderLines(os, columnsFor(compareResult[0], true), config.mBenchmarkTitle);
|
|
|
|
for (size_t i = 0; i < compareResult.size(); ++i) {
|
|
auto const columns = columnsFor(compareResult[i], 0U == i);
|
|
for (auto const& col : columns) {
|
|
os << col.value();
|
|
}
|
|
os << "| " << detail::fmt::MarkDownCode(compareResult[i].name) << std::endl;
|
|
}
|
|
}
|
|
|
|
// Two alternatives read better as a sentence than as a row to look up.
|
|
// One spelling of the interval, so the table and the two summary shapes cannot drift apart.
|
|
static void writeCompareInterval(std::ostream& os, double low, double high) {
|
|
os << "95% CI [" << detail::fmt::Number(1, 2, low) << " .. " << detail::fmt::Number(1, 2, high) << "]";
|
|
}
|
|
|
|
static void writeComparePair(std::ostream& os, CompareResult const& compareResult) {
|
|
auto const baseline = detail::fmt::MarkDownCode(compareResult[0].name);
|
|
auto const& entry = compareResult[1];
|
|
auto const other = detail::fmt::MarkDownCode(entry.name);
|
|
|
|
if (!compareResult.isSignificant(1)) {
|
|
os << " no difference resolved between " << baseline << " and " << other << std::endl
|
|
<< " ratio " << detail::fmt::Number(1, 2, entry.relative) << "x, ";
|
|
writeCompareInterval(os, entry.relativeLow, entry.relativeHigh);
|
|
return;
|
|
}
|
|
|
|
// stated the way round the reader asked the question, so the interval inverts with it
|
|
auto const faster = entry.relative > 1.0;
|
|
os << " " << other << " ran " << detail::fmt::Number(1, 2, faster ? entry.relative : 1.0 / entry.relative)
|
|
<< (faster ? "x faster than " : "x slower than ") << baseline << std::endl
|
|
<< " ";
|
|
writeCompareInterval(os, faster ? entry.relativeLow : 1.0 / entry.relativeHigh,
|
|
faster ? entry.relativeHigh : 1.0 / entry.relativeLow);
|
|
}
|
|
|
|
// Picking the winner out of many is a selection, not a test: whichever came first is flattered by the
|
|
// same luck that made it first. So the claim is not "this one is fastest" on its own, it is how far
|
|
// ahead of the *runner-up* it is - and if that interval contains 1, the top two were not separated.
|
|
static void writeCompareWinner(std::ostream& os, CompareResult const& compareResult) {
|
|
auto const best = compareResult.fastest();
|
|
auto runnerUp = fastestExcept(compareResult, best);
|
|
if (runnerUp == compareResult.size()) {
|
|
// a comparison of one: there is nothing to be ahead of, so it is compared against itself and
|
|
// the empty interval that comes out reads as "not separated"
|
|
runnerUp = best;
|
|
}
|
|
|
|
// The rounds are paired for every alternative, not only against the baseline, so the top two can
|
|
// be compared to each other after the fact from the per-round measurements.
|
|
auto const logRatios = detail::pairedLogRatios(compareResult[runnerUp].result, compareResult[best].result);
|
|
auto const confidence = detail::bonferroniConfidence(compareResult.comparisons());
|
|
auto const interval = detail::medianInterval(logRatios, confidence);
|
|
auto const lead = std::exp(detail::medianOf(logRatios));
|
|
auto const leadLow = std::exp(interval.first);
|
|
auto const leadHigh = std::exp(interval.second);
|
|
|
|
auto const bestCode = detail::fmt::MarkDownCode(compareResult[best].name);
|
|
auto const runnerUpCode = detail::fmt::MarkDownCode(compareResult[runnerUp].name);
|
|
if (leadLow > 1.0) {
|
|
os << " " << bestCode << " is fastest of " << compareResult.size() << ", " << detail::fmt::Number(1, 2, lead)
|
|
<< "x ahead of " << runnerUpCode << std::endl;
|
|
} else {
|
|
os << " " << bestCode << " and " << runnerUpCode << " are the two fastest of " << compareResult.size()
|
|
<< ", and were not separated" << std::endl;
|
|
}
|
|
os << " ";
|
|
writeCompareInterval(os, leadLow, leadHigh);
|
|
os << ", intervals corrected for " << compareResult.comparisons() << " comparisons";
|
|
}
|
|
|
|
std::ostream& operator<<(std::ostream& os, CompareResult const& compareResult) {
|
|
detail::fmt::StreamStateRestorer const restorer(os);
|
|
if (0U == compareResult.size()) {
|
|
return os;
|
|
}
|
|
|
|
// The same table an ordinary benchmark prints, plus the ratio to the baseline and an interval for
|
|
// it. A ratio with no scale beside it cannot be told from the same ratio on a completely
|
|
// different scale, and a side that was wildly unstable is invisible in one - so the absolute
|
|
// numbers come first and the verdict after.
|
|
writeCompareTable(os, compareResult);
|
|
|
|
os << std::endl << " Summary" << std::endl;
|
|
if (2U == compareResult.size()) {
|
|
writeComparePair(os, compareResult);
|
|
} else {
|
|
writeCompareWinner(os, compareResult);
|
|
}
|
|
os << ", " << compareResult.rounds() << " paired rounds, interleaved" << std::endl;
|
|
|
|
size_t tied = 0;
|
|
for (size_t i = 1; i < compareResult.size(); ++i) {
|
|
tied = (std::max)(tied, compareResult[i].tiedRounds);
|
|
}
|
|
if (0U != tied) {
|
|
os << " up to " << tied << " of " << compareResult.rounds() << " rounds tied at the clock's resolution" << std::endl;
|
|
}
|
|
return os;
|
|
}
|
|
|
|
uint64_t Bench::compareIterations(std::vector<detail::ErasedOp> const& ops) const {
|
|
// An exact epochIterations() overrides any calculated count, exactly as it does in a normal run.
|
|
if (0 != epochIterations()) {
|
|
return epochIterations();
|
|
}
|
|
|
|
// The count that fills a normal epoch, per alternative. The smallest of them belongs to the
|
|
// slowest alternative - it needs the fewest iterations to fill an epoch - and the largest to the
|
|
// fastest.
|
|
auto const target = detail::targetRuntimePerEpoch(*this);
|
|
auto fewest = (std::numeric_limits<uint64_t>::max)();
|
|
uint64_t most = 0;
|
|
for (auto const& op : ops) {
|
|
auto const calibrated = compareCalibrate(op, target);
|
|
fewest = (std::min)(fewest, calibrated);
|
|
most = (std::max)(most, calibrated);
|
|
}
|
|
|
|
// Every count above fills the same target, so they are inversely proportional to the
|
|
// per-iteration times and convert to any other epoch length by scaling: a count that fills
|
|
// `target` times `wanted / target` is the count that fills `wanted`. Calibration stops at the
|
|
// first count that reaches the target rather than at the exact one, so the per-iteration time
|
|
// behind it is if anything underestimated and the scaled counts err upwards.
|
|
auto const targetSeconds = detail::d(target);
|
|
if (!(targetSeconds > 0.0)) {
|
|
// Nothing to scale by. A maxEpochTime of zero gets here, and every alternative then ran
|
|
// minEpochIterations during calibration, which is what `fewest` holds.
|
|
return fewest;
|
|
}
|
|
auto const scaled = [&](uint64_t iters, Clock::duration wanted) {
|
|
return detail::u64(detail::d(iters) * (detail::d(wanted) / targetSeconds));
|
|
};
|
|
|
|
// Taking `fewest` is what keeps the slowest alternative inside maxEpochTime, but on its own it
|
|
// makes every faster alternative run an epoch shorter than asked for by however much faster it
|
|
// is - a 1ns operation against a 50ns one gets a fiftieth of an epoch. So the count is raised
|
|
// until the *fastest* alternative clears the epoch this library calls long enough to measure:
|
|
// the clock's resolution times the multiple, which is where targetRuntimePerEpoch() starts from.
|
|
// That sits far below minEpochTime wherever the clock is good, so this binds only when the
|
|
// alternatives are orders of magnitude apart or the clock is coarse.
|
|
auto const measurable = scaled(most, detail::clockResolution() * clockResolutionMultiple());
|
|
|
|
// Raising it stretches the slowest alternative's epoch by the same factor, and maxEpochTime is
|
|
// the bound on that. A 1ns operation against a 1ms one cannot have both, and epochs a million
|
|
// times longer than asked for are the worse of the two failures.
|
|
auto const affordable = scaled(fewest, maxEpochTime());
|
|
|
|
return (std::max)(fewest, (std::min)(measurable, affordable));
|
|
}
|
|
|
|
uint64_t Bench::compareCalibrate(detail::ErasedOp const& op, Clock::duration target) const {
|
|
uint64_t numIters = minEpochIterations();
|
|
uint64_t reached = 0;
|
|
for (size_t attempt = 0; attempt < 64; ++attempt) {
|
|
Clock::time_point const before = Clock::now();
|
|
op.run(op.op, numIters);
|
|
auto const elapsed = Clock::now() - before;
|
|
if (elapsed >= target) {
|
|
// Measured again at the same count before it is believed. A single reading is one
|
|
// interruption away from being far too high, and this loop grows in steps of up to 10x -
|
|
// so one preempted attempt ends the search an order of magnitude early, and every
|
|
// alternative then runs epochs that much shorter than they were asked to be. That is not
|
|
// hypothetical: an Alpine leg on a shared two core runner returned 11111 for an operation
|
|
// whose 11111 iterations take 14us against a 100us target, because one attempt was
|
|
// interrupted for 86us. Two readings in a row have to cross.
|
|
if (reached == numIters) {
|
|
return numIters;
|
|
}
|
|
reached = numIters;
|
|
continue;
|
|
}
|
|
reached = 0;
|
|
|
|
// grow towards the target, but never by more than 10x at once - the same shape as the normal
|
|
// upscaling, so a wildly wrong first guess still converges in a few steps
|
|
auto const elapsedSeconds = detail::d(elapsed);
|
|
auto factor = elapsedSeconds > 0.0 ? detail::d(target) / elapsedSeconds : 10.0;
|
|
factor = (std::min)(10.0, (std::max)(1.5, factor));
|
|
if (numIters > (std::numeric_limits<uint64_t>::max)() / 16U) {
|
|
break;
|
|
}
|
|
numIters = detail::u64(detail::d(numIters) * factor) + 1U;
|
|
}
|
|
return numIters;
|
|
}
|
|
|
|
CompareResult Bench::compareImpl(std::vector<std::string> const& names, std::vector<detail::ErasedOp> const& ops) {
|
|
auto const numOps = ops.size();
|
|
|
|
// A comparison on a machine with frequency scaling on needs these at least as much as a run()
|
|
// does, and a program that only ever calls compare() would otherwise never see them.
|
|
detail::printStabilityInformationOnce(output());
|
|
detail::printPerformanceCounterHintOnce(output(), performanceCounters());
|
|
|
|
// Honored, though it is not the tool it looks like here: calibration below already runs each side
|
|
// for about a full epoch, and the serial correlation a warmup would be aimed at is removed by the
|
|
// pairing rather than by running longer first. Measured over 200 rounds, the raw per-round times
|
|
// carry a lag-1 autocorrelation around +0.10 while the paired differences carry -0.01 to -0.08,
|
|
// and dropping the first twenty rounds changes neither.
|
|
for (auto const& op : ops) {
|
|
op.run(op.op, warmup());
|
|
}
|
|
|
|
// One count for every alternative - see compareIterations() for why that matters so much.
|
|
auto const iters = compareIterations(ops);
|
|
|
|
std::vector<Result> results;
|
|
results.reserve(numOps);
|
|
for (auto const& name : names) {
|
|
Config entryConfig = mConfig;
|
|
entryConfig.mBenchmarkName = name;
|
|
results.emplace_back(std::move(entryConfig));
|
|
}
|
|
|
|
// Every alternative besides the baseline is one more chance to be wrong, so the intervals are
|
|
// widened to keep the whole table at 95% rather than each row separately.
|
|
auto const confidence = detail::bonferroniConfidence(numOps - 1U);
|
|
|
|
// Rounds come in blocks of one epoch per alternative, and the count is raised until the interval
|
|
// is possible at all - with many alternatives the corrected confidence needs more rounds before
|
|
// any pair of order statistics reaches it.
|
|
auto numRounds = ((epochs() + numOps - 1U) / numOps) * numOps;
|
|
while (true) {
|
|
auto const indices = detail::medianIntervalIndices(numRounds, confidence);
|
|
if (indices.first <= indices.second) {
|
|
break;
|
|
}
|
|
numRounds += numOps;
|
|
}
|
|
|
|
// Rounds, into whichever results are handed in. A stage is one call.
|
|
auto runRounds = [&](size_t rounds, std::vector<Result>& into) {
|
|
Rng orderRng;
|
|
std::vector<uint32_t> order(numOps);
|
|
for (size_t i = 0; i < numOps; ++i) {
|
|
order[i] = static_cast<uint32_t>(i);
|
|
}
|
|
for (size_t round = 0; round < rounds; ++round) {
|
|
auto const positionInBlock = round % numOps;
|
|
if (0 == positionInBlock) {
|
|
// A fresh random permutation per block, then rotated one step per round: every
|
|
// alternative occupies every position exactly once over the block, so their mean
|
|
// positions are equal and a drift that is linear over the block cancels. For two
|
|
// alternatives this produces exactly ABBA or BAAB.
|
|
orderRng.shuffle(order);
|
|
}
|
|
for (size_t slot = 0; slot < numOps; ++slot) {
|
|
auto const which = order[(slot + positionInBlock) % numOps];
|
|
compareEpoch(ops[which], iters, into[which]);
|
|
}
|
|
}
|
|
};
|
|
|
|
if (targetIntervalWidth() > 0.0) {
|
|
// A pilot, whose only job is to say how many rounds the target needs. Its measurements are
|
|
// then dropped: choosing the count from the same data that the interval is computed from is
|
|
// the optional-stopping mistake, and it makes an interval that covers less often than it
|
|
// claims. See targetIntervalWidth() for the argument.
|
|
std::vector<Result> pilot;
|
|
pilot.reserve(numOps);
|
|
for (auto const& name : names) {
|
|
Config pilotConfig = mConfig;
|
|
pilotConfig.mBenchmarkName = name;
|
|
pilot.emplace_back(std::move(pilotConfig));
|
|
}
|
|
runRounds(numRounds, pilot);
|
|
|
|
// The widest interval over the alternatives, so that every row reaches the target, not just
|
|
// the luckiest one.
|
|
auto widest = 0.0;
|
|
for (size_t i = 1; i < numOps; ++i) {
|
|
auto const logRatios = detail::pairedLogRatios(pilot[0], pilot[i]);
|
|
auto const interval = detail::medianInterval(logRatios, confidence);
|
|
auto const width = interval.second - interval.first;
|
|
if (width > widest) {
|
|
widest = width;
|
|
}
|
|
}
|
|
numRounds = detail::roundsForTargetWidth(numRounds, widest, targetIntervalWidth(), numOps, maxEpochs());
|
|
}
|
|
|
|
runRounds(numRounds, results);
|
|
|
|
std::vector<CompareResult::Entry> entries;
|
|
entries.reserve(numOps);
|
|
entries.emplace_back(names[0], std::move(results[0]), 1.0, 1.0, 1.0, static_cast<size_t>(0));
|
|
for (size_t i = 1; i < numOps; ++i) {
|
|
// ln(t_baseline) - ln(t_i), so a positive difference means the alternative was quicker and
|
|
// the ratio comes out above 1 - the same direction relative() reports. The baseline was moved
|
|
// into entries[0] above, which is where it is read from now.
|
|
auto const logRatios = detail::pairedLogRatios(entries[0].result, results[i]);
|
|
auto const interval = detail::medianInterval(logRatios, confidence);
|
|
entries.emplace_back(names[i], std::move(results[i]), std::exp(detail::medianOf(logRatios)), std::exp(interval.first),
|
|
std::exp(interval.second), detail::countTiedRounds(logRatios));
|
|
}
|
|
|
|
CompareResult compareResult{std::move(entries), numRounds};
|
|
if (nullptr != output()) {
|
|
*output() << compareResult;
|
|
}
|
|
return compareResult;
|
|
}
|
|
|
|
void Bench::compareEpoch(detail::ErasedOp const& op, uint64_t numIters, Result& result) {
|
|
auto& pc = detail::performanceCounters();
|
|
|
|
pc.beginMeasure();
|
|
Clock::time_point const before = Clock::now();
|
|
op.run(op.op, numIters);
|
|
Clock::time_point const after = Clock::now();
|
|
pc.endMeasure();
|
|
pc.updateResults(numIters);
|
|
|
|
result.add(after - before, numIters, pc);
|
|
}
|
|
|
|
std::vector<double> const& Result::measurements(Measure m) const {
|
|
return mNameToMeasurements.at(detail::u(m));
|
|
}
|
|
|
|
double Result::median(Measure m) const {
|
|
// create a copy so we can sort
|
|
auto data = measurements(m);
|
|
return calcMedian(data);
|
|
}
|
|
|
|
double Result::average(Measure m) const {
|
|
using detail::d;
|
|
auto const& data = measurements(m);
|
|
if (data.empty()) {
|
|
return 0.0;
|
|
}
|
|
|
|
return sum(m) / d(data.size());
|
|
}
|
|
|
|
double Result::medianAbsolutePercentError(Measure m) const {
|
|
// create copy
|
|
auto data = measurements(m);
|
|
|
|
// calculates MdAPE which is the median of percentage error
|
|
// see https://support.numxl.com/hc/en-us/articles/115001223503-MdAPE-Median-Absolute-Percentage-Error
|
|
auto med = calcMedian(data);
|
|
|
|
// The error is relative to the measurement itself, so a measurement of 0 has none that is finite: it is 0 when the
|
|
// median is 0 as well (they agree), and infinite otherwise, which is the limit of |(x - med) / x| for x -> 0.
|
|
// Calculating it directly would give NaN, and a NaN in the data breaks the ordering that calcMedian's sort needs.
|
|
auto const zeroError = med <= 0.0 ? 0.0 : std::numeric_limits<double>::infinity();
|
|
|
|
// transform the data to absolute error
|
|
for (auto& x : data) {
|
|
if (x <= 0.0) {
|
|
x = zeroError;
|
|
continue;
|
|
}
|
|
x = (x - med) / x;
|
|
if (x < 0) {
|
|
x = -x;
|
|
}
|
|
}
|
|
return calcMedian(data);
|
|
}
|
|
|
|
double Result::sum(Measure m) const noexcept {
|
|
auto const& data = measurements(m);
|
|
return std::accumulate(data.begin(), data.end(), 0.0);
|
|
}
|
|
|
|
double Result::sumProduct(Measure m1, Measure m2) const noexcept {
|
|
auto const& data1 = measurements(m1);
|
|
auto const& data2 = measurements(m2);
|
|
|
|
if (data1.size() != data2.size()) {
|
|
return 0.0;
|
|
}
|
|
|
|
double result = 0.0;
|
|
for (size_t i = 0, s = data1.size(); i != s; ++i) {
|
|
result += data1[i] * data2[i];
|
|
}
|
|
return result;
|
|
}
|
|
|
|
bool Result::has(Measure m) const noexcept {
|
|
return !measurements(m).empty();
|
|
}
|
|
|
|
double Result::get(size_t idx, Measure m) const {
|
|
return measurements(m).at(idx);
|
|
}
|
|
|
|
bool Result::empty() const noexcept {
|
|
return 0U == size();
|
|
}
|
|
|
|
size_t Result::size() const noexcept {
|
|
return measurements(Measure::elapsed).size();
|
|
}
|
|
|
|
double Result::minimum(Measure m) const noexcept {
|
|
auto const& data = measurements(m);
|
|
if (data.empty()) {
|
|
return 0.0;
|
|
}
|
|
|
|
// here its save to assume that at least one element is there
|
|
return *std::min_element(data.begin(), data.end());
|
|
}
|
|
|
|
double Result::maximum(Measure m) const noexcept {
|
|
auto const& data = measurements(m);
|
|
if (data.empty()) {
|
|
return 0.0;
|
|
}
|
|
|
|
// here its save to assume that at least one element is there
|
|
return *std::max_element(data.begin(), data.end());
|
|
}
|
|
|
|
std::string const& Result::context(char const* variableName) const {
|
|
return mConfig.mContext.at(variableName);
|
|
}
|
|
|
|
std::string const& Result::context(std::string const& variableName) const {
|
|
return mConfig.mContext.at(variableName);
|
|
}
|
|
|
|
Result::Measure Result::fromString(std::string const& str) {
|
|
if (str == "elapsed") {
|
|
return Measure::elapsed;
|
|
}
|
|
if (str == "iterations") {
|
|
return Measure::iterations;
|
|
}
|
|
if (str == "pagefaults") {
|
|
return Measure::pagefaults;
|
|
}
|
|
if (str == "cpucycles") {
|
|
return Measure::cpucycles;
|
|
}
|
|
if (str == "contextswitches") {
|
|
return Measure::contextswitches;
|
|
}
|
|
if (str == "instructions") {
|
|
return Measure::instructions;
|
|
}
|
|
if (str == "branchinstructions") {
|
|
return Measure::branchinstructions;
|
|
}
|
|
if (str == "branchmisses") {
|
|
return Measure::branchmisses;
|
|
}
|
|
// not found, return _size
|
|
return Measure::_size;
|
|
}
|
|
|
|
// Configuration of a microbenchmark.
|
|
Bench::Bench() {
|
|
mConfig.mOut = &std::cout;
|
|
detail::applyEnvConfig(*this);
|
|
}
|
|
|
|
Bench::Bench(Bench&&) noexcept = default;
|
|
Bench& Bench::operator=(Bench&&) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)) = default;
|
|
Bench::Bench(Bench const&) = default;
|
|
Bench& Bench::operator=(Bench const&) = default;
|
|
Bench::~Bench() noexcept = default;
|
|
|
|
double Bench::batch() const noexcept {
|
|
return mConfig.mBatch;
|
|
}
|
|
|
|
double Bench::complexityN() const noexcept {
|
|
return mConfig.mComplexityN;
|
|
}
|
|
|
|
// Set a baseline to compare it to. 100% it is exactly as fast as the baseline, >100% means it is faster than the baseline, <100%
|
|
// means it is slower than the baseline.
|
|
Bench& Bench::relative(bool isRelativeEnabled) noexcept {
|
|
mConfig.mIsRelative = isRelativeEnabled;
|
|
return *this;
|
|
}
|
|
bool Bench::relative() const noexcept {
|
|
return mConfig.mIsRelative;
|
|
}
|
|
|
|
Bench& Bench::performanceCounters(bool showPerformanceCounters) noexcept {
|
|
mConfig.mShowPerformanceCounters = showPerformanceCounters;
|
|
return *this;
|
|
}
|
|
bool Bench::performanceCounters() const noexcept {
|
|
return mConfig.mShowPerformanceCounters;
|
|
}
|
|
|
|
namespace detail {
|
|
|
|
// One bit of mHiddenColumns per Column, so there had better be at most 32 of them.
|
|
static_assert(static_cast<size_t>(Column::_size) <= 32, "Column no longer fits in Config::mHiddenColumns");
|
|
|
|
ANKERL_NANOBENCH(NODISCARD) inline uint32_t columnBit(Column column) noexcept {
|
|
return UINT32_C(1) << static_cast<uint32_t>(column);
|
|
}
|
|
|
|
bool isColumnVisible(Config const& config, Column column) noexcept {
|
|
return 0 == (config.mHiddenColumns & columnBit(column));
|
|
}
|
|
|
|
} // namespace detail
|
|
|
|
Bench& Bench::hideColumn(Column column) noexcept {
|
|
mConfig.mHiddenColumns |= detail::columnBit(column);
|
|
return *this;
|
|
}
|
|
|
|
Bench& Bench::showColumn(Column column) noexcept {
|
|
mConfig.mHiddenColumns &= ~detail::columnBit(column);
|
|
return *this;
|
|
}
|
|
|
|
bool Bench::isColumnVisible(Column column) const noexcept {
|
|
return detail::isColumnVisible(mConfig, column);
|
|
}
|
|
|
|
Bench& Bench::contextColumn(std::string const& variableName) {
|
|
// adding the same name twice would print the same column twice
|
|
if (mConfig.mContextColumns.end() != std::find(mConfig.mContextColumns.begin(), mConfig.mContextColumns.end(), variableName)) {
|
|
return *this;
|
|
}
|
|
mConfig.mContextColumns.push_back(variableName);
|
|
return *this;
|
|
}
|
|
|
|
Bench& Bench::clearContextColumns() noexcept {
|
|
mConfig.mContextColumns.clear();
|
|
return *this;
|
|
}
|
|
|
|
// Operation unit. Defaults to "op", could be e.g. "byte" for string processing.
|
|
// If u differs from currently set unit, the stored results will be cleared.
|
|
// Use singular (byte, not bytes).
|
|
Bench& Bench::unit(char const* u) {
|
|
if (u != mConfig.mUnit) {
|
|
mResults.clear();
|
|
}
|
|
mConfig.mUnit = u;
|
|
return *this;
|
|
}
|
|
|
|
Bench& Bench::unit(std::string const& u) {
|
|
return unit(u.c_str());
|
|
}
|
|
|
|
std::string const& Bench::unit() const noexcept {
|
|
return mConfig.mUnit;
|
|
}
|
|
|
|
Bench& Bench::timeUnit(std::chrono::duration<double> const& tu, std::string const& tuName) {
|
|
mConfig.mTimeUnit = tu;
|
|
mConfig.mTimeUnitName = tuName;
|
|
return *this;
|
|
}
|
|
|
|
std::string const& Bench::timeUnitName() const noexcept {
|
|
return mConfig.mTimeUnitName;
|
|
}
|
|
|
|
std::chrono::duration<double> const& Bench::timeUnit() const noexcept {
|
|
return mConfig.mTimeUnit;
|
|
}
|
|
|
|
Bench& Bench::title(const char* benchmarkTitle) {
|
|
return title(std::string(benchmarkTitle));
|
|
}
|
|
|
|
// If benchmarkTitle differs from currently set title, the stored results will be cleared. One body
|
|
// for both overloads, so that rule cannot come to mean two things.
|
|
Bench& Bench::title(std::string const& benchmarkTitle) {
|
|
if (benchmarkTitle != mConfig.mBenchmarkTitle) {
|
|
mResults.clear();
|
|
}
|
|
mConfig.mBenchmarkTitle = benchmarkTitle;
|
|
return *this;
|
|
}
|
|
|
|
std::string const& Bench::title() const noexcept {
|
|
return mConfig.mBenchmarkTitle;
|
|
}
|
|
|
|
Bench& Bench::name(const char* benchmarkName) {
|
|
mConfig.mBenchmarkName = benchmarkName;
|
|
return *this;
|
|
}
|
|
|
|
Bench& Bench::name(std::string const& benchmarkName) {
|
|
mConfig.mBenchmarkName = benchmarkName;
|
|
return *this;
|
|
}
|
|
|
|
# if ANKERL_NANOBENCH(HAS_STRING_VIEW)
|
|
Bench& Bench::name(std::string_view benchmarkName) {
|
|
mConfig.mBenchmarkName.assign(benchmarkName.data(), benchmarkName.size());
|
|
return *this;
|
|
}
|
|
# endif
|
|
|
|
std::string const& Bench::name() const noexcept {
|
|
return mConfig.mBenchmarkName;
|
|
}
|
|
|
|
Bench& Bench::context(char const* variableName, char const* variableValue) {
|
|
mConfig.mContext[variableName] = variableValue;
|
|
return *this;
|
|
}
|
|
|
|
Bench& Bench::context(std::string const& variableName, std::string const& variableValue) {
|
|
mConfig.mContext[variableName] = variableValue;
|
|
return *this;
|
|
}
|
|
|
|
Bench& Bench::clearContext() {
|
|
mConfig.mContext.clear();
|
|
return *this;
|
|
}
|
|
|
|
// Number of epochs to evaluate. The reported result will be the median of evaluation of each epoch.
|
|
Bench& Bench::epochs(size_t numEpochs) noexcept {
|
|
mConfig.mNumEpochs = numEpochs;
|
|
return *this;
|
|
}
|
|
size_t Bench::epochs() const noexcept {
|
|
return mConfig.mNumEpochs;
|
|
}
|
|
|
|
Bench& Bench::targetIntervalWidth(double width) noexcept {
|
|
mConfig.mTargetIntervalWidth = width;
|
|
return *this;
|
|
}
|
|
double Bench::targetIntervalWidth() const noexcept {
|
|
return mConfig.mTargetIntervalWidth;
|
|
}
|
|
|
|
Bench& Bench::maxEpochs(size_t numEpochs) noexcept {
|
|
mConfig.mMaxEpochs = numEpochs;
|
|
return *this;
|
|
}
|
|
size_t Bench::maxEpochs() const noexcept {
|
|
return mConfig.mMaxEpochs;
|
|
}
|
|
|
|
// Desired evaluation time is a multiple of clock resolution. Default is to be 1000 times above this measurement precision.
|
|
Bench& Bench::clockResolutionMultiple(size_t multiple) noexcept {
|
|
mConfig.mClockResolutionMultiple = multiple;
|
|
return *this;
|
|
}
|
|
size_t Bench::clockResolutionMultiple() const noexcept {
|
|
return mConfig.mClockResolutionMultiple;
|
|
}
|
|
|
|
// Sets the maximum time each epoch should take. Default is 100ms.
|
|
Bench& Bench::maxEpochTime(std::chrono::nanoseconds t) noexcept {
|
|
mConfig.mMaxEpochTime = t;
|
|
return *this;
|
|
}
|
|
std::chrono::nanoseconds Bench::maxEpochTime() const noexcept {
|
|
return mConfig.mMaxEpochTime;
|
|
}
|
|
|
|
// Sets the minimum time each epoch should take. Default is 1ms.
|
|
Bench& Bench::minEpochTime(std::chrono::nanoseconds t) noexcept {
|
|
mConfig.mMinEpochTime = t;
|
|
return *this;
|
|
}
|
|
std::chrono::nanoseconds Bench::minEpochTime() const noexcept {
|
|
return mConfig.mMinEpochTime;
|
|
}
|
|
|
|
Bench& Bench::minEpochIterations(uint64_t numIters) noexcept {
|
|
mConfig.mMinEpochIterations = (numIters == 0) ? 1 : numIters;
|
|
return *this;
|
|
}
|
|
uint64_t Bench::minEpochIterations() const noexcept {
|
|
return mConfig.mMinEpochIterations;
|
|
}
|
|
|
|
Bench& Bench::epochIterations(uint64_t numIters) noexcept {
|
|
mConfig.mEpochIterations = numIters;
|
|
return *this;
|
|
}
|
|
uint64_t Bench::epochIterations() const noexcept {
|
|
return mConfig.mEpochIterations;
|
|
}
|
|
|
|
Bench& Bench::warmup(uint64_t numWarmupIters) noexcept {
|
|
mConfig.mWarmup = numWarmupIters;
|
|
return *this;
|
|
}
|
|
uint64_t Bench::warmup() const noexcept {
|
|
return mConfig.mWarmup;
|
|
}
|
|
|
|
Bench& Bench::config(Config const& benchmarkConfig) {
|
|
mConfig = benchmarkConfig;
|
|
return *this;
|
|
}
|
|
Config const& Bench::config() const noexcept {
|
|
return mConfig;
|
|
}
|
|
|
|
Bench& Bench::output(std::ostream* outstream) noexcept {
|
|
mConfig.mOut = outstream;
|
|
return *this;
|
|
}
|
|
|
|
ANKERL_NANOBENCH(NODISCARD) std::ostream* Bench::output() const noexcept {
|
|
return mConfig.mOut;
|
|
}
|
|
|
|
std::vector<Result> const& Bench::results() const noexcept {
|
|
return mResults;
|
|
}
|
|
|
|
Bench& Bench::render(char const* templateContent, std::ostream& os) {
|
|
::ankerl::nanobench::render(templateContent, *this, os);
|
|
return *this;
|
|
}
|
|
|
|
Bench& Bench::render(std::string const& templateContent, std::ostream& os) {
|
|
::ankerl::nanobench::render(templateContent, *this, os);
|
|
return *this;
|
|
}
|
|
|
|
std::vector<BigO> Bench::complexityBigO() const {
|
|
std::vector<BigO> bigOs;
|
|
auto rangeMeasure = BigO::collectRangeMeasure(mResults);
|
|
bigOs.emplace_back("O(1)", rangeMeasure, [](double) {
|
|
return 1.0;
|
|
});
|
|
bigOs.emplace_back("O(n)", rangeMeasure, [](double n) {
|
|
return n;
|
|
});
|
|
bigOs.emplace_back("O(log n)", rangeMeasure, [](double n) {
|
|
return std::log2(n);
|
|
});
|
|
bigOs.emplace_back("O(n log n)", rangeMeasure, [](double n) {
|
|
return n * std::log2(n);
|
|
});
|
|
bigOs.emplace_back("O(n^2)", rangeMeasure, [](double n) {
|
|
return n * n;
|
|
});
|
|
bigOs.emplace_back("O(n^3)", rangeMeasure, [](double n) {
|
|
return n * n * n;
|
|
});
|
|
std::sort(bigOs.begin(), bigOs.end());
|
|
return bigOs;
|
|
}
|
|
|
|
Rng::Rng()
|
|
: mX(0)
|
|
, mY(0) {
|
|
std::random_device rd;
|
|
std::uniform_int_distribution<uint64_t> dist;
|
|
do {
|
|
mX = dist(rd);
|
|
mY = dist(rd);
|
|
} while (mX == 0 && mY == 0);
|
|
}
|
|
|
|
ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined")
|
|
uint64_t splitMix64(uint64_t& state) noexcept {
|
|
uint64_t z = (state += UINT64_C(0x9e3779b97f4a7c15));
|
|
z = (z ^ (z >> 30U)) * UINT64_C(0xbf58476d1ce4e5b9);
|
|
z = (z ^ (z >> 27U)) * UINT64_C(0x94d049bb133111eb);
|
|
return z ^ (z >> 31U);
|
|
}
|
|
|
|
// Seeded as described in romu paper (update april 2020)
|
|
Rng::Rng(uint64_t seed) noexcept
|
|
: mX(splitMix64(seed))
|
|
, mY(splitMix64(seed)) {
|
|
for (size_t i = 0; i < 10; ++i) {
|
|
operator()();
|
|
}
|
|
}
|
|
|
|
// only internally used to copy the RNG.
|
|
Rng::Rng(uint64_t x, uint64_t y) noexcept
|
|
: mX(x)
|
|
, mY(y) {}
|
|
|
|
Rng Rng::copy() const noexcept {
|
|
return Rng{mX, mY};
|
|
}
|
|
|
|
Rng::Rng(std::vector<uint64_t> const& data)
|
|
: mX(0)
|
|
, mY(0) {
|
|
if (data.size() != 2) {
|
|
ANKERL_NANOBENCH_THROW(std::runtime_error("ankerl::nanobench::Rng::Rng: needed exactly 2 entries in data, but got " +
|
|
detail::fmt::to_s(data.size())));
|
|
}
|
|
mX = data[0];
|
|
mY = data[1];
|
|
}
|
|
|
|
std::vector<uint64_t> Rng::state() const {
|
|
std::vector<uint64_t> data(2);
|
|
data[0] = mX;
|
|
data[1] = mY;
|
|
return data;
|
|
}
|
|
|
|
BigO::RangeMeasure BigO::collectRangeMeasure(std::vector<Result> const& results) {
|
|
BigO::RangeMeasure rangeMeasure;
|
|
for (auto const& result : results) {
|
|
if (result.config().mComplexityN > 0.0) {
|
|
rangeMeasure.emplace_back(result.config().mComplexityN, result.median(Result::Measure::elapsed));
|
|
}
|
|
}
|
|
return rangeMeasure;
|
|
}
|
|
|
|
BigO::BigO(std::string bigOName, RangeMeasure const& rangeMeasure)
|
|
: mName(std::move(bigOName)) {
|
|
|
|
// Nothing to fit against: leave the constant and the error at 0 rather than dividing by zero
|
|
// three times over. complexityBigO() reaches this whenever it is called on a Bench where no
|
|
// complexityN was ever set, since collectRangeMeasure() then returns nothing - the six models
|
|
// used to come out all-NaN, which prints as "nan" and, being unordered, is a poor thing to hand
|
|
// to std::sort.
|
|
if (rangeMeasure.empty()) {
|
|
return;
|
|
}
|
|
|
|
// estimate the constant factor
|
|
double sumRangeMeasure = 0.0;
|
|
double sumRangeRange = 0.0;
|
|
|
|
for (const auto& rm : rangeMeasure) {
|
|
sumRangeMeasure += rm.first * rm.second;
|
|
sumRangeRange += rm.first * rm.first;
|
|
}
|
|
// a sum of squares, so this is only 0 when every n is
|
|
mConstant = sumRangeRange <= 0.0 ? 0.0 : sumRangeMeasure / sumRangeRange;
|
|
|
|
// calculate root mean square
|
|
double err = 0.0;
|
|
double sumMeasure = 0.0;
|
|
for (const auto& rm : rangeMeasure) {
|
|
auto diff = mConstant * rm.first - rm.second;
|
|
err += diff * diff;
|
|
|
|
sumMeasure += rm.second;
|
|
}
|
|
|
|
auto n = detail::d(rangeMeasure.size());
|
|
auto mean = sumMeasure / n;
|
|
// the error is relative to the mean measurement, so there is no relative error to report when
|
|
// everything measured as 0 - and 0/0 is not it
|
|
mNormalizedRootMeanSquare = mean <= 0.0 ? 0.0 : std::sqrt(err / n) / mean;
|
|
}
|
|
|
|
BigO::BigO(const char* bigOName, RangeMeasure const& rangeMeasure)
|
|
: BigO(std::string(bigOName), rangeMeasure) {}
|
|
|
|
std::string const& BigO::name() const noexcept {
|
|
return mName;
|
|
}
|
|
|
|
double BigO::constant() const noexcept {
|
|
return mConstant;
|
|
}
|
|
|
|
double BigO::normalizedRootMeanSquare() const noexcept {
|
|
return mNormalizedRootMeanSquare;
|
|
}
|
|
|
|
bool BigO::operator<(BigO const& other) const noexcept {
|
|
return (mNormalizedRootMeanSquare < other.mNormalizedRootMeanSquare) ||
|
|
(!(mNormalizedRootMeanSquare > other.mNormalizedRootMeanSquare) && mName < other.mName);
|
|
}
|
|
|
|
std::ostream& operator<<(std::ostream& os, BigO const& bigO) {
|
|
return os << bigO.constant() << " * " << bigO.name() << ", rms=" << bigO.normalizedRootMeanSquare();
|
|
}
|
|
|
|
std::ostream& operator<<(std::ostream& os, std::vector<ankerl::nanobench::BigO> const& bigOs) {
|
|
detail::fmt::StreamStateRestorer const restorer(os);
|
|
os << std::endl << "| coefficient | err% | complexity" << std::endl << "|--------------:|-------:|------------" << std::endl;
|
|
for (auto const& bigO : bigOs) {
|
|
os << "|" << std::setw(14) << std::setprecision(7) << std::scientific << bigO.constant() << " ";
|
|
os << "|" << detail::fmt::Number(6, 1, bigO.normalizedRootMeanSquare() * 100.0) << "% ";
|
|
os << "| " << bigO.name();
|
|
os << std::endl;
|
|
}
|
|
return os;
|
|
}
|
|
|
|
} // namespace nanobench
|
|
} // namespace ankerl
|
|
|
|
#endif // ANKERL_NANOBENCH_IMPLEMENT
|
|
#endif // ANKERL_NANOBENCH_H_INCLUDED
|