From f6e09e537bab92bac0c0f9cfaa4f79761356bf7e Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Tue, 9 Aug 2016 18:56:48 +0000 Subject: [PATCH] Update in-tree Google Benchmark to current ToT. I've put some work into the Google Benchmark library in order to make it easier to benchmark libc++. These changes have already been upstreamed into Google Benchmark and this patch applies the changes to the in-tree version. The main improvement in the addition of a 'compare_bench.py' script which makes it very easy to compare benchmarks. For example to compare the native STL to libc++ you would run: `$ compare_bench.py ./util_smartptr.native.out ./util_smartptr.libcxx.out` And the output would look like: RUNNING: ./util_smartptr.native.out Benchmark Time CPU Iterations ---------------------------------------------------------------- BM_SharedPtrCreateDestroy 62 ns 62 ns 10937500 BM_SharedPtrIncDecRef 31 ns 31 ns 23972603 BM_WeakPtrIncDecRef 28 ns 28 ns 23648649 RUNNING: ./util_smartptr.libcxx.out Benchmark Time CPU Iterations ---------------------------------------------------------------- BM_SharedPtrCreateDestroy 46 ns 46 ns 14957265 BM_SharedPtrIncDecRef 31 ns 31 ns 22435897 BM_WeakPtrIncDecRef 34 ns 34 ns 21084337 Comparing ./util_smartptr.native.out to ./util_smartptr.libcxx.out Benchmark Time CPU ----------------------------------------------------- BM_SharedPtrCreateDestroy -0.26 -0.26 BM_SharedPtrIncDecRef +0.00 +0.00 BM_WeakPtrIncDecRef +0.21 +0.21 llvm-svn: 278147 --- libcxx/benchmarks/CMakeLists.txt | 7 +- libcxx/benchmarks/ContainerBenchmarks.hpp | 14 +- libcxx/benchmarks/algorithms.bench.cpp | 2 +- .../benchmarks/unordered_set_operations.bench.cpp | 2 +- libcxx/utils/google-benchmark/AUTHORS | 1 + libcxx/utils/google-benchmark/README.md | 110 ++++++--- .../include/benchmark/benchmark_api.h | 167 ++++++++----- .../google-benchmark/include/benchmark/macros.h | 10 +- .../google-benchmark/include/benchmark/reporter.h | 15 +- libcxx/utils/google-benchmark/src/benchmark.cc | 263 +++++++++++++-------- libcxx/utils/google-benchmark/src/colorprint.cc | 16 +- libcxx/utils/google-benchmark/src/colorprint.h | 1 + libcxx/utils/google-benchmark/src/complexity.cc | 4 +- .../utils/google-benchmark/src/console_reporter.cc | 41 ++-- libcxx/utils/google-benchmark/src/cycleclock.h | 6 +- libcxx/utils/google-benchmark/src/sysinfo.cc | 8 +- libcxx/utils/google-benchmark/test/CMakeLists.txt | 6 + libcxx/utils/google-benchmark/test/basic_test.cc | 20 +- .../utils/google-benchmark/test/benchmark_test.cc | 26 +- .../utils/google-benchmark/test/complexity_test.cc | 48 ++-- libcxx/utils/google-benchmark/test/fixture_test.cc | 2 +- libcxx/utils/google-benchmark/test/map_test.cc | 6 +- .../google-benchmark/test/multiple_ranges_test.cc | 46 ++++ libcxx/utils/google-benchmark/test/options_test.cc | 6 +- .../test/register_benchmark_test.cc | 149 ++++++++++++ .../google-benchmark/test/reporter_output_test.cc | 16 +- .../google-benchmark/test/skip_with_error_test.cc | 4 +- .../utils/google-benchmark/tools/compare_bench.py | 30 +++ .../tools/gbench/Inputs/test1_run1.json | 46 ++++ .../tools/gbench/Inputs/test1_run2.json | 46 ++++ .../google-benchmark/tools/gbench/__init__.py | 8 + .../utils/google-benchmark/tools/gbench/report.py | 136 +++++++++++ libcxx/utils/google-benchmark/tools/gbench/util.py | 130 ++++++++++ 33 files changed, 1102 insertions(+), 290 deletions(-) create mode 100644 libcxx/utils/google-benchmark/test/multiple_ranges_test.cc create mode 100644 libcxx/utils/google-benchmark/test/register_benchmark_test.cc create mode 100644 libcxx/utils/google-benchmark/tools/compare_bench.py create mode 100644 libcxx/utils/google-benchmark/tools/gbench/Inputs/test1_run1.json create mode 100644 libcxx/utils/google-benchmark/tools/gbench/Inputs/test1_run2.json create mode 100644 libcxx/utils/google-benchmark/tools/gbench/__init__.py create mode 100644 libcxx/utils/google-benchmark/tools/gbench/report.py create mode 100644 libcxx/utils/google-benchmark/tools/gbench/util.py diff --git a/libcxx/benchmarks/CMakeLists.txt b/libcxx/benchmarks/CMakeLists.txt index 1adbf6f..a1bc362 100644 --- a/libcxx/benchmarks/CMakeLists.txt +++ b/libcxx/benchmarks/CMakeLists.txt @@ -87,7 +87,12 @@ macro(add_benchmark_test name source_file) add_executable(${libcxx_target} EXCLUDE_FROM_ALL ${source_file}) add_dependencies(${libcxx_target} cxx google-benchmark-libcxx) add_dependencies(libcxx-benchmarks ${libcxx_target}) - target_link_libraries(${libcxx_target} cxx -lbenchmark) + if (LIBCXX_ENABLE_SHARED) + target_link_libraries(${libcxx_target} cxx_shared) + else() + target_link_libraries(${libcxx_target} cxx_static) + endif() + target_link_libraries(${libcxx_target} -lbenchmark) set_target_properties(${libcxx_target} PROPERTIES OUTPUT_NAME "${name}.libcxx.out" diff --git a/libcxx/benchmarks/ContainerBenchmarks.hpp b/libcxx/benchmarks/ContainerBenchmarks.hpp index 6350e6b..973a135 100644 --- a/libcxx/benchmarks/ContainerBenchmarks.hpp +++ b/libcxx/benchmarks/ContainerBenchmarks.hpp @@ -10,7 +10,7 @@ namespace ContainerBenchmarks { template void BM_ConstructIterIter(benchmark::State& st, Container, GenInputs gen) { - auto in = gen(st.range_x()); + auto in = gen(st.range(0)); const auto end = in.end(); benchmark::DoNotOptimize(&in); while (st.KeepRunning()) { @@ -21,7 +21,7 @@ void BM_ConstructIterIter(benchmark::State& st, Container, GenInputs gen) { template void BM_InsertValue(benchmark::State& st, Container c, GenInputs gen) { - auto in = gen(st.range_x()); + auto in = gen(st.range(0)); const auto end = in.end(); while (st.KeepRunning()) { c.clear(); @@ -34,7 +34,7 @@ void BM_InsertValue(benchmark::State& st, Container c, GenInputs gen) { template void BM_InsertValueRehash(benchmark::State& st, Container c, GenInputs gen) { - auto in = gen(st.range_x()); + auto in = gen(st.range(0)); const auto end = in.end(); while (st.KeepRunning()) { c.clear(); @@ -49,7 +49,7 @@ void BM_InsertValueRehash(benchmark::State& st, Container c, GenInputs gen) { template void BM_InsertDuplicate(benchmark::State& st, Container c, GenInputs gen) { - auto in = gen(st.range_x()); + auto in = gen(st.range(0)); const auto end = in.end(); c.insert(in.begin(), in.end()); benchmark::DoNotOptimize(&c); @@ -65,7 +65,7 @@ void BM_InsertDuplicate(benchmark::State& st, Container c, GenInputs gen) { template void BM_EmplaceDuplicate(benchmark::State& st, Container c, GenInputs gen) { - auto in = gen(st.range_x()); + auto in = gen(st.range(0)); const auto end = in.end(); c.insert(in.begin(), in.end()); benchmark::DoNotOptimize(&c); @@ -80,7 +80,7 @@ void BM_EmplaceDuplicate(benchmark::State& st, Container c, GenInputs gen) { template static void BM_Find(benchmark::State& st, Container c, GenInputs gen) { - auto in = gen(st.range_x()); + auto in = gen(st.range(0)); c.insert(in.begin(), in.end()); benchmark::DoNotOptimize(&(*c.begin())); const auto end = in.data() + in.size(); @@ -95,7 +95,7 @@ static void BM_Find(benchmark::State& st, Container c, GenInputs gen) { template static void BM_FindRehash(benchmark::State& st, Container c, GenInputs gen) { c.rehash(8); - auto in = gen(st.range_x()); + auto in = gen(st.range(0)); c.insert(in.begin(), in.end()); benchmark::DoNotOptimize(&(*c.begin())); const auto end = in.data() + in.size(); diff --git a/libcxx/benchmarks/algorithms.bench.cpp b/libcxx/benchmarks/algorithms.bench.cpp index 82ed992..745cc17 100644 --- a/libcxx/benchmarks/algorithms.bench.cpp +++ b/libcxx/benchmarks/algorithms.bench.cpp @@ -10,7 +10,7 @@ constexpr std::size_t TestNumInputs = 1024; template void BM_Sort(benchmark::State& st, GenInputs gen) { using ValueType = typename decltype(gen(0))::value_type; - const auto in = gen(st.range_x()); + const auto in = gen(st.range(0)); std::vector inputs[5]; auto reset_inputs = [&]() { for (auto& C : inputs) { diff --git a/libcxx/benchmarks/unordered_set_operations.bench.cpp b/libcxx/benchmarks/unordered_set_operations.bench.cpp index 8c44638..e2afdde 100644 --- a/libcxx/benchmarks/unordered_set_operations.bench.cpp +++ b/libcxx/benchmarks/unordered_set_operations.bench.cpp @@ -109,7 +109,7 @@ struct UInt64Hash2 { template void BM_Hash(benchmark::State& st, HashFn fn, GenInputs gen) { - auto in = gen(st.range_x()); + auto in = gen(st.range(0)); const auto end = in.data() + in.size(); std::size_t last_hash = 0; benchmark::DoNotOptimize(&last_hash); diff --git a/libcxx/utils/google-benchmark/AUTHORS b/libcxx/utils/google-benchmark/AUTHORS index 0f93e01..0e40922 100644 --- a/libcxx/utils/google-benchmark/AUTHORS +++ b/libcxx/utils/google-benchmark/AUTHORS @@ -13,6 +13,7 @@ Arne Beer Christopher Seymour David Coeurjolly Dominic Hamon +Eric Fiselier Eugene Zhuk Evgeny Safronov Felix Homann diff --git a/libcxx/utils/google-benchmark/README.md b/libcxx/utils/google-benchmark/README.md index a0bcc61..8cf853cb 100644 --- a/libcxx/utils/google-benchmark/README.md +++ b/libcxx/utils/google-benchmark/README.md @@ -40,13 +40,13 @@ measuring the speed of `memcpy()` calls of different lengths: ```c++ static void BM_memcpy(benchmark::State& state) { - char* src = new char[state.range_x()]; - char* dst = new char[state.range_x()]; - memset(src, 'x', state.range_x()); + char* src = new char[state.range(0)]; + char* dst = new char[state.range(0)]; + memset(src, 'x', state.range(0)); while (state.KeepRunning()) - memcpy(dst, src, state.range_x()); + memcpy(dst, src, state.range(0)); state.SetBytesProcessed(int64_t(state.iterations()) * - int64_t(state.range_x())); + int64_t(state.range(0))); delete[] src; delete[] dst; } @@ -70,7 +70,7 @@ BENCHMARK(BM_memcpy)->RangeMultiplier(2)->Range(8, 8<<10); ``` Now arguments generated are [ 8, 16, 32, 64, 128, 256, 512, 1024, 2k, 4k, 8k ]. -You might have a benchmark that depends on two inputs. For example, the +You might have a benchmark that depends on two or more inputs. For example, the following code defines a family of benchmarks for measuring the speed of set insertion. @@ -78,21 +78,21 @@ insertion. static void BM_SetInsert(benchmark::State& state) { while (state.KeepRunning()) { state.PauseTiming(); - std::set data = ConstructRandomSet(state.range_x()); + std::set data = ConstructRandomSet(state.range(0)); state.ResumeTiming(); - for (int j = 0; j < state.range_y(); ++j) + for (int j = 0; j < state.range(1); ++j) data.insert(RandomNumber()); } } BENCHMARK(BM_SetInsert) - ->ArgPair(1<<10, 1) - ->ArgPair(1<<10, 8) - ->ArgPair(1<<10, 64) - ->ArgPair(1<<10, 512) - ->ArgPair(8<<10, 1) - ->ArgPair(8<<10, 8) - ->ArgPair(8<<10, 64) - ->ArgPair(8<<10, 512); + ->Args({1<<10, 1}) + ->Args({1<<10, 8}) + ->Args({1<<10, 64}) + ->Args({1<<10, 512}) + ->Args({8<<10, 1}) + ->Args({8<<10, 8}) + ->Args({8<<10, 64}) + ->Args({8<<10, 512}); ``` The preceding code is quite repetitive, and can be replaced with the following @@ -101,7 +101,7 @@ product of the two specified ranges and will generate a benchmark for each such pair. ```c++ -BENCHMARK(BM_SetInsert)->RangePair(1<<10, 8<<10, 1, 512); +BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {1, 512}}); ``` For more complex patterns of inputs, passing a custom function to `Apply` allows @@ -113,7 +113,7 @@ and a sparse range on the second. static void CustomArguments(benchmark::internal::Benchmark* b) { for (int i = 0; i <= 10; ++i) for (int j = 32; j <= 1024*1024; j *= 8) - b->ArgPair(i, j); + b->Args({i, j}); } BENCHMARK(BM_SetInsert)->Apply(CustomArguments); ``` @@ -125,12 +125,12 @@ running time and the normalized root-mean square error of string comparison. ```c++ static void BM_StringCompare(benchmark::State& state) { - std::string s1(state.range_x(), '-'); - std::string s2(state.range_x(), '-'); + std::string s1(state.range(0), '-'); + std::string s2(state.range(0), '-'); while (state.KeepRunning()) { benchmark::DoNotOptimize(s1.compare(s2)); } - state.SetComplexityN(state.range_x()); + state.SetComplexityN(state.range(0)); } BENCHMARK(BM_StringCompare) ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::oN); @@ -162,14 +162,14 @@ template int BM_Sequential(benchmark::State& state) { Q q; typename Q::value_type v; while (state.KeepRunning()) { - for (int i = state.range_x(); i--; ) + for (int i = state.range(0); i--; ) q.push(v); - for (int e = state.range_x(); e--; ) + for (int e = state.range(0); e--; ) q.Wait(&v); } // actually messages, not bytes: state.SetBytesProcessed( - static_cast(state.iterations())*state.range_x()); + static_cast(state.iterations())*state.range(0)); } BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue)->Range(1<<0, 1<<10); ``` @@ -206,6 +206,34 @@ BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); Note that elements of `...args` may refer to global variables. Users should avoid modifying global state inside of a benchmark. +## Using RegisterBenchmark(name, fn, args...) + +The `RegisterBenchmark(name, func, args...)` function provides an alternative +way to create and register benchmarks. +`RegisterBenchmark(name, func, args...)` creates, registers, and returns a +pointer to a new benchmark with the specified `name` that invokes +`func(st, args...)` where `st` is a `benchmark::State` object. + +Unlike the `BENCHMARK` registration macros, which can only be used at the global +scope, the `RegisterBenchmark` can be called anywhere. This allows for +benchmark tests to be registered programmatically. + +Additionally `RegisterBenchmark` allows any callable object to be registered +as a benchmark. Including capturing lambdas and function objects. This +allows the creation + +For Example: +```c++ +auto BM_test = [](benchmark::State& st, auto Inputs) { /* ... */ }; + +int main(int argc, char** argv) { + for (auto& test_input : { /* ... */ }) + benchmark::RegisterBenchmark(test_input.name(), BM_test, test_input); + benchmark::Initialize(&argc, argv); + benchmark::RunSpecifiedBenchmarks(); +} +``` + ### Multithreaded benchmarks In a multithreaded test (benchmark invoked by multiple threads simultaneously), it is guaranteed that none of the threads will start until all have called @@ -256,7 +284,7 @@ can be reported back with `SetIterationTime`. ```c++ static void BM_ManualTiming(benchmark::State& state) { - int microseconds = state.range_x(); + int microseconds = state.range(0); std::chrono::duration sleep_duration { static_cast(microseconds) }; @@ -427,10 +455,10 @@ static void BM_test(benchmark::State& state) { ## Output Formats The library supports multiple output formats. Use the -`--benchmark_format=` flag to set the format type. `tabular` is -the default format. +`--benchmark_format=` flag to set the format type. `console` +is the default format. -The Tabular format is intended to be a human readable format. By default +The Console format is intended to be a human readable format. By default the format generates color output. Context is output on stderr and the tabular data on stdout. Example tabular output looks like: ``` @@ -493,6 +521,12 @@ name,iterations,real_time,cpu_time,bytes_per_second,items_per_second,label "BM_SetInsert/1024/10",106365,17238.4,8421.53,4.74973e+06,1.18743e+06, ``` +## Output Files +The library supports writing the output of the benchmark to a file specified +by `--benchmark_out=`. The format of the output can be specified +using `--benchmark_out_format={json|console|csv}`. Specifying +`--benchmark_out` does not suppress the console output. + ## Debug vs Release By default, benchmark builds as a debug library. You will see a warning in the output when this is the case. To build it as a release library instead, use: @@ -507,4 +541,22 @@ cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_LTO=true ``` ## Linking against the library -When using gcc, it is necessary to link against pthread to avoid runtime exceptions. This is due to how gcc implements std::thread. See [issue #67](https://github.com/google/benchmark/issues/67) for more details. +When using gcc, it is necessary to link against pthread to avoid runtime exceptions. +This is due to how gcc implements std::thread. +See [issue #67](https://github.com/google/benchmark/issues/67) for more details. + +## Compiler Support + +Google Benchmark uses C++11 when building the library. As such we require +a modern C++ toolchain, both compiler and standard library. + +The following minimum versions are strongly recommended build the library: + +* GCC 4.8 +* Clang 3.4 +* Visual Studio 2013 + +Anything older *may* work. + +Note: Using the library and its headers in C++03 is supported. C++11 is only +required to build the library. diff --git a/libcxx/utils/google-benchmark/include/benchmark/benchmark_api.h b/libcxx/utils/google-benchmark/include/benchmark/benchmark_api.h index 664ca2a..1a481ac 100644 --- a/libcxx/utils/google-benchmark/include/benchmark/benchmark_api.h +++ b/libcxx/utils/google-benchmark/include/benchmark/benchmark_api.h @@ -38,12 +38,12 @@ int main(int argc, char** argv) { // of memcpy() calls of different lengths: static void BM_memcpy(benchmark::State& state) { - char* src = new char[state.range_x()]; char* dst = new char[state.range_x()]; - memset(src, 'x', state.range_x()); + char* src = new char[state.range(0)]; char* dst = new char[state.range(0)]; + memset(src, 'x', state.range(0)); while (state.KeepRunning()) - memcpy(dst, src, state.range_x()); + memcpy(dst, src, state.range(0)); state.SetBytesProcessed(int64_t(state.iterations()) * - int64_t(state.range_x())); + int64_t(state.range(0))); delete[] src; delete[] dst; } BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10); @@ -60,27 +60,27 @@ BENCHMARK(BM_memcpy)->Range(8, 8<<10); static void BM_SetInsert(benchmark::State& state) { while (state.KeepRunning()) { state.PauseTiming(); - set data = ConstructRandomSet(state.range_x()); + set data = ConstructRandomSet(state.range(0)); state.ResumeTiming(); - for (int j = 0; j < state.range_y(); ++j) + for (int j = 0; j < state.range(1); ++j) data.insert(RandomNumber()); } } BENCHMARK(BM_SetInsert) - ->ArgPair(1<<10, 1) - ->ArgPair(1<<10, 8) - ->ArgPair(1<<10, 64) - ->ArgPair(1<<10, 512) - ->ArgPair(8<<10, 1) - ->ArgPair(8<<10, 8) - ->ArgPair(8<<10, 64) - ->ArgPair(8<<10, 512); + ->Args({1<<10, 1}) + ->Args({1<<10, 8}) + ->Args({1<<10, 64}) + ->Args({1<<10, 512}) + ->Args({8<<10, 1}) + ->Args({8<<10, 8}) + ->Args({8<<10, 64}) + ->Args({8<<10, 512}); // The preceding code is quite repetitive, and can be replaced with // the following short-hand. The following macro will pick a few // appropriate arguments in the product of the two specified ranges // and will generate a microbenchmark for each such pair. -BENCHMARK(BM_SetInsert)->RangePair(1<<10, 8<<10, 1, 512); +BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {1, 512}}); // For more complex patterns of inputs, passing a custom function // to Apply allows programmatic specification of an @@ -90,7 +90,7 @@ BENCHMARK(BM_SetInsert)->RangePair(1<<10, 8<<10, 1, 512); static void CustomArguments(benchmark::internal::Benchmark* b) { for (int i = 0; i <= 10; ++i) for (int j = 32; j <= 1024*1024; j *= 8) - b->ArgPair(i, j); + b->Args({i, j}); } BENCHMARK(BM_SetInsert)->Apply(CustomArguments); @@ -101,14 +101,14 @@ template int BM_Sequential(benchmark::State& state) { Q q; typename Q::value_type v; while (state.KeepRunning()) { - for (int i = state.range_x(); i--; ) + for (int i = state.range(0); i--; ) q.push(v); - for (int e = state.range_x(); e--; ) + for (int e = state.range(0); e--; ) q.Wait(&v); } // actually messages, not bytes: state.SetBytesProcessed( - static_cast(state.iterations())*state.range_x()); + static_cast(state.iterations())*state.range(0)); } BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue)->Range(1<<0, 1<<10); @@ -153,8 +153,15 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #include #include +#include + #include "macros.h" +#if defined(BENCHMARK_HAS_CXX11) +#include +#include +#endif + namespace benchmark { class BenchmarkReporter; @@ -165,11 +172,16 @@ void Initialize(int* argc, char** argv); // of each matching benchmark. Otherwise run each matching benchmark and // report the results. // -// The second overload reports the results using the specified 'reporter'. +// The second and third overload use the specified 'console_reporter' and +// 'file_reporter' respectively. 'file_reporter' will write to the file specified +// by '--benchmark_output'. If '--benchmark_output' is not given the +// 'file_reporter' is ignored. // // RETURNS: The number of matching benchmarks. size_t RunSpecifiedBenchmarks(); -size_t RunSpecifiedBenchmarks(BenchmarkReporter* reporter); +size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter); +size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter, + BenchmarkReporter* file_reporter); // If this routine is called, peak memory allocation past this point in the @@ -258,7 +270,7 @@ typedef double(BigOFunc)(int); // benchmark to use. class State { public: - State(size_t max_iters, bool has_x, int x, bool has_y, int y, + State(size_t max_iters, const std::vector& ranges, int thread_i, int n_threads); // Returns true if the benchmark should continue through another iteration. @@ -367,7 +379,7 @@ public: } BENCHMARK_ALWAYS_INLINE - size_t complexity_length_n() { + int complexity_length_n() { return complexity_n_; } @@ -413,17 +425,9 @@ public: // Range arguments for this run. CHECKs if the argument has been set. BENCHMARK_ALWAYS_INLINE - int range_x() const { - assert(has_range_x_); - ((void)has_range_x_); // Prevent unused warning. - return range_x_; - } - - BENCHMARK_ALWAYS_INLINE - int range_y() const { - assert(has_range_y_); - ((void)has_range_y_); // Prevent unused warning. - return range_y_; + int range(std::size_t pos) const { + assert(range_.size() > pos); + return range_[pos]; } BENCHMARK_ALWAYS_INLINE @@ -434,11 +438,7 @@ private: bool finished_; size_t total_iterations_; - bool has_range_x_; - int range_x_; - - bool has_range_y_; - int range_y_; + std::vector range_; size_t bytes_processed_; size_t items_processed_; @@ -489,24 +489,22 @@ public: // REQUIRES: The function passed to the constructor must accept an arg1. Benchmark* Range(int start, int limit); - // Run this benchmark once for every value in the range [start..limit] + // Run this benchmark once for all values in the range [start..limit] with specific step // REQUIRES: The function passed to the constructor must accept an arg1. - Benchmark* DenseRange(int start, int limit); + Benchmark* DenseRange(int start, int limit, int step = 1); - // Run this benchmark once with "x,y" as the extra arguments passed + // Run this benchmark once with "args" as the extra arguments passed // to the function. - // REQUIRES: The function passed to the constructor must accept arg1,arg2. - Benchmark* ArgPair(int x, int y); + // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... + Benchmark* Args(const std::vector& args); - // Pick a set of values A from the range [lo1..hi1] and a set - // of values B from the range [lo2..hi2]. Run the benchmark for - // every pair of values in the cartesian product of A and B - // (i.e., for all combinations of the values in A and B). - // REQUIRES: The function passed to the constructor must accept arg1,arg2. - Benchmark* RangePair(int lo1, int hi1, int lo2, int hi2); + // Run this benchmark once for a number of values picked from the + // ranges [start..limit]. (starts and limits are always picked.) + // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... + Benchmark* Ranges(const std::vector >& ranges); // Pass this benchmark object to *func, which can customize - // the benchmark by calling various methods like Arg, ArgPair, + // the benchmark by calling various methods like Arg, Args, // Threads, etc. Benchmark* Apply(void (*func)(Benchmark* benchmark)); @@ -587,6 +585,20 @@ private: Benchmark& operator=(Benchmark const&); }; +} // namespace internal + +// Create and register a benchmark with the specified 'name' that invokes +// the specified functor 'fn'. +// +// RETURNS: A pointer to the registered benchmark. +internal::Benchmark* RegisterBenchmark(const char* name, internal::Function* fn); + +#if defined(BENCHMARK_HAS_CXX11) +template +internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn); +#endif + +namespace internal { // The class used to hold all Benchmarks created from static function. // (ie those created using the BENCHMARK(...) macros. class FunctionBenchmark : public Benchmark { @@ -600,8 +612,57 @@ private: Function* func_; }; +#ifdef BENCHMARK_HAS_CXX11 +template +class LambdaBenchmark : public Benchmark { +public: + virtual void Run(State& st) { lambda_(st); } + +private: + template + LambdaBenchmark(const char* name, OLambda&& lam) + : Benchmark(name), lambda_(std::forward(lam)) {} + + LambdaBenchmark(LambdaBenchmark const&) = delete; + +private: + template + friend Benchmark* ::benchmark::RegisterBenchmark(const char*, Lam&&); + + Lambda lambda_; +}; +#endif + } // end namespace internal +inline internal::Benchmark* +RegisterBenchmark(const char* name, internal::Function* fn) { + return internal::RegisterBenchmarkInternal( + ::new internal::FunctionBenchmark(name, fn)); +} + +#ifdef BENCHMARK_HAS_CXX11 +template +internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn) { + using BenchType = internal::LambdaBenchmark::type>; + return internal::RegisterBenchmarkInternal( + ::new BenchType(name, std::forward(fn))); +} +#endif + +#if defined(BENCHMARK_HAS_CXX11) && \ + (!defined(BENCHMARK_GCC_VERSION) || BENCHMARK_GCC_VERSION >= 409) +template +internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn, + Args&&... args) { + return benchmark::RegisterBenchmark(name, + [=](benchmark::State& st) { fn(st, args...); }); +} +#else +#define BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK +#endif + + // The base class for all fixture tests. class Fixture: public internal::Benchmark { public: @@ -652,11 +713,11 @@ protected: // Old-style macros #define BENCHMARK_WITH_ARG(n, a) BENCHMARK(n)->Arg((a)) -#define BENCHMARK_WITH_ARG2(n, a1, a2) BENCHMARK(n)->ArgPair((a1), (a2)) +#define BENCHMARK_WITH_ARG2(n, a1, a2) BENCHMARK(n)->Args({(a1), (a2)}) #define BENCHMARK_WITH_UNIT(n, t) BENCHMARK(n)->Unit((t)) #define BENCHMARK_RANGE(n, lo, hi) BENCHMARK(n)->Range((lo), (hi)) #define BENCHMARK_RANGE2(n, l1, h1, l2, h2) \ - BENCHMARK(n)->RangePair((l1), (h1), (l2), (h2)) + BENCHMARK(n)->RangePair({{(l1), (h1)}, {(l2), (h2)}}) #if __cplusplus >= 201103L diff --git a/libcxx/utils/google-benchmark/include/benchmark/macros.h b/libcxx/utils/google-benchmark/include/benchmark/macros.h index 09d13c1..5892ac4 100644 --- a/libcxx/utils/google-benchmark/include/benchmark/macros.h +++ b/libcxx/utils/google-benchmark/include/benchmark/macros.h @@ -14,7 +14,11 @@ #ifndef BENCHMARK_MACROS_H_ #define BENCHMARK_MACROS_H_ -#if __cplusplus < 201103L +#if __cplusplus >= 201103L +#define BENCHMARK_HAS_CXX11 +#endif + +#ifndef BENCHMARK_HAS_CXX11 # define BENCHMARK_DISALLOW_COPY_AND_ASSIGN(TypeName) \ TypeName(const TypeName&); \ TypeName& operator=(const TypeName&) @@ -53,4 +57,8 @@ # define BENCHMARK_BUILTIN_EXPECT(x, y) x #endif +#if defined(__GNUC__) && !defined(__clang__) +#define BENCHMARK_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#endif + #endif // BENCHMARK_MACROS_H_ diff --git a/libcxx/utils/google-benchmark/include/benchmark/reporter.h b/libcxx/utils/google-benchmark/include/benchmark/reporter.h index 22c97a0..6d36c41 100644 --- a/libcxx/utils/google-benchmark/include/benchmark/reporter.h +++ b/libcxx/utils/google-benchmark/include/benchmark/reporter.h @@ -156,14 +156,23 @@ class BenchmarkReporter { // Simple reporter that outputs benchmark data to the console. This is the // default reporter used by RunSpecifiedBenchmarks(). class ConsoleReporter : public BenchmarkReporter { - public: +public: + enum OutputOptions { + OO_None, + OO_Color + }; + explicit ConsoleReporter(OutputOptions color_output = OO_Color) + : color_output_(color_output == OO_Color) {} + virtual bool ReportContext(const Context& context); virtual void ReportRuns(const std::vector& reports); - protected: +protected: virtual void PrintRunData(const Run& report); - size_t name_field_width_; + +private: + bool color_output_; }; class JSONReporter : public BenchmarkReporter { diff --git a/libcxx/utils/google-benchmark/src/benchmark.cc b/libcxx/utils/google-benchmark/src/benchmark.cc index cb8e132..515abd7 100644 --- a/libcxx/utils/google-benchmark/src/benchmark.cc +++ b/libcxx/utils/google-benchmark/src/benchmark.cc @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -69,6 +70,12 @@ DEFINE_string(benchmark_format, "console", "The format to use for console output. Valid values are " "'console', 'json', or 'csv'."); +DEFINE_string(benchmark_out_format, "json", + "The format to use for file output. Valid values are " + "'console', 'json', or 'csv'."); + +DEFINE_string(benchmark_out, "", "The file to write additonal output to"); + DEFINE_bool(color_print, true, "Enables colorized logging."); DEFINE_int32(v, 0, "The level of verbose logging to output"); @@ -306,23 +313,20 @@ namespace internal { // Information kept per benchmark we may want to run struct Benchmark::Instance { - std::string name; - Benchmark* benchmark; - bool has_arg1; - int arg1; - bool has_arg2; - int arg2; - TimeUnit time_unit; - int range_multiplier; - bool use_real_time; - bool use_manual_time; - BigO complexity; - BigOFunc* complexity_lambda; - bool last_benchmark_instance; - int repetitions; - double min_time; - int threads; // Number of concurrent threads to use - bool multithreaded; // Is benchmark multi-threaded? + std::string name; + Benchmark* benchmark; + std::vector arg; + TimeUnit time_unit; + int range_multiplier; + bool use_real_time; + bool use_manual_time; + BigO complexity; + BigOFunc* complexity_lambda; + bool last_benchmark_instance; + int repetitions; + double min_time; + int threads; // Number of concurrent threads to use + bool multithreaded; // Is benchmark multi-threaded? }; // Class for managing registered benchmarks. Note that each registered @@ -354,9 +358,9 @@ public: void Arg(int x); void Unit(TimeUnit unit); void Range(int start, int limit); - void DenseRange(int start, int limit); - void ArgPair(int start, int limit); - void RangePair(int lo1, int hi1, int lo2, int hi2); + void DenseRange(int start, int limit, int step = 1); + void Args(const std::vector& args); + void Ranges(const std::vector>& ranges); void RangeMultiplier(int multiplier); void MinTime(double n); void Repetitions(int n); @@ -371,12 +375,13 @@ public: static void AddRange(std::vector* dst, int lo, int hi, int mult); + int ArgsCnt() const { return args_.empty() ? -1 : static_cast(args_.front().size()); } + private: friend class BenchmarkFamilies; std::string name_; - int arg_count_; - std::vector< std::pair > args_; // Args for all benchmark runs + std::vector< std::vector > args_; // Args for all benchmark runs TimeUnit time_unit_; int range_multiplier_; double min_time_; @@ -424,10 +429,10 @@ bool BenchmarkFamilies::FindBenchmarks( if (!bench_family) continue; BenchmarkImp* family = bench_family->imp_; - if (family->arg_count_ == -1) { - family->arg_count_ = 0; - family->args_.emplace_back(-1, -1); + if (family->ArgsCnt() == -1) { + family->Args({}); } + for (auto const& args : family->args_) { const std::vector* thread_counts = (family->thread_counts_.empty() @@ -438,10 +443,7 @@ bool BenchmarkFamilies::FindBenchmarks( Benchmark::Instance instance; instance.name = family->name_; instance.benchmark = bench_family.get(); - instance.has_arg1 = family->arg_count_ >= 1; - instance.arg1 = args.first; - instance.has_arg2 = family->arg_count_ == 2; - instance.arg2 = args.second; + instance.arg = args; instance.time_unit = family->time_unit_; instance.range_multiplier = family->range_multiplier_; instance.min_time = family->min_time_; @@ -454,12 +456,10 @@ bool BenchmarkFamilies::FindBenchmarks( instance.multithreaded = !(family->thread_counts_.empty()); // Add arguments to instance name - if (family->arg_count_ >= 1) { - AppendHumanReadable(instance.arg1, &instance.name); - } - if (family->arg_count_ >= 2) { - AppendHumanReadable(instance.arg2, &instance.name); + for (auto const& arg : args) { + AppendHumanReadable(arg, &instance.name); } + if (!IsZero(family->min_time_)) { instance.name += StringPrintF("/min_time:%0.3f", family->min_time_); } @@ -488,7 +488,7 @@ bool BenchmarkFamilies::FindBenchmarks( } BenchmarkImp::BenchmarkImp(const char* name) - : name_(name), arg_count_(-1), time_unit_(kNanosecond), + : name_(name), time_unit_(kNanosecond), range_multiplier_(kRangeMultiplier), min_time_(0.0), repetitions_(0), use_real_time_(false), use_manual_time_(false), complexity_(oNone) { @@ -498,9 +498,8 @@ BenchmarkImp::~BenchmarkImp() { } void BenchmarkImp::Arg(int x) { - CHECK(arg_count_ == -1 || arg_count_ == 1); - arg_count_ = 1; - args_.emplace_back(x, -1); + CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); + args_.push_back({x}); } void BenchmarkImp::Unit(TimeUnit unit) { @@ -508,42 +507,54 @@ void BenchmarkImp::Unit(TimeUnit unit) { } void BenchmarkImp::Range(int start, int limit) { - CHECK(arg_count_ == -1 || arg_count_ == 1); - arg_count_ = 1; + CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); std::vector arglist; AddRange(&arglist, start, limit, range_multiplier_); for (int i : arglist) { - args_.emplace_back(i, -1); + args_.push_back({i}); } } -void BenchmarkImp::DenseRange(int start, int limit) { - CHECK(arg_count_ == -1 || arg_count_ == 1); - arg_count_ = 1; +void BenchmarkImp::DenseRange(int start, int limit, int step) { + CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); CHECK_GE(start, 0); CHECK_LE(start, limit); - for (int arg = start; arg <= limit; arg++) { - args_.emplace_back(arg, -1); + for (int arg = start; arg <= limit; arg+= step) { + args_.push_back({arg}); } } -void BenchmarkImp::ArgPair(int x, int y) { - CHECK(arg_count_ == -1 || arg_count_ == 2); - arg_count_ = 2; - args_.emplace_back(x, y); +void BenchmarkImp::Args(const std::vector& args) +{ + args_.push_back(args); } -void BenchmarkImp::RangePair(int lo1, int hi1, int lo2, int hi2) { - CHECK(arg_count_ == -1 || arg_count_ == 2); - arg_count_ = 2; - std::vector arglist1, arglist2; - AddRange(&arglist1, lo1, hi1, range_multiplier_); - AddRange(&arglist2, lo2, hi2, range_multiplier_); +void BenchmarkImp::Ranges(const std::vector>& ranges) { + std::vector> arglists(ranges.size()); + int total = 1; + for (std::size_t i = 0; i < ranges.size(); i++) { + AddRange(&arglists[i], ranges[i].first, ranges[i].second, range_multiplier_); + total *= arglists[i].size(); + } + + std::vector ctr(total, 0); - for (int i : arglist1) { - for (int j : arglist2) { - args_.emplace_back(i, j); + for (int i = 0; i < total; i++) { + std::vector tmp; + + for (std::size_t j = 0; j < arglists.size(); j++) { + tmp.push_back(arglists[j][ctr[j]]); + } + + args_.push_back(tmp); + + for (std::size_t j = 0; j < arglists.size(); j++) { + if (ctr[j] + 1 < arglists[j].size()) { + ++ctr[j]; + break; + } + ctr[j] = 0; } } } @@ -641,6 +652,7 @@ Benchmark::Benchmark(Benchmark const& other) } Benchmark* Benchmark::Arg(int x) { + CHECK(imp_->ArgsCnt() == -1 || imp_->ArgsCnt() == 1); imp_->Arg(x); return this; } @@ -651,22 +663,27 @@ Benchmark* Benchmark::Unit(TimeUnit unit) { } Benchmark* Benchmark::Range(int start, int limit) { + CHECK(imp_->ArgsCnt() == -1 || imp_->ArgsCnt() == 1); imp_->Range(start, limit); return this; } -Benchmark* Benchmark::DenseRange(int start, int limit) { - imp_->DenseRange(start, limit); +Benchmark* Benchmark::Ranges(const std::vector>& ranges) +{ + CHECK(imp_->ArgsCnt() == -1 || imp_->ArgsCnt() == static_cast(ranges.size())); + imp_->Ranges(ranges); return this; } -Benchmark* Benchmark::ArgPair(int x, int y) { - imp_->ArgPair(x, y); +Benchmark* Benchmark::DenseRange(int start, int limit, int step) { + CHECK(imp_->ArgsCnt() == -1 || imp_->ArgsCnt() == 1); + imp_->DenseRange(start, limit, step); return this; } -Benchmark* Benchmark::RangePair(int lo1, int hi1, int lo2, int hi2) { - imp_->RangePair(lo1, hi1, lo2, hi2); +Benchmark* Benchmark::Args(const std::vector& args) { + CHECK(imp_->ArgsCnt() == -1 || imp_->ArgsCnt() == static_cast(args.size())); + imp_->Args(args); return this; } @@ -744,7 +761,7 @@ namespace { void RunInThread(const benchmark::internal::Benchmark::Instance* b, size_t iters, int thread_id, ThreadStats* total) EXCLUDES(GetBenchmarkLock()) { - State st(iters, b->has_arg1, b->arg1, b->has_arg2, b->arg2, thread_id, b->threads); + State st(iters, b->arg, thread_id, b->threads); b->benchmark->Run(st); CHECK(st.iterations() == st.max_iterations) << "Benchmark returned before State::KeepRunning() returned false!"; @@ -758,14 +775,13 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b, timer_manager->Finalize(); } -void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, - BenchmarkReporter* br, - std::vector& complexity_reports) +std::vector +RunBenchmark(const benchmark::internal::Benchmark::Instance& b, + std::vector* complexity_reports) EXCLUDES(GetBenchmarkLock()) { + std::vector reports; // return value size_t iters = 1; - std::vector reports; - std::vector pool; if (b.multithreaded) pool.resize(b.threads); @@ -872,7 +888,7 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, report.complexity = b.complexity; report.complexity_lambda = b.complexity_lambda; if(report.complexity != oNone) - complexity_reports.push_back(report); + complexity_reports->push_back(report); } reports.push_back(report); @@ -903,27 +919,26 @@ void RunBenchmark(const benchmark::internal::Benchmark::Instance& b, additional_run_stats.end()); if((b.complexity != oNone) && b.last_benchmark_instance) { - additional_run_stats = ComputeBigO(complexity_reports); + additional_run_stats = ComputeBigO(*complexity_reports); reports.insert(reports.end(), additional_run_stats.begin(), additional_run_stats.end()); - complexity_reports.clear(); + complexity_reports->clear(); } - br->ReportRuns(reports); - if (b.multithreaded) { for (std::thread& thread : pool) thread.join(); } + + return reports; } } // namespace -State::State(size_t max_iters, bool has_x, int x, bool has_y, int y, +State::State(size_t max_iters, const std::vector& ranges, int thread_i, int n_threads) : started_(false), finished_(false), total_iterations_(0), - has_range_x_(has_x), range_x_(x), - has_range_y_(has_y), range_y_(y), + range_(ranges), bytes_processed_(0), items_processed_(0), complexity_n_(0), error_occurred_(false), @@ -975,8 +990,10 @@ namespace internal { namespace { void RunMatchingBenchmarks(const std::vector& benchmarks, - BenchmarkReporter* reporter) { - CHECK(reporter != nullptr); + BenchmarkReporter* console_reporter, + BenchmarkReporter* file_reporter) { + // Note the file_reporter can be null. + CHECK(console_reporter != nullptr); // Determine the width of the name field using a minimum width of 10. bool has_repetitions = FLAGS_benchmark_repetitions > 1; @@ -1000,23 +1017,30 @@ void RunMatchingBenchmarks(const std::vector& benchmarks, // Keep track of runing times of all instances of current benchmark std::vector complexity_reports; - if (reporter->ReportContext(context)) { + if (console_reporter->ReportContext(context) + && (!file_reporter || file_reporter->ReportContext(context))) { for (const auto& benchmark : benchmarks) { - RunBenchmark(benchmark, reporter, complexity_reports); + std::vector reports = + RunBenchmark(benchmark, &complexity_reports); + console_reporter->ReportRuns(reports); + if (file_reporter) file_reporter->ReportRuns(reports); } } + console_reporter->Finalize(); + if (file_reporter) file_reporter->Finalize(); } -std::unique_ptr GetDefaultReporter() { +std::unique_ptr +CreateReporter(std::string const& name, ConsoleReporter::OutputOptions allow_color) { typedef std::unique_ptr PtrType; - if (FLAGS_benchmark_format == "console") { - return PtrType(new ConsoleReporter); - } else if (FLAGS_benchmark_format == "json") { + if (name == "console") { + return PtrType(new ConsoleReporter(allow_color)); + } else if (name == "json") { return PtrType(new JSONReporter); - } else if (FLAGS_benchmark_format == "csv") { + } else if (name == "csv") { return PtrType(new CSVReporter); } else { - std::cerr << "Unexpected format: '" << FLAGS_benchmark_format << "'\n"; + std::cerr << "Unexpected format: '" << name << "'\n"; std::exit(1); } } @@ -1025,10 +1049,17 @@ std::unique_ptr GetDefaultReporter() { } // end namespace internal size_t RunSpecifiedBenchmarks() { - return RunSpecifiedBenchmarks(nullptr); + return RunSpecifiedBenchmarks(nullptr, nullptr); +} + + +size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter) { + return RunSpecifiedBenchmarks(console_reporter, nullptr); } -size_t RunSpecifiedBenchmarks(BenchmarkReporter* reporter) { + +size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter, + BenchmarkReporter* file_reporter) { std::string spec = FLAGS_benchmark_filter; if (spec.empty() || spec == "all") spec = "."; // Regexp that matches all benchmarks @@ -1041,13 +1072,38 @@ size_t RunSpecifiedBenchmarks(BenchmarkReporter* reporter) { for (auto const& benchmark : benchmarks) std::cout << benchmark.name << "\n"; } else { - std::unique_ptr default_reporter; - if (!reporter) { - default_reporter = internal::GetDefaultReporter(); - reporter = default_reporter.get(); + // Setup the reporters + std::ofstream output_file; + std::unique_ptr default_console_reporter; + std::unique_ptr default_file_reporter; + if (!console_reporter) { + auto output_opts = FLAGS_color_print ? ConsoleReporter::OO_Color + : ConsoleReporter::OO_None; + default_console_reporter = internal::CreateReporter( + FLAGS_benchmark_format, output_opts); + console_reporter = default_console_reporter.get(); } - internal::RunMatchingBenchmarks(benchmarks, reporter); - reporter->Finalize(); + std::string const& fname = FLAGS_benchmark_out; + if (fname == "" && file_reporter) { + std::cerr << "A custom file reporter was provided but " + "--benchmark_out= was not specified." << std::endl; + std::exit(1); + } + if (fname != "") { + output_file.open(fname); + if (!output_file.is_open()) { + std::cerr << "invalid file name: '" << fname << std::endl; + std::exit(1); + } + if (!file_reporter) { + default_file_reporter = internal::CreateReporter( + FLAGS_benchmark_out_format, ConsoleReporter::OO_None); + file_reporter = default_file_reporter.get(); + } + file_reporter->SetOutputStream(&output_file); + file_reporter->SetErrorStream(&output_file); + } + internal::RunMatchingBenchmarks(benchmarks, console_reporter, file_reporter); } return benchmarks.size(); } @@ -1062,6 +1118,8 @@ void PrintUsageAndExit() { " [--benchmark_min_time=]\n" " [--benchmark_repetitions=]\n" " [--benchmark_format=]\n" + " [--benchmark_out=]\n" + " [--benchmark_out_format=]\n" " [--color_print={true|false}]\n" " [--v=]\n"); exit(0); @@ -1081,6 +1139,10 @@ void ParseCommandLineFlags(int* argc, char** argv) { &FLAGS_benchmark_repetitions) || ParseStringFlag(argv[i], "benchmark_format", &FLAGS_benchmark_format) || + ParseStringFlag(argv[i], "benchmark_out", + &FLAGS_benchmark_out) || + ParseStringFlag(argv[i], "benchmark_out_format", + &FLAGS_benchmark_out_format) || ParseBoolFlag(argv[i], "color_print", &FLAGS_color_print) || ParseInt32Flag(argv[i], "v", &FLAGS_v)) { @@ -1092,10 +1154,9 @@ void ParseCommandLineFlags(int* argc, char** argv) { PrintUsageAndExit(); } } - - if (FLAGS_benchmark_format != "console" && - FLAGS_benchmark_format != "json" && - FLAGS_benchmark_format != "csv") { + for (auto const* flag : {&FLAGS_benchmark_format, + &FLAGS_benchmark_out_format}) + if (*flag != "console" && *flag != "json" && *flag != "csv") { PrintUsageAndExit(); } } diff --git a/libcxx/utils/google-benchmark/src/colorprint.cc b/libcxx/utils/google-benchmark/src/colorprint.cc index efb8626..b7d316f 100644 --- a/libcxx/utils/google-benchmark/src/colorprint.cc +++ b/libcxx/utils/google-benchmark/src/colorprint.cc @@ -20,7 +20,6 @@ #include #include -#include "commandlineflags.h" #include "check.h" #include "internal_macros.h" @@ -28,8 +27,6 @@ #include #endif -DECLARE_bool(color_print); - namespace benchmark { namespace { #ifdef BENCHMARK_OS_WINDOWS @@ -120,14 +117,14 @@ std::string FormatString(const char *msg, ...) { void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, ...) { va_list args; va_start(args, fmt); + ColorPrintf(out, color, fmt, args); + va_end(args); +} - if (!FLAGS_color_print) { - out << FormatString(fmt, args); - va_end(args); - return; - } - +void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, va_list args) { #ifdef BENCHMARK_OS_WINDOWS + ((void)out); // suppress unused warning + const HANDLE stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE); // Gets the current text color. @@ -152,7 +149,6 @@ void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, ...) { out << FormatString(fmt, args) << "\033[m"; #endif - va_end(args); } } // end namespace benchmark diff --git a/libcxx/utils/google-benchmark/src/colorprint.h b/libcxx/utils/google-benchmark/src/colorprint.h index 2b3c082..d924795 100644 --- a/libcxx/utils/google-benchmark/src/colorprint.h +++ b/libcxx/utils/google-benchmark/src/colorprint.h @@ -20,6 +20,7 @@ enum LogColor { std::string FormatString(const char* msg, va_list args); std::string FormatString(const char* msg, ...); +void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, va_list args); void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, ...); } // end namespace benchmark diff --git a/libcxx/utils/google-benchmark/src/complexity.cc b/libcxx/utils/google-benchmark/src/complexity.cc index b42bd38..7d55794 100644 --- a/libcxx/utils/google-benchmark/src/complexity.cc +++ b/libcxx/utils/google-benchmark/src/complexity.cc @@ -31,9 +31,9 @@ BigOFunc* FittingCurve(BigO complexity) { case oN: return [](int n) -> double { return n; }; case oNSquared: - return [](int n) -> double { return n * n; }; + return [](int n) -> double { return std::pow(n, 2); }; case oNCubed: - return [](int n) -> double { return n * n * n; }; + return [](int n) -> double { return std::pow(n, 3); }; case oLogN: return [](int n) { return std::log2(n); }; case oNLogN: diff --git a/libcxx/utils/google-benchmark/src/console_reporter.cc b/libcxx/utils/google-benchmark/src/console_reporter.cc index 080c324..a663b93 100644 --- a/libcxx/utils/google-benchmark/src/console_reporter.cc +++ b/libcxx/utils/google-benchmark/src/console_reporter.cc @@ -30,8 +30,6 @@ #include "string_util.h" #include "walltime.h" -DECLARE_bool(color_print); - namespace benchmark { bool ConsoleReporter::ReportContext(const Context& context) { @@ -40,10 +38,10 @@ bool ConsoleReporter::ReportContext(const Context& context) { PrintBasicContext(&GetErrorStream(), context); #ifdef BENCHMARK_OS_WINDOWS - if (FLAGS_color_print && &std::cout != &GetOutputStream()) { + if (color_output_ && &std::cout != &GetOutputStream()) { GetErrorStream() << "Color printing is only supported for stdout on windows." " Disabling color printing\n"; - FLAGS_color_print = false; + color_output_ = false; } #endif std::string str = FormatString("%-*s %13s %13s %10s\n", @@ -59,18 +57,29 @@ void ConsoleReporter::ReportRuns(const std::vector& reports) { PrintRunData(run); } +static void IgnoreColorPrint(std::ostream& out, LogColor, + const char* fmt, ...) +{ + va_list args; + va_start(args, fmt); + out << FormatString(fmt, args); + va_end(args); +} + void ConsoleReporter::PrintRunData(const Run& result) { + typedef void(PrinterFn)(std::ostream&, LogColor, const char*, ...); auto& Out = GetOutputStream(); - + PrinterFn* printer = color_output_ ? (PrinterFn*)ColorPrintf + : IgnoreColorPrint; auto name_color = (result.report_big_o || result.report_rms) ? COLOR_BLUE : COLOR_GREEN; - ColorPrintf(Out, name_color, "%-*s ", name_field_width_, + printer(Out, name_color, "%-*s ", name_field_width_, result.benchmark_name.c_str()); if (result.error_occurred) { - ColorPrintf(Out, COLOR_RED, "ERROR OCCURRED: \'%s\'", + printer(Out, COLOR_RED, "ERROR OCCURRED: \'%s\'", result.error_message.c_str()); - ColorPrintf(Out, COLOR_DEFAULT, "\n"); + printer(Out, COLOR_DEFAULT, "\n"); return; } // Format bytes per second @@ -91,34 +100,34 @@ void ConsoleReporter::PrintRunData(const Run& result) { if (result.report_big_o) { std::string big_o = GetBigOString(result.complexity); - ColorPrintf(Out, COLOR_YELLOW, "%10.2f %s %10.2f %s ", real_time, + printer(Out, COLOR_YELLOW, "%10.2f %s %10.2f %s ", real_time, big_o.c_str(), cpu_time, big_o.c_str()); } else if (result.report_rms) { - ColorPrintf(Out, COLOR_YELLOW, "%10.0f %% %10.0f %% ", real_time * 100, + printer(Out, COLOR_YELLOW, "%10.0f %% %10.0f %% ", real_time * 100, cpu_time * 100); } else { const char* timeLabel = GetTimeUnitString(result.time_unit); - ColorPrintf(Out, COLOR_YELLOW, "%10.0f %s %10.0f %s ", real_time, timeLabel, + printer(Out, COLOR_YELLOW, "%10.0f %s %10.0f %s ", real_time, timeLabel, cpu_time, timeLabel); } if (!result.report_big_o && !result.report_rms) { - ColorPrintf(Out, COLOR_CYAN, "%10lld", result.iterations); + printer(Out, COLOR_CYAN, "%10lld", result.iterations); } if (!rate.empty()) { - ColorPrintf(Out, COLOR_DEFAULT, " %*s", 13, rate.c_str()); + printer(Out, COLOR_DEFAULT, " %*s", 13, rate.c_str()); } if (!items.empty()) { - ColorPrintf(Out, COLOR_DEFAULT, " %*s", 18, items.c_str()); + printer(Out, COLOR_DEFAULT, " %*s", 18, items.c_str()); } if (!result.report_label.empty()) { - ColorPrintf(Out, COLOR_DEFAULT, " %s", result.report_label.c_str()); + printer(Out, COLOR_DEFAULT, " %s", result.report_label.c_str()); } - ColorPrintf(Out, COLOR_DEFAULT, "\n"); + printer(Out, COLOR_DEFAULT, "\n"); } } // end namespace benchmark diff --git a/libcxx/utils/google-benchmark/src/cycleclock.h b/libcxx/utils/google-benchmark/src/cycleclock.h index 3110804..e4825d4 100644 --- a/libcxx/utils/google-benchmark/src/cycleclock.h +++ b/libcxx/utils/google-benchmark/src/cycleclock.h @@ -113,11 +113,11 @@ inline BENCHMARK_ALWAYS_INLINE int64_t Now() { uint32_t pmuseren; uint32_t pmcntenset; // Read the user mode perf monitor counter access permissions. - asm("mrc p15, 0, %0, c9, c14, 0" : "=r"(pmuseren)); + asm volatile("mrc p15, 0, %0, c9, c14, 0" : "=r"(pmuseren)); if (pmuseren & 1) { // Allows reading perfmon counters for user mode code. - asm("mrc p15, 0, %0, c9, c12, 1" : "=r"(pmcntenset)); + asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r"(pmcntenset)); if (pmcntenset & 0x80000000ul) { // Is it counting? - asm("mrc p15, 0, %0, c9, c13, 0" : "=r"(pmccntr)); + asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r"(pmccntr)); // The counter is set up to count every 64th cycle return static_cast(pmccntr) * 64; // Should optimize to << 6 } diff --git a/libcxx/utils/google-benchmark/src/sysinfo.cc b/libcxx/utils/google-benchmark/src/sysinfo.cc index e10e19d..b5b9851 100644 --- a/libcxx/utils/google-benchmark/src/sysinfo.cc +++ b/libcxx/utils/google-benchmark/src/sysinfo.cc @@ -239,6 +239,7 @@ void InitializeSystemInfo() { } // TODO: also figure out cpuinfo_num_cpus + #elif defined BENCHMARK_OS_WINDOWS // In NT, read MHz from the registry. If we fail to do so or we're in win9x // then make a crude estimate. @@ -251,7 +252,12 @@ void InitializeSystemInfo() { cpuinfo_cycles_per_second = static_cast((int64_t)data * (int64_t)(1000 * 1000)); // was mhz else cpuinfo_cycles_per_second = static_cast(EstimateCyclesPerSecond()); -// TODO: also figure out cpuinfo_num_cpus + + SYSTEM_INFO sysinfo; + // Use memset as opposed to = {} to avoid GCC missing initializer false positives. + std::memset(&sysinfo, 0, sizeof(SYSTEM_INFO)); + GetSystemInfo(&sysinfo); + cpuinfo_num_cpus = sysinfo.dwNumberOfProcessors; // number of logical processors in the current group #elif defined BENCHMARK_OS_MACOSX // returning "mach time units" per second. the current number of elapsed diff --git a/libcxx/utils/google-benchmark/test/CMakeLists.txt b/libcxx/utils/google-benchmark/test/CMakeLists.txt index aeb720a..93855a3 100644 --- a/libcxx/utils/google-benchmark/test/CMakeLists.txt +++ b/libcxx/utils/google-benchmark/test/CMakeLists.txt @@ -45,9 +45,15 @@ add_test(donotoptimize_test donotoptimize_test --benchmark_min_time=0.01) compile_benchmark_test(fixture_test) add_test(fixture_test fixture_test --benchmark_min_time=0.01) +compile_benchmark_test(register_benchmark_test) +add_test(register_benchmark_test register_benchmark_test --benchmark_min_time=0.01) + compile_benchmark_test(map_test) add_test(map_test map_test --benchmark_min_time=0.01) +compile_benchmark_test(multiple_ranges_test) +add_test(multiple_ranges_test multiple_ranges_test --benchmark_min_time=0.01) + compile_benchmark_test(reporter_output_test) add_test(reporter_output_test reporter_output_test --benchmark_min_time=0.01) diff --git a/libcxx/utils/google-benchmark/test/basic_test.cc b/libcxx/utils/google-benchmark/test/basic_test.cc index 3435415..570ff22 100644 --- a/libcxx/utils/google-benchmark/test/basic_test.cc +++ b/libcxx/utils/google-benchmark/test/basic_test.cc @@ -14,7 +14,7 @@ BENCHMARK(BM_empty)->ThreadPerCpu(); void BM_spin_empty(benchmark::State& state) { while (state.KeepRunning()) { - for (int x = 0; x < state.range_x(); ++x) { + for (int x = 0; x < state.range(0); ++x) { benchmark::DoNotOptimize(x); } } @@ -23,11 +23,11 @@ BASIC_BENCHMARK_TEST(BM_spin_empty); BASIC_BENCHMARK_TEST(BM_spin_empty)->ThreadPerCpu(); void BM_spin_pause_before(benchmark::State& state) { - for (int i = 0; i < state.range_x(); ++i) { + for (int i = 0; i < state.range(0); ++i) { benchmark::DoNotOptimize(i); } while(state.KeepRunning()) { - for (int i = 0; i < state.range_x(); ++i) { + for (int i = 0; i < state.range(0); ++i) { benchmark::DoNotOptimize(i); } } @@ -39,11 +39,11 @@ BASIC_BENCHMARK_TEST(BM_spin_pause_before)->ThreadPerCpu(); void BM_spin_pause_during(benchmark::State& state) { while(state.KeepRunning()) { state.PauseTiming(); - for (int i = 0; i < state.range_x(); ++i) { + for (int i = 0; i < state.range(0); ++i) { benchmark::DoNotOptimize(i); } state.ResumeTiming(); - for (int i = 0; i < state.range_x(); ++i) { + for (int i = 0; i < state.range(0); ++i) { benchmark::DoNotOptimize(i); } } @@ -64,11 +64,11 @@ BENCHMARK(BM_pause_during)->UseRealTime()->ThreadPerCpu(); void BM_spin_pause_after(benchmark::State& state) { while(state.KeepRunning()) { - for (int i = 0; i < state.range_x(); ++i) { + for (int i = 0; i < state.range(0); ++i) { benchmark::DoNotOptimize(i); } } - for (int i = 0; i < state.range_x(); ++i) { + for (int i = 0; i < state.range(0); ++i) { benchmark::DoNotOptimize(i); } } @@ -77,15 +77,15 @@ BASIC_BENCHMARK_TEST(BM_spin_pause_after)->ThreadPerCpu(); void BM_spin_pause_before_and_after(benchmark::State& state) { - for (int i = 0; i < state.range_x(); ++i) { + for (int i = 0; i < state.range(0); ++i) { benchmark::DoNotOptimize(i); } while(state.KeepRunning()) { - for (int i = 0; i < state.range_x(); ++i) { + for (int i = 0; i < state.range(0); ++i) { benchmark::DoNotOptimize(i); } } - for (int i = 0; i < state.range_x(); ++i) { + for (int i = 0; i < state.range(0); ++i) { benchmark::DoNotOptimize(i); } } diff --git a/libcxx/utils/google-benchmark/test/benchmark_test.cc b/libcxx/utils/google-benchmark/test/benchmark_test.cc index 66f5956..fe7d82c 100644 --- a/libcxx/utils/google-benchmark/test/benchmark_test.cc +++ b/libcxx/utils/google-benchmark/test/benchmark_test.cc @@ -67,7 +67,7 @@ BENCHMARK(BM_Factorial)->UseRealTime(); static void BM_CalculatePiRange(benchmark::State& state) { double pi = 0.0; while (state.KeepRunning()) - pi = CalculatePi(state.range_x()); + pi = CalculatePi(state.range(0)); std::stringstream ss; ss << pi; state.SetLabel(ss.str()); @@ -87,25 +87,25 @@ BENCHMARK(BM_CalculatePi)->ThreadPerCpu(); static void BM_SetInsert(benchmark::State& state) { while (state.KeepRunning()) { state.PauseTiming(); - std::set data = ConstructRandomSet(state.range_x()); + std::set data = ConstructRandomSet(state.range(0)); state.ResumeTiming(); - for (int j = 0; j < state.range_y(); ++j) + for (int j = 0; j < state.range(1); ++j) data.insert(rand()); } - state.SetItemsProcessed(state.iterations() * state.range_y()); - state.SetBytesProcessed(state.iterations() * state.range_y() * sizeof(int)); + state.SetItemsProcessed(state.iterations() * state.range(1)); + state.SetBytesProcessed(state.iterations() * state.range(1) * sizeof(int)); } -BENCHMARK(BM_SetInsert)->RangePair(1<<10,8<<10, 1,10); +BENCHMARK(BM_SetInsert)->Ranges({{1<<10,8<<10}, {1,10}}); template static void BM_Sequential(benchmark::State& state) { ValueType v = 42; while (state.KeepRunning()) { Container c; - for (int i = state.range_x(); --i; ) + for (int i = state.range(0); --i; ) c.push_back(v); } - const size_t items_processed = state.iterations() * state.range_x(); + const size_t items_processed = state.iterations() * state.range(0); state.SetItemsProcessed(items_processed); state.SetBytesProcessed(items_processed * sizeof(v)); } @@ -117,8 +117,8 @@ BENCHMARK_TEMPLATE(BM_Sequential, std::vector, int)->Arg(512); #endif static void BM_StringCompare(benchmark::State& state) { - std::string s1(state.range_x(), '-'); - std::string s2(state.range_x(), '-'); + std::string s1(state.range(0), '-'); + std::string s2(state.range(0), '-'); while (state.KeepRunning()) benchmark::DoNotOptimize(s1.compare(s2)); } @@ -147,14 +147,14 @@ BENCHMARK(BM_SetupTeardown)->ThreadPerCpu(); static void BM_LongTest(benchmark::State& state) { double tracker = 0.0; while (state.KeepRunning()) { - for (int i = 0; i < state.range_x(); ++i) + for (int i = 0; i < state.range(0); ++i) benchmark::DoNotOptimize(tracker += i); } } BENCHMARK(BM_LongTest)->Range(1<<16,1<<28); static void BM_ParallelMemset(benchmark::State& state) { - int size = state.range_x() / sizeof(int); + int size = state.range(0) / sizeof(int); int thread_size = size / state.threads; int from = thread_size * state.thread_index; int to = from + thread_size; @@ -179,7 +179,7 @@ BENCHMARK(BM_ParallelMemset)->Arg(10 << 20)->ThreadRange(1, 4); static void BM_ManualTiming(benchmark::State& state) { size_t slept_for = 0; - int microseconds = state.range_x(); + int microseconds = state.range(0); std::chrono::duration sleep_duration { static_cast(microseconds) }; diff --git a/libcxx/utils/google-benchmark/test/complexity_test.cc b/libcxx/utils/google-benchmark/test/complexity_test.cc index 8ab88f9..db04183 100644 --- a/libcxx/utils/google-benchmark/test/complexity_test.cc +++ b/libcxx/utils/google-benchmark/test/complexity_test.cc @@ -36,18 +36,27 @@ struct TestCase { CHECK(err_str.empty()) << "Could not construct regex \"" << regex << "\"" << " got Error: " << err_str; + std::string near = ""; std::string line; + bool first = true; while (remaining_output.eof() == false) { CHECK(remaining_output.good()); std::getline(remaining_output, line); + // Keep the first line as context. + if (first) { + near = line; + first = false; + } if (r.Match(line)) return; CHECK(match_rule != MR_Next) << "Expected line \"" << line - << "\" to match regex \"" << regex << "\""; + << "\" to match regex \"" << regex << "\"" + << "\nstarted matching at line: \"" << near << "\""; } CHECK(remaining_output.eof() == false) << "End of output reached before match for regex \"" << regex - << "\" was found"; + << "\" was found" + << "\nstarted matching at line: \"" << near << "\""; } }; @@ -112,7 +121,7 @@ std::string join(First f, Args&&... args) { return std::string(std::move(f)) + "[ ]+" + join(std::forward(args)...); } -std::string dec_re = "[0-9]+\\.[0-9]+"; +std::string dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; #define ADD_COMPLEXITY_CASES(...) \ int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__) @@ -138,7 +147,7 @@ int AddComplexityTest(std::vector* console_out, std::vector* }); AddCases(csv_out, { {"^\"" + big_o_test_name + "\",," + dec_re + "," + dec_re + "," + big_o + ",,,,,$"}, - {"^\"" + rms_test_name + "\",," + dec_re + "," + dec_re + ",,,,,,$"} + {"^\"" + rms_test_name + "\",," + dec_re + "," + dec_re + ",,,,,,$", MR_Next} }); return 0; } @@ -151,12 +160,15 @@ int AddComplexityTest(std::vector* console_out, std::vector* void BM_Complexity_O1(benchmark::State& state) { while (state.KeepRunning()) { + for (int i=0; i < 1024; ++i) { + benchmark::DoNotOptimize(&i); + } } - state.SetComplexityN(state.range_x()); + state.SetComplexityN(state.range(0)); } BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(benchmark::o1); -BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity([](int){return 1.0; }); BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity(); +BENCHMARK(BM_Complexity_O1) -> Range(1, 1<<18) -> Complexity([](int){return 1.0; }); const char* big_o_1_test_name = "BM_Complexity_O1_BigO"; const char* rms_o_1_test_name = "BM_Complexity_O1_RMS"; @@ -167,6 +179,10 @@ const char* lambda_big_o_1 = "f\\(N\\)"; ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, big_o_1_test_name, rms_o_1_test_name, enum_auto_big_o_1); +// Add auto enum tests +ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, + big_o_1_test_name, rms_o_1_test_name, enum_auto_big_o_1); + // Add lambda tests ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, big_o_1_test_name, rms_o_1_test_name, lambda_big_o_1); @@ -185,12 +201,12 @@ std::vector ConstructRandomVector(int size) { } void BM_Complexity_O_N(benchmark::State& state) { - auto v = ConstructRandomVector(state.range_x()); - const int item_not_in_vector = state.range_x()*2; // Test worst case scenario (item not in vector) + auto v = ConstructRandomVector(state.range(0)); + const int item_not_in_vector = state.range(0)*2; // Test worst case scenario (item not in vector) while (state.KeepRunning()) { benchmark::DoNotOptimize(std::find(v.begin(), v.end(), item_not_in_vector)); } - state.SetComplexityN(state.range_x()); + state.SetComplexityN(state.range(0)); } BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oN); BENCHMARK(BM_Complexity_O_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](int n) -> double{return n; }); @@ -214,11 +230,11 @@ ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, // ========================================================================= // static void BM_Complexity_O_N_log_N(benchmark::State& state) { - auto v = ConstructRandomVector(state.range_x()); + auto v = ConstructRandomVector(state.range(0)); while (state.KeepRunning()) { std::sort(v.begin(), v.end()); } - state.SetComplexityN(state.range_x()); + state.SetComplexityN(state.range(0)); } BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity(benchmark::oNLogN); BENCHMARK(BM_Complexity_O_N_log_N) -> RangeMultiplier(2) -> Range(1<<10, 1<<16) -> Complexity([](int n) {return n * std::log2(n); }); @@ -244,14 +260,8 @@ ADD_COMPLEXITY_CASES(&ConsoleOutputTests, &JSONOutputTests, &CSVOutputTests, int main(int argc, char* argv[]) { - // Add --color_print=false to argv since we don't want to match color codes. - char new_arg[64]; - char* new_argv[64]; - std::copy(argv, argv + argc, new_argv); - new_argv[argc++] = std::strcpy(new_arg, "--color_print=false"); - benchmark::Initialize(&argc, new_argv); - - benchmark::ConsoleReporter CR; + benchmark::Initialize(&argc, argv); + benchmark::ConsoleReporter CR(benchmark::ConsoleReporter::OO_None); benchmark::JSONReporter JR; benchmark::CSVReporter CSVR; struct ReporterTest { diff --git a/libcxx/utils/google-benchmark/test/fixture_test.cc b/libcxx/utils/google-benchmark/test/fixture_test.cc index bf800fd..226bb2a 100644 --- a/libcxx/utils/google-benchmark/test/fixture_test.cc +++ b/libcxx/utils/google-benchmark/test/fixture_test.cc @@ -44,7 +44,7 @@ BENCHMARK_DEFINE_F(MyFixture, Bar)(benchmark::State& st) { assert(data.get() != nullptr); assert(*data == 42); } - st.SetItemsProcessed(st.range_x()); + st.SetItemsProcessed(st.range(0)); } BENCHMARK_REGISTER_F(MyFixture, Bar)->Arg(42); BENCHMARK_REGISTER_F(MyFixture, Bar)->Arg(42)->ThreadPerCpu(); diff --git a/libcxx/utils/google-benchmark/test/map_test.cc b/libcxx/utils/google-benchmark/test/map_test.cc index 5eccf8d..729c012 100644 --- a/libcxx/utils/google-benchmark/test/map_test.cc +++ b/libcxx/utils/google-benchmark/test/map_test.cc @@ -17,7 +17,7 @@ std::map ConstructRandomMap(int size) { // Basic version. static void BM_MapLookup(benchmark::State& state) { - const int size = state.range_x(); + const int size = state.range(0); while (state.KeepRunning()) { state.PauseTiming(); std::map m = ConstructRandomMap(size); @@ -34,7 +34,7 @@ BENCHMARK(BM_MapLookup)->Range(1 << 3, 1 << 12); class MapFixture : public ::benchmark::Fixture { public: void SetUp(const ::benchmark::State& st) { - m = ConstructRandomMap(st.range_x()); + m = ConstructRandomMap(st.range(0)); } void TearDown(const ::benchmark::State&) { @@ -45,7 +45,7 @@ class MapFixture : public ::benchmark::Fixture { }; BENCHMARK_DEFINE_F(MapFixture, Lookup)(benchmark::State& state) { - const int size = state.range_x(); + const int size = state.range(0); while (state.KeepRunning()) { for (int i = 0; i < size; ++i) { benchmark::DoNotOptimize(m.find(rand() % size)); diff --git a/libcxx/utils/google-benchmark/test/multiple_ranges_test.cc b/libcxx/utils/google-benchmark/test/multiple_ranges_test.cc new file mode 100644 index 0000000..b425acd --- /dev/null +++ b/libcxx/utils/google-benchmark/test/multiple_ranges_test.cc @@ -0,0 +1,46 @@ +#include "benchmark/benchmark.h" + +#include +#include + +class MultipleRangesFixture : public ::benchmark::Fixture { + public: + MultipleRangesFixture() + : expectedValues({ + {1, 3, 5}, {1, 3, 8}, {1, 3, 15}, {2, 3, 5}, {2, 3, 8}, {2, 3, 15}, + {1, 4, 5}, {1, 4, 8}, {1, 4, 15}, {2, 4, 5}, {2, 4, 8}, {2, 4, 15}, + {1, 7, 5}, {1, 7, 8}, {1, 7, 15}, {2, 7, 5}, {2, 7, 8}, {2, 7, 15}, + {7, 6, 3} + }) + { + } + + void SetUp(const ::benchmark::State& state) { + std::vector ranges = {state.range(0), state.range(1), state.range(2)}; + + assert(expectedValues.find(ranges) != expectedValues.end()); + + actualValues.insert(ranges); + } + + virtual ~MultipleRangesFixture() { + assert(actualValues.size() == expectedValues.size()); + } + + std::set> expectedValues; + std::set> actualValues; +}; + + +BENCHMARK_DEFINE_F(MultipleRangesFixture, Empty)(benchmark::State& state) { + while (state.KeepRunning()) { + int product = state.range(0) * state.range(1) * state.range(2); + for (int x = 0; x < product; x++) { + benchmark::DoNotOptimize(x); + } + } +} + +BENCHMARK_REGISTER_F(MultipleRangesFixture, Empty)->RangeMultiplier(2)->Ranges({{1, 2}, {3, 7}, {5, 15}})->Args({7, 6, 3}); + +BENCHMARK_MAIN() diff --git a/libcxx/utils/google-benchmark/test/options_test.cc b/libcxx/utils/google-benchmark/test/options_test.cc index 78cedae..06bc9c2 100644 --- a/libcxx/utils/google-benchmark/test/options_test.cc +++ b/libcxx/utils/google-benchmark/test/options_test.cc @@ -9,7 +9,7 @@ void BM_basic(benchmark::State& state) { } void BM_basic_slow(benchmark::State& state) { - std::chrono::milliseconds sleep_duration(state.range_x()); + std::chrono::milliseconds sleep_duration(state.range(0)); while (state.KeepRunning()) { std::this_thread::sleep_for( std::chrono::duration_cast(sleep_duration) @@ -25,8 +25,8 @@ BENCHMARK(BM_basic_slow)->Arg(1000)->Unit(benchmark::kMillisecond); BENCHMARK(BM_basic)->Range(1, 8); BENCHMARK(BM_basic)->RangeMultiplier(2)->Range(1, 8); BENCHMARK(BM_basic)->DenseRange(10, 15); -BENCHMARK(BM_basic)->ArgPair(42, 42); -BENCHMARK(BM_basic)->RangePair(64, 512, 64, 512); +BENCHMARK(BM_basic)->Args({42, 42}); +BENCHMARK(BM_basic)->Ranges({{64, 512}, {64, 512}}); BENCHMARK(BM_basic)->MinTime(0.7); BENCHMARK(BM_basic)->UseRealTime(); BENCHMARK(BM_basic)->ThreadRange(2, 4); diff --git a/libcxx/utils/google-benchmark/test/register_benchmark_test.cc b/libcxx/utils/google-benchmark/test/register_benchmark_test.cc new file mode 100644 index 0000000..f25ae85 --- /dev/null +++ b/libcxx/utils/google-benchmark/test/register_benchmark_test.cc @@ -0,0 +1,149 @@ + +#undef NDEBUG +#include "benchmark/benchmark.h" +#include "../src/check.h" // NOTE: check.h is for internal use only! +#include +#include + +namespace { + +class TestReporter : public benchmark::ConsoleReporter { +public: + virtual void ReportRuns(const std::vector& report) { + all_runs_.insert(all_runs_.end(), begin(report), end(report)); + ConsoleReporter::ReportRuns(report); + } + + std::vector all_runs_; +}; + +struct TestCase { + std::string name; + const char* label; + TestCase(const char* xname) : name(xname), label(nullptr) {} + TestCase(const char* xname, const char* xlabel) + : name(xname), label(xlabel) {} + + typedef benchmark::BenchmarkReporter::Run Run; + + void CheckRun(Run const& run) const { + CHECK(name == run.benchmark_name) << "expected " << name + << " got " << run.benchmark_name; + if (label) { + CHECK(run.report_label == label) << "expected " << label + << " got " << run.report_label; + } else { + CHECK(run.report_label == ""); + } + } +}; + +std::vector ExpectedResults; + +int AddCases(std::initializer_list const& v) { + for (auto N : v) { + ExpectedResults.push_back(N); + } + return 0; +} + +#define CONCAT(x, y) CONCAT2(x, y) +#define CONCAT2(x, y) x##y +#define ADD_CASES(...) \ +int CONCAT(dummy, __LINE__) = AddCases({__VA_ARGS__}) + +} // end namespace + +typedef benchmark::internal::Benchmark* ReturnVal; + +//----------------------------------------------------------------------------// +// Test RegisterBenchmark with no additional arguments +//----------------------------------------------------------------------------// +void BM_function(benchmark::State& state) { while (state.KeepRunning()) {} } +BENCHMARK(BM_function); +ReturnVal dummy = benchmark::RegisterBenchmark( + "BM_function_manual_registration", + BM_function); +ADD_CASES({"BM_function"}, {"BM_function_manual_registration"}); + +//----------------------------------------------------------------------------// +// Test RegisterBenchmark with additional arguments +// Note: GCC <= 4.8 do not support this form of RegisterBenchmark because they +// reject the variadic pack expansion of lambda captures. +//----------------------------------------------------------------------------// +#ifndef BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK + +void BM_extra_args(benchmark::State& st, const char* label) { + while (st.KeepRunning()) {} + st.SetLabel(label); +} +int RegisterFromFunction() { + std::pair cases[] = { + {"test1", "One"}, + {"test2", "Two"}, + {"test3", "Three"} + }; + for (auto& c : cases) + benchmark::RegisterBenchmark(c.first, &BM_extra_args, c.second); + return 0; +} +int dummy2 = RegisterFromFunction(); +ADD_CASES( + {"test1", "One"}, + {"test2", "Two"}, + {"test3", "Three"} +); + +#endif // BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK + +//----------------------------------------------------------------------------// +// Test RegisterBenchmark with different callable types +//----------------------------------------------------------------------------// + +struct CustomFixture { + void operator()(benchmark::State& st) { + while (st.KeepRunning()) {} + } +}; + +void TestRegistrationAtRuntime() { +#ifdef BENCHMARK_HAS_CXX11 + { + CustomFixture fx; + benchmark::RegisterBenchmark("custom_fixture", fx); + AddCases({"custom_fixture"}); + } +#endif +#ifndef BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK + { + int x = 42; + auto capturing_lam = [=](benchmark::State& st) { + while (st.KeepRunning()) {} + st.SetLabel(std::to_string(x)); + }; + benchmark::RegisterBenchmark("lambda_benchmark", capturing_lam); + AddCases({{"lambda_benchmark", "42"}}); + } +#endif +} + +int main(int argc, char* argv[]) { + TestRegistrationAtRuntime(); + + benchmark::Initialize(&argc, argv); + + TestReporter test_reporter; + benchmark::RunSpecifiedBenchmarks(&test_reporter); + + typedef benchmark::BenchmarkReporter::Run Run; + auto EB = ExpectedResults.begin(); + + for (Run const& run : test_reporter.all_runs_) { + assert(EB != ExpectedResults.end()); + EB->CheckRun(run); + ++EB; + } + assert(EB == ExpectedResults.end()); + + return 0; +} diff --git a/libcxx/utils/google-benchmark/test/reporter_output_test.cc b/libcxx/utils/google-benchmark/test/reporter_output_test.cc index b3898ac..e580008 100644 --- a/libcxx/utils/google-benchmark/test/reporter_output_test.cc +++ b/libcxx/utils/google-benchmark/test/reporter_output_test.cc @@ -114,7 +114,9 @@ std::string join(First f, Args&&... args) { return std::string(std::move(f)) + "[ ]+" + join(std::forward(args)...); } -std::string dec_re = "[0-9]+\\.[0-9]+"; + + +std::string dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; } // end namespace @@ -185,7 +187,7 @@ ADD_CASES(&CSVOutputTests, { void BM_Complexity_O1(benchmark::State& state) { while (state.KeepRunning()) { } - state.SetComplexityN(state.range_x()); + state.SetComplexityN(state.range(0)); } BENCHMARK(BM_Complexity_O1)->Range(1, 1<<18)->Complexity(benchmark::o1); @@ -203,14 +205,8 @@ ADD_CASES(&ConsoleOutputTests, { int main(int argc, char* argv[]) { - // Add --color_print=false to argv since we don't want to match color codes. - char new_arg[64]; - char* new_argv[64]; - std::copy(argv, argv + argc, new_argv); - new_argv[argc++] = std::strcpy(new_arg, "--color_print=false"); - benchmark::Initialize(&argc, new_argv); - - benchmark::ConsoleReporter CR; + benchmark::Initialize(&argc, argv); + benchmark::ConsoleReporter CR(benchmark::ConsoleReporter::OO_None); benchmark::JSONReporter JR; benchmark::CSVReporter CSVR; struct ReporterTest { diff --git a/libcxx/utils/google-benchmark/test/skip_with_error_test.cc b/libcxx/utils/google-benchmark/test/skip_with_error_test.cc index dafbd64..1ad1366 100644 --- a/libcxx/utils/google-benchmark/test/skip_with_error_test.cc +++ b/libcxx/utils/google-benchmark/test/skip_with_error_test.cc @@ -74,7 +74,7 @@ ADD_CASES("BM_error_before_running", void BM_error_during_running(benchmark::State& state) { int first_iter = true; while (state.KeepRunning()) { - if (state.range_x() == 1 && state.thread_index <= (state.threads / 2)) { + if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) { assert(first_iter); first_iter = false; state.SkipWithError("error message"); @@ -116,7 +116,7 @@ ADD_CASES( void BM_error_while_paused(benchmark::State& state) { bool first_iter = true; while (state.KeepRunning()) { - if (state.range_x() == 1 && state.thread_index <= (state.threads / 2)) { + if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) { assert(first_iter); first_iter = false; state.PauseTiming(); diff --git a/libcxx/utils/google-benchmark/tools/compare_bench.py b/libcxx/utils/google-benchmark/tools/compare_bench.py new file mode 100644 index 0000000..ed0f133e --- /dev/null +++ b/libcxx/utils/google-benchmark/tools/compare_bench.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +""" +compare_bench.py - Compare two benchmarks or their results and report the + difference. +""" +import sys +import gbench +from gbench import util, report + +def main(): + # Parse the command line flags + def usage(): + print('compare_bench.py [benchmark options]...') + exit(1) + if '--help' in sys.argv or len(sys.argv) < 3: + usage() + tests = sys.argv[1:3] + bench_opts = sys.argv[3:] + bench_opts = list(bench_opts) + # Run the benchmarks and report the results + json1 = gbench.util.run_or_load_benchmark(tests[0], bench_opts) + json2 = gbench.util.run_or_load_benchmark(tests[1], bench_opts) + output_lines = gbench.report.generate_difference_report(json1, json2) + print 'Comparing %s to %s' % (tests[0], tests[1]) + for ln in output_lines: + print(ln) + + +if __name__ == '__main__': + main() diff --git a/libcxx/utils/google-benchmark/tools/gbench/Inputs/test1_run1.json b/libcxx/utils/google-benchmark/tools/gbench/Inputs/test1_run1.json new file mode 100644 index 0000000..da9425e --- /dev/null +++ b/libcxx/utils/google-benchmark/tools/gbench/Inputs/test1_run1.json @@ -0,0 +1,46 @@ +{ + "context": { + "date": "2016-08-02 17:44:46", + "num_cpus": 4, + "mhz_per_cpu": 4228, + "cpu_scaling_enabled": false, + "library_build_type": "release" + }, + "benchmarks": [ + { + "name": "BM_SameTimes", + "iterations": 1000, + "real_time": 10, + "cpu_time": 10, + "time_unit": "ns" + }, + { + "name": "BM_2xFaster", + "iterations": 1000, + "real_time": 50, + "cpu_time": 50, + "time_unit": "ns" + }, + { + "name": "BM_2xSlower", + "iterations": 1000, + "real_time": 50, + "cpu_time": 50, + "time_unit": "ns" + }, + { + "name": "BM_10PercentFaster", + "iterations": 1000, + "real_time": 100, + "cpu_time": 100, + "time_unit": "ns" + }, + { + "name": "BM_10PercentSlower", + "iterations": 1000, + "real_time": 100, + "cpu_time": 100, + "time_unit": "ns" + } + ] +} \ No newline at end of file diff --git a/libcxx/utils/google-benchmark/tools/gbench/Inputs/test1_run2.json b/libcxx/utils/google-benchmark/tools/gbench/Inputs/test1_run2.json new file mode 100644 index 0000000..d8bc72d --- /dev/null +++ b/libcxx/utils/google-benchmark/tools/gbench/Inputs/test1_run2.json @@ -0,0 +1,46 @@ +{ + "context": { + "date": "2016-08-02 17:44:46", + "num_cpus": 4, + "mhz_per_cpu": 4228, + "cpu_scaling_enabled": false, + "library_build_type": "release" + }, + "benchmarks": [ + { + "name": "BM_SameTimes", + "iterations": 1000, + "real_time": 10, + "cpu_time": 10, + "time_unit": "ns" + }, + { + "name": "BM_2xFaster", + "iterations": 1000, + "real_time": 25, + "cpu_time": 25, + "time_unit": "ns" + }, + { + "name": "BM_2xSlower", + "iterations": 20833333, + "real_time": 100, + "cpu_time": 100, + "time_unit": "ns" + }, + { + "name": "BM_10PercentFaster", + "iterations": 1000, + "real_time": 90, + "cpu_time": 90, + "time_unit": "ns" + }, + { + "name": "BM_10PercentSlower", + "iterations": 1000, + "real_time": 110, + "cpu_time": 110, + "time_unit": "ns" + } + ] +} \ No newline at end of file diff --git a/libcxx/utils/google-benchmark/tools/gbench/__init__.py b/libcxx/utils/google-benchmark/tools/gbench/__init__.py new file mode 100644 index 0000000..fce1a1a --- /dev/null +++ b/libcxx/utils/google-benchmark/tools/gbench/__init__.py @@ -0,0 +1,8 @@ +"""Google Benchmark tooling""" + +__author__ = 'Eric Fiselier' +__email__ = 'eric@efcs.ca' +__versioninfo__ = (0, 5, 0) +__version__ = '.'.join(str(v) for v in __versioninfo__) + 'dev' + +__all__ = [] diff --git a/libcxx/utils/google-benchmark/tools/gbench/report.py b/libcxx/utils/google-benchmark/tools/gbench/report.py new file mode 100644 index 0000000..44fa4a5 --- /dev/null +++ b/libcxx/utils/google-benchmark/tools/gbench/report.py @@ -0,0 +1,136 @@ +"""report.py - Utilities for reporting statistics about benchmark results +""" +import os + +class BenchmarkColor(object): + def __init__(self, name, code): + self.name = name + self.code = code + + def __repr__(self): + return '%s%r' % (self.__class__.__name__, + (self.name, self.code)) + + def __format__(self, format): + return self.code + +# Benchmark Colors Enumeration +BC_NONE = BenchmarkColor('NONE', '') +BC_MAGENTA = BenchmarkColor('MAGENTA', '\033[95m') +BC_CYAN = BenchmarkColor('CYAN', '\033[96m') +BC_OKBLUE = BenchmarkColor('OKBLUE', '\033[94m') +BC_HEADER = BenchmarkColor('HEADER', '\033[92m') +BC_WARNING = BenchmarkColor('WARNING', '\033[93m') +BC_WHITE = BenchmarkColor('WHITE', '\033[97m') +BC_FAIL = BenchmarkColor('FAIL', '\033[91m') +BC_ENDC = BenchmarkColor('ENDC', '\033[0m') +BC_BOLD = BenchmarkColor('BOLD', '\033[1m') +BC_UNDERLINE = BenchmarkColor('UNDERLINE', '\033[4m') + +def color_format(use_color, fmt_str, *args, **kwargs): + """ + Return the result of 'fmt_str.format(*args, **kwargs)' after transforming + 'args' and 'kwargs' according to the value of 'use_color'. If 'use_color' + is False then all color codes in 'args' and 'kwargs' are replaced with + the empty string. + """ + assert use_color is True or use_color is False + if not use_color: + args = [arg if not isinstance(arg, BenchmarkColor) else BC_NONE + for arg in args] + kwargs = {key: arg if not isinstance(arg, BenchmarkColor) else BC_NONE + for key, arg in kwargs.items()} + return fmt_str.format(*args, **kwargs) + + +def find_longest_name(benchmark_list): + """ + Return the length of the longest benchmark name in a given list of + benchmark JSON objects + """ + longest_name = 1 + for bc in benchmark_list: + if len(bc['name']) > longest_name: + longest_name = len(bc['name']) + return longest_name + + +def calculate_change(old_val, new_val): + """ + Return a float representing the decimal change between old_val and new_val. + """ + return float(new_val - old_val) / abs(old_val) + + +def generate_difference_report(json1, json2, use_color=True): + """ + Calculate and report the difference between each test of two benchmarks + runs specified as 'json1' and 'json2'. + """ + first_col_width = find_longest_name(json1['benchmarks']) + 5 + def find_test(name): + for b in json2['benchmarks']: + if b['name'] == name: + return b + return None + first_line = "{:<{}s} Time CPU".format( + 'Benchmark', first_col_width) + output_strs = [first_line, '-' * len(first_line)] + for bn in json1['benchmarks']: + other_bench = find_test(bn['name']) + if not other_bench: + continue + + def get_color(res): + if res > 0.05: + return BC_FAIL + elif res > -0.07: + return BC_WHITE + else: + return BC_CYAN + fmt_str = "{}{:<{}s}{endc} {}{:+.2f}{endc} {}{:+.2f}{endc}" + tres = calculate_change(bn['real_time'], other_bench['real_time']) + cpures = calculate_change(bn['cpu_time'], other_bench['cpu_time']) + output_strs += [color_format(use_color, fmt_str, + BC_HEADER, bn['name'], first_col_width, + get_color(tres), tres, get_color(cpures), cpures, + endc=BC_ENDC)] + return output_strs + +############################################################################### +# Unit tests + +import unittest + +class TestReportDifference(unittest.TestCase): + def load_results(self): + import json + testInputs = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'Inputs') + testOutput1 = os.path.join(testInputs, 'test1_run1.json') + testOutput2 = os.path.join(testInputs, 'test1_run2.json') + with open(testOutput1, 'r') as f: + json1 = json.load(f) + with open(testOutput2, 'r') as f: + json2 = json.load(f) + return json1, json2 + + def test_basic(self): + expect_lines = [ + ['BM_SameTimes', '+0.00', '+0.00'], + ['BM_2xFaster', '-0.50', '-0.50'], + ['BM_2xSlower', '+1.00', '+1.00'], + ['BM_10PercentFaster', '-0.10', '-0.10'], + ['BM_10PercentSlower', '+0.10', '+0.10'] + ] + json1, json2 = self.load_results() + output_lines = generate_difference_report(json1, json2, use_color=False) + print output_lines + self.assertEqual(len(output_lines), len(expect_lines)) + for i in xrange(0, len(output_lines)): + parts = [x for x in output_lines[i].split(' ') if x] + self.assertEqual(len(parts), 3) + self.assertEqual(parts, expect_lines[i]) + + +if __name__ == '__main__': + unittest.main() diff --git a/libcxx/utils/google-benchmark/tools/gbench/util.py b/libcxx/utils/google-benchmark/tools/gbench/util.py new file mode 100644 index 0000000..169b71c --- /dev/null +++ b/libcxx/utils/google-benchmark/tools/gbench/util.py @@ -0,0 +1,130 @@ +"""util.py - General utilities for running, loading, and processing benchmarks +""" +import json +import os +import tempfile +import subprocess +import sys + +# Input file type enumeration +IT_Invalid = 0 +IT_JSON = 1 +IT_Executable = 2 + +_num_magic_bytes = 2 if sys.platform.startswith('win') else 4 +def is_executable_file(filename): + """ + Return 'True' if 'filename' names a valid file which is likely + an executable. A file is considered an executable if it starts with the + magic bytes for a EXE, Mach O, or ELF file. + """ + if not os.path.isfile(filename): + return False + with open(filename, 'r') as f: + magic_bytes = f.read(_num_magic_bytes) + if sys.platform == 'darwin': + return magic_bytes in [ + '\xfe\xed\xfa\xce', # MH_MAGIC + '\xce\xfa\xed\xfe', # MH_CIGAM + '\xfe\xed\xfa\xcf', # MH_MAGIC_64 + '\xcf\xfa\xed\xfe', # MH_CIGAM_64 + '\xca\xfe\xba\xbe', # FAT_MAGIC + '\xbe\xba\xfe\xca' # FAT_CIGAM + ] + elif sys.platform.startswith('win'): + return magic_bytes == 'MZ' + else: + return magic_bytes == '\x7FELF' + + +def is_json_file(filename): + """ + Returns 'True' if 'filename' names a valid JSON output file. + 'False' otherwise. + """ + try: + with open(filename, 'r') as f: + json.load(f) + return True + except: + pass + return False + + +def classify_input_file(filename): + """ + Return a tuple (type, msg) where 'type' specifies the classified type + of 'filename'. If 'type' is 'IT_Invalid' then 'msg' is a human readable + string represeting the error. + """ + ftype = IT_Invalid + err_msg = None + if not os.path.exists(filename): + err_msg = "'%s' does not exist" % filename + elif not os.path.isfile(filename): + err_msg = "'%s' does not name a file" % filename + elif is_executable_file(filename): + ftype = IT_Executable + elif is_json_file(filename): + ftype = IT_JSON + else: + err_msg = "'%s' does not name a valid benchmark executable or JSON file" + return ftype, err_msg + + +def check_input_file(filename): + """ + Classify the file named by 'filename' and return the classification. + If the file is classified as 'IT_Invalid' print an error message and exit + the program. + """ + ftype, msg = classify_input_file(filename) + if ftype == IT_Invalid: + print "Invalid input file: %s" % msg + sys.exit(1) + return ftype + + +def load_benchmark_results(fname): + """ + Read benchmark output from a file and return the JSON object. + REQUIRES: 'fname' names a file containing JSON benchmark output. + """ + with open(fname, 'r') as f: + return json.load(f) + + +def run_benchmark(exe_name, benchmark_flags): + """ + Run a benchmark specified by 'exe_name' with the specified + 'benchmark_flags'. The benchmark is run directly as a subprocess to preserve + real time console output. + RETURNS: A JSON object representing the benchmark output + """ + thandle, tname = tempfile.mkstemp() + os.close(thandle) + cmd = [exe_name] + benchmark_flags + print("RUNNING: %s" % ' '.join(cmd)) + exitCode = subprocess.call(cmd + ['--benchmark_out=%s' % tname]) + if exitCode != 0: + print('TEST FAILED...') + sys.exit(exitCode) + json_res = load_benchmark_results(tname) + os.unlink(tname) + return json_res + + +def run_or_load_benchmark(filename, benchmark_flags): + """ + Get the results for a specified benchmark. If 'filename' specifies + an executable benchmark then the results are generated by running the + benchmark. Otherwise 'filename' must name a valid JSON output file, + which is loaded and the result returned. + """ + ftype = check_input_file(filename) + if ftype == IT_JSON: + return load_benchmark_results(filename) + elif ftype == IT_Executable: + return run_benchmark(filename, benchmark_flags) + else: + assert False # This branch is unreachable \ No newline at end of file -- 2.7.4