From 336a1a681189bf2b50b7bd0f8229656c58ac1b37 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Wed, 14 Nov 2018 20:38:46 +0000 Subject: [PATCH] Rename cxx-benchmark-unittests target and convert to LIT. This patch renames the cxx-benchmark-unittests to check-cxx-benchmarks and converts the target to use LIT in order to make the tests run faster and provide better output. In particular this runs each benchmark in a suite one by one, allowing more parallelism while ensuring output isn't garbage with multiple threads. Additionally, it adds the CMake flag '-DLIBCXX_BENCHMARK_TEST_ARGS=' to specify what options are passed when running the benchmarks. llvm-svn: 346888 --- libcxx/CMakeLists.txt | 17 ++++ libcxx/benchmarks/CMakeLists.txt | 29 ++++--- libcxx/benchmarks/lit.cfg.py | 23 ++++++ libcxx/benchmarks/lit.site.cfg.py.in | 10 +++ libcxx/docs/BuildingLibcxx.rst | 9 ++ libcxx/test/CMakeLists.txt | 13 +-- libcxx/utils/libcxx/test/googlebenchmark.py | 122 ++++++++++++++++++++++++++++ 7 files changed, 201 insertions(+), 22 deletions(-) create mode 100644 libcxx/benchmarks/lit.cfg.py create mode 100644 libcxx/benchmarks/lit.site.cfg.py.in create mode 100644 libcxx/utils/libcxx/test/googlebenchmark.py diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt index 88bf496..d090860 100644 --- a/libcxx/CMakeLists.txt +++ b/libcxx/CMakeLists.txt @@ -83,6 +83,11 @@ option(LIBCXX_INCLUDE_TESTS "Build the libc++ tests." ${LLVM_INCLUDE_TESTS}) # Benchmark options ----------------------------------------------------------- option(LIBCXX_INCLUDE_BENCHMARKS "Build the libc++ benchmarks and their dependencies" ON) + +set(LIBCXX_BENCHMARK_TEST_ARGS_DEFAULT --benchmark_min_time=0.01) +set(LIBCXX_BENCHMARK_TEST_ARGS "${LIBCXX_BENCHMARK_TEST_ARGS_DEFAULT}" CACHE STRING + "Arguments to pass when running the benchmarks using check-cxx-benchmarks") + set(LIBCXX_BENCHMARK_NATIVE_STDLIB "" CACHE STRING "Build the benchmarks against the specified native STL. The value must be one of libc++/libstdc++") @@ -766,6 +771,18 @@ include_directories(include) add_subdirectory(include) add_subdirectory(lib) +set(LIBCXX_TEST_DEPS "") + +if (LIBCXX_ENABLE_EXPERIMENTAL_LIBRARY) + list(APPEND LIBCXX_TEST_DEPS cxx_experimental) +endif() +if (LIBCXX_ENABLE_FILESYSTEM) + list(APPEND LIBCXX_TEST_DEPS cxx_filesystem) +endif() + +if (LIBCXX_BUILD_EXTERNAL_THREAD_LIBRARY) + list(APPEND LIBCXX_TEST_DEPS cxx_external_threads) +endif() if (LIBCXX_INCLUDE_BENCHMARKS) add_subdirectory(benchmarks) diff --git a/libcxx/benchmarks/CMakeLists.txt b/libcxx/benchmarks/CMakeLists.txt index c5b4bf0..8931161 100644 --- a/libcxx/benchmarks/CMakeLists.txt +++ b/libcxx/benchmarks/CMakeLists.txt @@ -180,15 +180,22 @@ foreach(test_path ${BENCHMARK_TESTS}) add_benchmark_test(${test_name} ${test_file}) endforeach() +if (LIBCXX_INCLUDE_TESTS) + include(AddLLVM) -add_custom_target(cxx-benchmark-unittests) -foreach(libcxx_tg ${libcxx_benchmark_targets}) - message("Adding test ${libcxx_tg}") - # Add a target that runs the benchmark for the smallest possible time, simply so we get test - # and sanitizer coverage on the targets. - add_custom_target(${libcxx_tg}_test - COMMAND ${libcxx_tg} --benchmark_min_time=0.01 - COMMENT "Running test ${libcxx_tg}" - ) - add_dependencies(cxx-benchmark-unittests ${libcxx_tg}_test) -endforeach() + if (NOT DEFINED LIBCXX_TEST_DEPS) + message(FATAL_ERROR "Expected LIBCXX_TEST_DEPS to be defined") + endif() + + configure_lit_site_cfg( + ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in + ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py) + + set(BENCHMARK_LIT_ARGS "--show-all --show-xfail --show-unsupported ${LIT_ARGS_DEFAULT}") + + add_lit_testsuite(check-cxx-benchmarks + "Running libcxx benchmarks tests" + ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS cxx-benchmarks ${LIBCXX_TEST_DEPS} + ARGS ${BENCHMARK_LIT_ARGS}) +endif() diff --git a/libcxx/benchmarks/lit.cfg.py b/libcxx/benchmarks/lit.cfg.py new file mode 100644 index 0000000..84857d5 --- /dev/null +++ b/libcxx/benchmarks/lit.cfg.py @@ -0,0 +1,23 @@ +# -*- Python -*- vim: set ft=python ts=4 sw=4 expandtab tw=79: +# Configuration file for the 'lit' test runner. +import os +import site + +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'utils')) +from libcxx.test.googlebenchmark import GoogleBenchmark + +# Tell pylint that we know config and lit_config exist somewhere. +if 'PYLINT_IMPORT' in os.environ: + config = object() + lit_config = object() + +# name: The name of this test suite. +config.name = 'libc++ benchmarks' +config.suffixes = [] + +config.test_exec_root = os.path.join(config.libcxx_obj_root, 'benchmarks') +config.test_source_root = config.test_exec_root + +config.test_format = GoogleBenchmark(test_sub_dirs='.', + test_suffix='.libcxx.out', + benchmark_args=config.benchmark_args) \ No newline at end of file diff --git a/libcxx/benchmarks/lit.site.cfg.py.in b/libcxx/benchmarks/lit.site.cfg.py.in new file mode 100644 index 0000000..e3ce8b2 --- /dev/null +++ b/libcxx/benchmarks/lit.site.cfg.py.in @@ -0,0 +1,10 @@ +@LIT_SITE_CFG_IN_HEADER@ + +import sys + +config.libcxx_src_root = "@LIBCXX_SOURCE_DIR@" +config.libcxx_obj_root = "@LIBCXX_BINARY_DIR@" +config.benchmark_args = "@LIBCXX_BENCHMARK_TEST_ARGS@".split(';') + +# Let the main config do the real work. +lit_config.load_config(config, "@LIBCXX_SOURCE_DIR@/benchmarks/lit.cfg.py") \ No newline at end of file diff --git a/libcxx/docs/BuildingLibcxx.rst b/libcxx/docs/BuildingLibcxx.rst index 66cfb7b..12acd0b 100644 --- a/libcxx/docs/BuildingLibcxx.rst +++ b/libcxx/docs/BuildingLibcxx.rst @@ -316,6 +316,15 @@ libc++ Feature Options Build the libc++ benchmark tests and the Google Benchmark library needed to support them. +.. option:: LIBCXX_BENCHMARK_TEST_ARGS:STRING + + **Default**: ``--benchmark_min_time=0.01`` + + A semicolon list of arguments to pass when running the libc++ benchmarks using the + ``check-cxx-benchmarks`` rule. By default we run the benchmarks for a very short amount of time, + since the primary use of ``check-cxx-benchmarks`` is to get test and sanitizer coverage, not to + get accurate measurements. + .. option:: LIBCXX_BENCHMARK_NATIVE_STDLIB:STRING **Default**:: ``""`` diff --git a/libcxx/test/CMakeLists.txt b/libcxx/test/CMakeLists.txt index f844246..9435744 100644 --- a/libcxx/test/CMakeLists.txt +++ b/libcxx/test/CMakeLists.txt @@ -55,17 +55,8 @@ set(LIBCXX_EXECUTOR "None" CACHE STRING set(AUTO_GEN_COMMENT "## Autogenerated by libcxx configuration.\n# Do not edit!") -set(LIBCXX_TEST_DEPS "") - -if (LIBCXX_ENABLE_EXPERIMENTAL_LIBRARY) - list(APPEND LIBCXX_TEST_DEPS cxx_experimental) -endif() -if (LIBCXX_ENABLE_FILESYSTEM) - list(APPEND LIBCXX_TEST_DEPS cxx_filesystem) -endif() - -if (LIBCXX_BUILD_EXTERNAL_THREAD_LIBRARY) - list(APPEND LIBCXX_TEST_DEPS cxx_external_threads) +if (NOT DEFINED LIBCXX_TEST_DEPS) + message(FATAL_ERROR "Expected LIBCXX_TEST_DEPS to be defined") endif() if (LIBCXX_INCLUDE_TESTS) diff --git a/libcxx/utils/libcxx/test/googlebenchmark.py b/libcxx/utils/libcxx/test/googlebenchmark.py new file mode 100644 index 0000000..6fe731e --- /dev/null +++ b/libcxx/utils/libcxx/test/googlebenchmark.py @@ -0,0 +1,122 @@ +from __future__ import absolute_import +import os +import subprocess +import sys + +import lit.Test +import lit.TestRunner +import lit.util +from lit.formats.base import TestFormat + +kIsWindows = sys.platform in ['win32', 'cygwin'] + +class GoogleBenchmark(TestFormat): + def __init__(self, test_sub_dirs, test_suffix, benchmark_args=[]): + self.benchmark_args = list(benchmark_args) + self.test_sub_dirs = os.path.normcase(str(test_sub_dirs)).split(';') + + # On Windows, assume tests will also end in '.exe'. + exe_suffix = str(test_suffix) + if kIsWindows: + exe_suffix += '.exe' + + # Also check for .py files for testing purposes. + self.test_suffixes = {exe_suffix, test_suffix + '.py'} + + def getBenchmarkTests(self, path, litConfig, localConfig): + """getBenchmarkTests(path) - [name] + + Return the tests available in gtest executable. + + Args: + path: String path to a gtest executable + litConfig: LitConfig instance + localConfig: TestingConfig instance""" + + # TODO: allow splitting tests according to the "benchmark family" so + # the output for a single family of tests all belongs to the same test + # target. + list_test_cmd = [path, '--benchmark_list_tests'] + try: + output = subprocess.check_output(list_test_cmd, + env=localConfig.environment) + except subprocess.CalledProcessError as exc: + litConfig.warning( + "unable to discover google-benchmarks in %r: %s. Process output: %s" + % (path, sys.exc_info()[1], exc.output)) + raise StopIteration + + nested_tests = [] + for ln in output.splitlines(False): # Don't keep newlines. + ln = lit.util.to_string(ln) + if not ln.strip(): + continue + + index = 0 + while ln[index*2:index*2+2] == ' ': + index += 1 + while len(nested_tests) > index: + nested_tests.pop() + + ln = ln[index*2:] + if ln.endswith('.'): + nested_tests.append(ln) + elif any([name.startswith('DISABLED_') + for name in nested_tests + [ln]]): + # Gtest will internally skip these tests. No need to launch a + # child process for it. + continue + else: + yield ''.join(nested_tests) + ln + + def getTestsInDirectory(self, testSuite, path_in_suite, + litConfig, localConfig): + source_path = testSuite.getSourcePath(path_in_suite) + for subdir in self.test_sub_dirs: + dir_path = os.path.join(source_path, subdir) + if not os.path.isdir(dir_path): + continue + for fn in lit.util.listdir_files(dir_path, + suffixes=self.test_suffixes): + # Discover the tests in this executable. + execpath = os.path.join(source_path, subdir, fn) + testnames = self.getBenchmarkTests(execpath, litConfig, localConfig) + for testname in testnames: + testPath = path_in_suite + (subdir, fn, testname) + yield lit.Test.Test(testSuite, testPath, localConfig, + file_path=execpath) + + def execute(self, test, litConfig): + testPath,testName = os.path.split(test.getSourcePath()) + while not os.path.exists(testPath): + # Handle GTest parametrized and typed tests, whose name includes + # some '/'s. + testPath, namePrefix = os.path.split(testPath) + testName = namePrefix + '/' + testName + + cmd = [testPath, '--benchmark_filter=%s$' % testName ] + self.benchmark_args + + if litConfig.noExecute: + return lit.Test.PASS, '' + + try: + out, err, exitCode = lit.util.executeCommand( + cmd, env=test.config.environment, + timeout=litConfig.maxIndividualTestTime) + except lit.util.ExecuteCommandTimeoutException: + return (lit.Test.TIMEOUT, + 'Reached timeout of {} seconds'.format( + litConfig.maxIndividualTestTime) + ) + + if exitCode: + return lit.Test.FAIL, out + err + + passing_test_line = testName + if passing_test_line not in out: + msg = ('Unable to find %r in google benchmark output:\n\n%s%s' % + (passing_test_line, out, err)) + return lit.Test.UNRESOLVED, msg + + return lit.Test.PASS, err + out + -- 2.7.4