From 54a9b4c02ff57e9847e0c501578e51db6f73d3be Mon Sep 17 00:00:00 2001 From: Julian Lettner Date: Tue, 12 Nov 2019 08:24:26 -0800 Subject: [PATCH] [lit] Better/earlier errors for empty runs Fail early, when we discover no tests at all, or filter out all of them. There is also `--allow-empty-runs` to disable test to allow workflows like `LIT_FILTER=abc ninja check-all`. Apparently `check-all` invokes lit multiple times if certain projects are enabled, which would produce unwanted "empty runs". Specify via `LIT_OPTS=--allow-empty-runs`. There are 3 causes for empty runs: 1) No tests discovered. This is always an error. Fix test suite config or command line. 2) All tests filtered out. This is an error by default, but can be suppressed via `--alow-empty-runs`. Should prevent accidentally passing empty runs, but allow the workflow above. 3) The number of shards is greater than the number of tests. Currently, this is never an error. Personally, I think we should consider making this an error by default; if this happens, you are doing something wrong. I added a warning but did not change the behavior, since this warrants more discussion. Reviewed By: atrick, jdenny Differential Revision: https://reviews.llvm.org/D70105 --- llvm/utils/lit/lit/cl_arguments.py | 3 +++ llvm/utils/lit/lit/main.py | 20 +++++++++++++++++++- llvm/utils/lit/lit/run.py | 5 +---- llvm/utils/lit/tests/selecting.py | 22 ++++++++++++++++++---- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py index 7b15041..cbdd0ec 100644 --- a/llvm/utils/lit/lit/cl_arguments.py +++ b/llvm/utils/lit/lit/cl_arguments.py @@ -109,6 +109,9 @@ def parse_args(): dest="maxFailures", help="Stop execution after the given number of failures.", type=_positive_int) + execution_group.add_argument("--allow-empty-runs", + help="Do not fail the run if all tests are filtered out", + action="store_true") selection_group = parser.add_argument_group("Test Selection") selection_group.add_argument("--max-tests", diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py index a802b45..0eb95b9c 100755 --- a/llvm/utils/lit/lit/main.py +++ b/llvm/utils/lit/lit/main.py @@ -43,6 +43,9 @@ def main(builtin_params = {}): echo_all_commands = opts.echoAllCommands) tests = lit.discovery.find_tests_for_inputs(litConfig, opts.test_paths) + if not tests: + sys.stderr.write('Did not disover any tests for provided path(s).\n') + sys.exit(2) # Command line overrides configuration for maxIndividualTestTime. if opts.maxIndividualTestTime is not None: # `not None` is important (default: 0) @@ -63,12 +66,27 @@ def main(builtin_params = {}): if opts.filter: tests = [t for t in tests if opts.filter.search(t.getFullName())] + if not tests: + sys.stderr.write('Filter did not match any tests ' + '(of %d discovered). ' % numTotalTests) + if opts.allow_empty_runs: + sys.stderr.write('Suppressing error because ' + "'--allow-empty-runs' was specified.\n") + sys.exit(0) + else: + sys.stderr.write("Use '--allow-empty-runs' to suppress this " + 'error.\n') + sys.exit(2) determine_order(tests, opts.order) if opts.shard: (run, shards) = opts.shard tests = filter_by_shard(tests, run, shards, litConfig) + if not tests: + sys.stderr.write('Shard does not contain any tests. Consider ' + 'decreasing the number of shards.\n') + sys.exit(0) if opts.max_tests: tests = tests[:opts.max_tests] @@ -87,7 +105,7 @@ def main(builtin_params = {}): write_test_results_xunit(tests, opts) if litConfig.numErrors: - sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors) + sys.stderr.write('\n%d error(s) in tests.\n' % litConfig.numErrors) sys.exit(2) if litConfig.numWarnings: diff --git a/llvm/utils/lit/lit/run.py b/llvm/utils/lit/lit/run.py index 6a43f51..166b64e 100644 --- a/llvm/utils/lit/lit/run.py +++ b/llvm/utils/lit/lit/run.py @@ -13,7 +13,7 @@ class NopSemaphore(object): def release(self): pass def create_run(tests, lit_config, workers, progress_callback, timeout=None): - # TODO(yln) assert workers > 0 + assert workers > 0 if workers == 1: return SerialRun(tests, lit_config, progress_callback, timeout) return ParallelRun(tests, lit_config, progress_callback, timeout, workers) @@ -45,9 +45,6 @@ class Run(object): computed. Tests which were not actually executed (for any reason) will be given an UNRESOLVED result. """ - if not self.tests: - return 0.0 - self.failure_count = 0 self.hit_max_failures = False diff --git a/llvm/utils/lit/tests/selecting.py b/llvm/utils/lit/tests/selecting.py index 0921cdd..638c108 100644 --- a/llvm/utils/lit/tests/selecting.py +++ b/llvm/utils/lit/tests/selecting.py @@ -1,6 +1,21 @@ # RUN: %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-BASIC %s # CHECK-BASIC: Testing: 5 tests + +# Check that we exit with an error if we do not discover any tests, even with --allow-empty-runs. +# +# RUN: not %{lit} %{inputs}/nonexistent 2>&1 | FileCheck --check-prefix=CHECK-BAD-PATH %s +# RUN: not %{lit} %{inputs}/nonexistent --allow-empty-runs 2>&1 | FileCheck --check-prefix=CHECK-BAD-PATH %s +# CHECK-BAD-PATH: Did not disover any tests for provided path(s). + +# Check that we exit with an error if we filter out all tests, but allow it with --allow-empty-runs. +# +# RUN: not %{lit} --filter 'nonexistent' %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ERROR %s +# RUN: %{lit} --filter 'nonexistent' --allow-empty-runs %{inputs}/discovery 2>&1 | FileCheck --check-prefixes=CHECK-BAD-FILTER,CHECK-BAD-FILTER-ALLOW %s +# CHECK-BAD-FILTER: Filter did not match any tests (of 5 discovered). +# CHECK-BAD-FILTER-ERROR: Use '--allow-empty-runs' to suppress this error. +# CHECK-BAD-FILTER-ALLOW: Suppressing error because '--allow-empty-runs' was specified. + # Check that regex-filtering works, is case-insensitive, and can be configured via env var. # # RUN: %{lit} --filter 'o[a-z]e' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s @@ -8,6 +23,7 @@ # RUN: env LIT_FILTER='o[a-z]e' %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s # CHECK-FILTER: Testing: 2 of 5 tests + # Check that maximum counts work # # RUN: %{lit} --max-tests 3 %{inputs}/discovery | FileCheck --check-prefix=CHECK-MAX %s @@ -68,15 +84,13 @@ # # RUN: %{lit} --num-shards 100 --run-shard 6 %{inputs}/discovery >%t.out 2>%t.err # RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR2 < %t.err %s -# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-OUT2 < %t.out %s # CHECK-SHARD-BIG-ERR2: note: Selecting shard 6/100 = size 0/5 = tests #(100*k)+6 = [] -# CHECK-SHARD-BIG-OUT2: Testing: 0 of 5 tests +# CHECK-SHARD-BIG-ERR2: Shard does not contain any tests. Consider decreasing the number of shards. # # RUN: %{lit} --num-shards 100 --run-shard 50 %{inputs}/discovery >%t.out 2>%t.err # RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR3 < %t.err %s -# RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-OUT3 < %t.out %s # CHECK-SHARD-BIG-ERR3: note: Selecting shard 50/100 = size 0/5 = tests #(100*k)+50 = [] -# CHECK-SHARD-BIG-OUT3: Testing: 0 of 5 tests +# CHECK-SHARD-BIG-ERR3: Shard does not contain any tests. Consider decreasing the number of shards. # Check that range constraints are enforced -- 2.7.4