1 # SPDX-License-Identifier: GPL-2.0+
3 # Copyright (c) 2016 Google, Inc
6 from contextlib import contextmanager
13 from patman import command
14 from patman import test_util
16 from io import StringIO
20 from concurrencytest import ConcurrentTestSuite, fork_for_tests
22 use_concurrent = False
25 def RunTestCoverage(prog, filter_fname, exclude_list, build_dir, required=None):
26 """Run tests and check that we get 100% coverage
29 prog: Program to run (with be passed a '-t' argument to run tests
30 filter_fname: Normally all *.py files in the program's directory will
31 be included. If this is not None, then it is used to filter the
32 list so that only filenames that don't contain filter_fname are
34 exclude_list: List of file patterns to exclude from the coverage
36 build_dir: Build directory, used to locate libfdt.py
37 required: List of modules which must be in the coverage report
40 ValueError if the code coverage is not 100%
42 # This uses the build output from sandbox_spl to get _libfdt.so
43 path = os.path.dirname(prog)
45 glob_list = glob.glob(os.path.join(path, '*.py'))
46 glob_list = [fname for fname in glob_list if filter_fname in fname]
49 glob_list += exclude_list
50 glob_list += ['*libfdt.py', '*site-packages*', '*dist-packages*']
51 test_cmd = 'test' if 'binman' in prog else '-t'
54 prefix = 'PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools ' % build_dir
55 cmd = ('%spython3-coverage run '
56 '--omit "%s" %s %s -P1' % (prefix, ','.join(glob_list),
59 stdout = command.Output('python3-coverage', 'report')
60 lines = stdout.splitlines()
62 # Convert '/path/to/name.py' just the module name 'name'
63 test_set = set([os.path.splitext(os.path.basename(line.split()[0]))[0]
64 for line in lines if '/etype/' in line])
65 missing_list = required
66 missing_list.discard('__init__')
67 missing_list.difference_update(test_set)
69 print('Missing tests for %s' % (', '.join(missing_list)))
73 coverage = lines[-1].split(' ')[-1]
76 if coverage != '100%':
78 print("Type 'python3-coverage html' to get a report in "
80 print('Coverage error: %s, but should be 100%%' % coverage)
83 raise ValueError('Test coverage failure')
86 # Use this to suppress stdout/stderr output:
87 # with capture_sys_output() as (stdout, stderr)
90 def capture_sys_output():
91 capture_out, capture_err = StringIO(), StringIO()
92 old_out, old_err = sys.stdout, sys.stderr
94 sys.stdout, sys.stderr = capture_out, capture_err
95 yield capture_out, capture_err
97 sys.stdout, sys.stderr = old_out, old_err
100 def ReportResult(toolname:str, test_name: str, result: unittest.TestResult):
101 """Report the results from a suite of tests
104 toolname: Name of the tool that ran the tests
105 test_name: Name of test that was run, or None for all
106 result: A unittest.TestResult object containing the results
108 # Remove errors which just indicate a missing test. Since Python v3.5 If an
109 # ImportError or AttributeError occurs while traversing name then a
110 # synthetic test that raises that error when run will be returned. These
111 # errors are included in the errors accumulated by result.errors.
115 for test, err in result.errors:
116 if ("has no attribute '%s'" % test_name) not in err:
117 errors.append((test, err))
119 result.errors = errors
122 for test, err in result.errors:
123 print(test.id(), err)
124 for test, err in result.failures:
125 print(err, result.failures)
127 print('%d binman test%s SKIPPED:' %
128 (len(result.skipped), 's' if len(result.skipped) > 1 else ''))
129 for skip_info in result.skipped:
130 print('%s: %s' % (skip_info[0], skip_info[1]))
131 if result.errors or result.failures:
132 print('binman tests FAILED')
137 def RunTestSuites(result, debug, verbosity, test_preserve_dirs, processes,
138 test_name, toolpath, test_class_list):
139 """Run a series of test suites and collect the results
142 result: A unittest.TestResult object to add the results to
143 debug: True to enable debugging, which shows a full stack trace on error
144 verbosity: Verbosity level to use (0-4)
145 test_preserve_dirs: True to preserve the input directory used by tests
146 so that it can be examined afterwards (only useful for debugging
147 tests). If a single test is selected (in args[0]) it also preserves
148 the output directory for this test. Both directories are displayed
150 processes: Number of processes to use to run tests (None=same as #CPUs)
151 test_name: Name of test to run, or None for all
152 toolpath: List of paths to use for tools
153 test_class_list: List of test classes to run
156 suite = doctest.DocTestSuite(module)
159 sys.argv = [sys.argv[0]]
161 sys.argv.append('-D')
163 sys.argv.append('-v%d' % verbosity)
165 for path in toolpath:
166 sys.argv += ['--toolpath', path]
168 suite = unittest.TestSuite()
169 loader = unittest.TestLoader()
170 for module in test_class_list:
171 # Test the test module about our arguments, if it is interested
172 if hasattr(module, 'setup_test_args'):
173 setup_test_args = getattr(module, 'setup_test_args')
174 setup_test_args(preserve_indir=test_preserve_dirs,
175 preserve_outdirs=test_preserve_dirs and test_name is not None,
176 toolpath=toolpath, verbosity=verbosity)
179 suite.addTests(loader.loadTestsFromName(test_name, module))
180 except AttributeError:
183 suite.addTests(loader.loadTestsFromTestCase(module))
184 if use_concurrent and processes != 1:
185 concurrent_suite = ConcurrentTestSuite(suite,
186 fork_for_tests(processes or multiprocessing.cpu_count()))
187 concurrent_suite.run(result)