2 # Copyright 2014 The Chromium Authors
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Runs a test repeatedly to measure its flakiness. The return code is non-zero
7 if the failure rate is higher than the specified threshold, but is not 100%."""
10 import multiprocessing.dummy
16 parser = argparse.ArgumentParser(description=__doc__)
17 parser.add_argument('--retries', default=1000, type=int,
18 help='Number of test retries to measure flakiness.')
19 parser.add_argument('--threshold', default=0.05, type=float,
20 help='Minimum flakiness level at which test is '
22 parser.add_argument('--jobs', '-j', type=int, default=1,
23 help='Number of parallel jobs to run tests.')
24 parser.add_argument('command', nargs='+', help='Command to run test.')
25 return parser.parse_args()
28 print('Starting retry attempt %d out of %d' % (job['index'] + 1,
30 return subprocess.check_call(job['cmd'], stdout=subprocess.PIPE,
31 stderr=subprocess.STDOUT)
34 options = load_options()
35 num_passed = num_failed = 0
38 pool = multiprocessing.dummy.Pool(processes=options.jobs)
39 args = [{'index': index, 'retries': options.retries, 'cmd': options.command}
40 for index in range(options.retries)]
41 results = pool.map(run_test, args)
42 num_passed = len([retcode for retcode in results if retcode == 0])
43 num_failed = len(results) - num_passed
48 flakiness = num_failed / float(len(results))
50 print('Flakiness is %.2f' % flakiness)
51 if flakiness > options.threshold:
57 if __name__ == '__main__':