3 # Copyright 2017 The Chromium Authors. All rights reserved.
4 # Use of this source code is governed by a BSD-style license that can be
5 # found in the LICENSE file.
7 """Runs a Fuchsia gtest-based test on Swarming, optionally many times,
8 collecting the output of the runs into a directory. Useful for flake checking,
9 and faster than using trybots by avoiding repeated bot_update, compile, archive,
10 etc. and allowing greater parallelism.
12 To use, run in a new shell (it blocks until all Swarming jobs complete):
14 tools/run-swarmed.py -t content_unittests --out-dir=out/fuch
16 The logs of the runs will be stored in results/ (or specify a results directory
17 with --results=some_dir). You can then do something like `grep -L SUCCESS
18 results/*` to find the tests that failed or otherwise process the log files.
22 import multiprocessing
29 INTERNAL_ERROR_EXIT_CODE = -1000
33 """Triggers a swarming job. The arguments passed are:
34 - The index of the job;
35 - The command line arguments object;
36 - The hash of the isolate job used to trigger;
37 - Value of --gtest_filter arg, or empty if none.
39 The return value is passed to a collect-style map() and consists of:
40 - The index of the job;
41 - The json file created by triggering and used to collect results;
42 - The command line arguments object.
44 index, args, isolated_hash, gtest_filter = args
45 json_file = os.path.join(args.results, '%d.json' % index)
47 'tools/swarming_client/swarming.py', 'trigger',
48 '-S', 'https://chromium-swarm.appspot.com',
49 '-I', 'https://isolateserver.appspot.com',
50 '-d', 'pool', 'Chrome',
52 '--dump-json', json_file,
54 if args.target_os == 'fuchsia':
59 '-d', 'cpu', args.arch,
61 elif args.target_os == 'win':
62 trigger_args += [ '-d', 'os', 'Windows' ]
65 '--test-launcher-summary-output=${ISOLATED_OUTDIR}/output.json',
66 '--system-log-file=${ISOLATED_OUTDIR}/system_log']
68 trigger_args.append('--gtest_filter=' + gtest_filter)
69 elif args.target_os == 'fuchsia':
71 'testing/buildbot/filters/fuchsia.' + args.test_name + '.filter'
72 if os.path.isfile(filter_file):
73 trigger_args.append('--test-launcher-filter-file=../../' + filter_file)
74 with open(os.devnull, 'w') as nul:
75 subprocess.check_call(trigger_args, stdout=nul)
76 return (index, json_file, args)
79 def _Collect(spawn_result):
80 index, json_file, args = spawn_result
81 p = subprocess.Popen([
82 'tools/swarming_client/swarming.py', 'collect',
83 '-S', 'https://chromium-swarm.appspot.com',
85 '--task-output-stdout=console'],
86 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
87 stdout = p.communicate()[0]
88 if p.returncode != 0 and len(stdout) < 2**10 and 'Internal error!' in stdout:
89 exit_code = INTERNAL_ERROR_EXIT_CODE
90 file_suffix = '.INTERNAL_ERROR'
92 exit_code = p.returncode
93 file_suffix = '' if exit_code == 0 else '.FAILED'
94 filename = '%d%s.stdout.txt' % (index, file_suffix)
95 with open(os.path.join(args.results, filename), 'w') as f:
101 parser = argparse.ArgumentParser()
102 parser.add_argument('-C', '--out-dir', default='out/fuch',
103 help='Build directory.')
104 parser.add_argument('--target-os', default='detect', help='gn target_os')
105 parser.add_argument('--test-name', '-t', required=True,
106 help='Name of test to run.')
107 parser.add_argument('--arch', '-a', default='detect',
108 help='CPU architecture of the test binary.')
109 parser.add_argument('--copies', '-n', type=int, default=1,
110 help='Number of copies to spawn.')
111 parser.add_argument('--results', '-r', default='results',
112 help='Directory in which to store results.')
113 parser.add_argument('--gtest_filter',
114 help='Use the given gtest_filter, rather than the '
115 'default filter file, if any.')
117 args = parser.parse_args()
119 if args.target_os == 'detect':
120 with open(os.path.join(args.out_dir, 'args.gn')) as f:
123 l = l.split('#')[0].strip()
125 k, v = map(str.strip, l.split('=', 1))
127 if 'target_os' in gn_args:
128 args.target_os = gn_args['target_os'].strip('"')
130 args.target_os = { 'darwin': 'mac', 'linux2': 'linux', 'win32': 'win' }[
133 # Determine the CPU architecture of the test binary, if not specified.
134 if args.arch == 'detect' and args.target_os == 'fuchsia':
135 executable_info = subprocess.check_output(
136 ['file', os.path.join(args.out_dir, args.test_name)])
137 if 'ARM aarch64' in executable_info:
142 subprocess.check_call(
143 ['tools/mb/mb.py', 'isolate', '//' + args.out_dir, args.test_name])
145 print 'If you get authentication errors, follow:'
146 print ' https://www.chromium.org/developers/testing/isolated-testing/for-swes#TOC-Login-on-the-services'
148 print 'Uploading to isolate server, this can take a while...'
149 archive_output = subprocess.check_output(
150 ['tools/swarming_client/isolate.py', 'archive',
151 '-I', 'https://isolateserver.appspot.com',
152 '-i', os.path.join(args.out_dir, args.test_name + '.isolate'),
153 '-s', os.path.join(args.out_dir, args.test_name + '.isolated')])
154 isolated_hash = archive_output.split()[0]
156 if os.path.isdir(args.results):
157 shutil.rmtree(args.results)
158 os.makedirs(args.results)
161 print 'Triggering %d tasks...' % args.copies
162 pool = multiprocessing.Pool()
163 spawn_args = map(lambda i: (i, args, isolated_hash, args.gtest_filter),
165 spawn_results = pool.imap_unordered(_Spawn, spawn_args)
168 collect_results = pool.imap_unordered(_Collect, spawn_results)
169 for result in collect_results:
170 exit_codes.append(result)
171 successes = sum(1 for x in exit_codes if x == 0)
172 errors = sum(1 for x in exit_codes if x == INTERNAL_ERROR_EXIT_CODE)
173 failures = len(exit_codes) - successes - errors
174 clear_to_eol = '\033[K'
175 print('\r[%d/%d] collected: '
176 '%d successes, %d failures, %d bot errors...%s' % (len(exit_codes),
177 args.copies, successes, failures, errors, clear_to_eol)),
181 print 'Results logs collected into', os.path.abspath(args.results) + '.'
188 if __name__ == '__main__':