3 # Copyright 2017 The Chromium Authors
4 # Use of this source code is governed by a BSD-style license that can be
5 # found in the LICENSE file.
7 """Runs a gtest-based test on Swarming, optionally many times, collecting the
8 output of the runs into a directory. Useful for flake checking, and faster than
9 using trybots by avoiding repeated bot_update, compile, archive, etc. and
10 allowing greater parallelism.
12 To use, run in a new shell (it blocks until all Swarming jobs complete):
14 tools/run-swarmed.py out/rel base_unittests
16 The logs of the runs will be stored in results/ (or specify a results directory
17 with --results=some_dir). You can then do something like `grep -L SUCCESS
18 results/*` to find the tests that failed or otherwise process the log files.
20 See //docs/workflow/debugging-with-swarming.md for more details.
28 import multiprocessing.dummy
35 CHROMIUM_ROOT = os.path.join(os.path.dirname(__file__), os.pardir)
36 BUILD_DIR = os.path.join(CHROMIUM_ROOT, 'build')
38 if BUILD_DIR not in sys.path:
39 sys.path.insert(0, BUILD_DIR)
42 INTERNAL_ERROR_EXIT_CODE = -1000
44 DEFAULT_ANDROID_DEVICE_TYPE = "walleye"
48 """Triggers a swarming job. The arguments passed are:
49 - The index of the job;
50 - The command line arguments object;
51 - The digest of test files.
53 The return value is passed to a collect-style map() and consists of:
54 - The index of the job;
55 - The json file created by triggering and used to collect results;
56 - The command line arguments object.
60 except Exception as e:
66 index, args, cas_digest, swarming_command = args
68 json_file = os.path.join(args.results, '%d.json' % index)
70 'tools/luci-go/swarming',
73 'https://chromium-swarm.appspot.com',
78 '-tag=purpose:user-debug-run-swarmed',
80 if args.target_os == 'fuchsia':
90 elif args.target_os == 'android':
91 if args.arch == 'x86':
92 # No x86 Android devices are available in swarming. So assume we want to
93 # run on emulators when building for x86 on Android.
94 args.swarming_os = 'Linux'
95 args.pool = 'chromium.tests.avd'
96 # generic_android28 == Android P emulator. See //tools/android/avd/proto/
99 '--avd-config=../../tools/android/avd/proto/generic_android28.textpb')
100 elif args.device_type is None and args.device_os is None:
101 # The aliases for device type are stored here:
102 # luci/appengine/swarming/ui2/modules/alias.js
103 # for example 'blueline' = 'Pixel 3'
104 trigger_args += ['-d', 'device_type=' + DEFAULT_ANDROID_DEVICE_TYPE]
105 elif args.target_os == 'ios':
106 print('WARNING: iOS support is quite limited.\n'
107 '1) --gtest_filter does not work with unit tests.\n' +
108 '2) Wildcards do not work with EG tests (--gtest_filter=Foo*).\n' +
109 '3) Some arguments are hardcoded (e.g. xcode version) and will ' +
110 'break over time. \n')
112 runner_args.append('--xcode-build-version=14c18')
113 runner_args.append('--xctest')
114 runner_args.append('--xcode-parallelization')
115 runner_args.append('--out-dir=./test-data')
116 runner_args.extend(['--platform', 'iPhone 13'])
117 runner_args.append('--version=15.5')
118 trigger_args.extend([
120 'chromium-tester@chops-service-accounts.iam.gserviceaccount.com'
122 trigger_args.extend(['-named-cache', 'runtime_ios_15_5=Runtime-ios-15.5'])
123 trigger_args.extend(['-named-cache', 'xcode_ios_14c18=Xcode.app'])
124 trigger_args.extend([
125 '-cipd-package', '.:infra/tools/mac_toolchain/${platform}=' +
126 'git_revision:59ddedfe3849abf560cbe0b41bb8e431041cd2bb'
129 if args.arch != 'detect':
136 trigger_args += ['-d', 'device_type=' + args.device_type]
139 trigger_args += ['-d', 'device_os=' + args.device_os]
142 trigger_args += ['-d', 'gpu=' + args.gpu]
144 if not args.no_test_flags:
145 # These flags are recognized by our test runners, but do not work
146 # when running custom scripts.
148 '--test-launcher-summary-output=${ISOLATED_OUTDIR}/output.json'
150 if 'junit' not in args.target_name:
151 runner_args += ['--system-log-file=${ISOLATED_OUTDIR}/system_log']
152 if args.gtest_filter:
153 runner_args.append('--gtest_filter=' + args.gtest_filter)
154 if args.gtest_repeat:
155 runner_args.append('--gtest_repeat=' + args.gtest_repeat)
156 if args.test_launcher_shard_index and args.test_launcher_total_shards:
157 runner_args.append('--test-launcher-shard-index=' +
158 args.test_launcher_shard_index)
159 runner_args.append('--test-launcher-total-shards=' +
160 args.test_launcher_total_shards)
161 elif args.target_os == 'fuchsia':
163 'testing/buildbot/filters/fuchsia.' + args.target_name + '.filter'
164 if os.path.isfile(filter_file):
165 runner_args.append('--test-launcher-filter-file=../../' + filter_file)
167 runner_args.extend(args.runner_args)
169 trigger_args.extend(['-d', 'os=' + args.swarming_os])
170 trigger_args.extend(['-d', 'pool=' + args.pool])
171 trigger_args.extend(['--relative-cwd', args.out_dir, '--'])
172 trigger_args.extend(swarming_command)
173 trigger_args.extend(runner_args)
175 with open(os.devnull, 'w') as nul:
176 subprocess.check_call(trigger_args, stdout=nul)
177 return (index, json_file, args)
180 def _Collect(spawn_result):
181 if spawn_result is None:
184 index, json_file, args = spawn_result
185 with open(json_file) as f:
186 task_json = json.load(f)
187 task_ids = [task['task_id'] for task in task_json['tasks']]
190 print('Task {}: https://chromium-swarm.appspot.com/task?id={}'.format(
192 p = subprocess.Popen(
194 'tools/luci-go/swarming',
197 'https://chromium-swarm.appspot.com',
198 '--task-output-stdout=console',
200 stdout=subprocess.PIPE,
201 stderr=subprocess.STDOUT)
202 stdout = p.communicate()[0]
203 if p.returncode != 0 and len(stdout) < 2**10 and 'Internal error!' in stdout:
204 exit_code = INTERNAL_ERROR_EXIT_CODE
205 file_suffix = '.INTERNAL_ERROR'
207 exit_code = p.returncode
208 file_suffix = '' if exit_code == 0 else '.FAILED'
209 filename = '%d%s.stdout.txt' % (index, file_suffix)
210 with open(os.path.join(args.results, filename), 'wb') as f:
216 parser = argparse.ArgumentParser()
217 parser.add_argument('--swarming-os', help='OS specifier for Swarming.')
218 parser.add_argument('--target-os', default='detect', help='gn target_os')
219 parser.add_argument('--arch', '-a', default='detect',
220 help='CPU architecture of the test binary.')
221 parser.add_argument('--build',
224 help='Build before isolating.')
225 parser.add_argument('--no-build',
227 action='store_false',
228 help='Do not build, just isolate (default).')
229 parser.add_argument('--isolate-map-file', '-i',
230 help='path to isolate map file if not using default')
231 parser.add_argument('--copies', '-n', type=int, default=1,
232 help='Number of copies to spawn.')
234 '--device-os', help='Run tests on the given version of Android.')
235 parser.add_argument('--device-type',
236 help='device_type specifier for Swarming'
237 ' from https://chromium-swarm.appspot.com/botlist .')
238 parser.add_argument('--gpu',
239 help='gpu specifier for Swarming'
240 ' from https://chromium-swarm.appspot.com/botlist .')
241 parser.add_argument('--pool',
242 default='chromium.tests',
243 help='Use the given swarming pool.')
244 parser.add_argument('--results', '-r', default='results',
245 help='Directory in which to store results.')
248 help='Deprecated. Pass as test runner arg instead, like \'-- '
249 '--gtest_filter="*#testFoo"\'')
252 help='Deprecated. Pass as test runner arg instead, like \'-- '
253 '--gtest_repeat=99\'')
255 '--test-launcher-shard-index',
256 help='Shard index to run. Use with --test-launcher-total-shards.')
257 parser.add_argument('--test-launcher-total-shards',
258 help='Number of shards to split the test into. Use with'
259 ' --test-launcher-shard-index.')
260 parser.add_argument('--no-test-flags', action='store_true',
261 help='Do not add --test-launcher-summary-output and '
262 '--system-log-file flags to the comment.')
263 parser.add_argument('out_dir', type=str, help='Build directory.')
264 parser.add_argument('target_name', type=str, help='Name of target to run.')
269 help='Arguments to pass to the test runner, e.g. gtest_filter and '
272 args = parser.parse_intermixed_args()
274 with open(os.path.join(args.out_dir, 'args.gn')) as f:
275 gn_args = gn_helpers.FromGNArgs(f.read())
277 if args.target_os == 'detect':
278 if 'target_os' in gn_args:
279 args.target_os = gn_args['target_os'].strip('"')
287 if args.swarming_os is None:
293 'android': 'Android',
297 if args.target_os == 'win' and args.target_name.endswith('.exe'):
298 # The machinery expects not to have a '.exe' suffix.
299 args.target_name = os.path.splitext(args.target_name)[0]
301 # Determine the CPU architecture of the test binary, if not specified.
302 if args.arch == 'detect':
303 if args.target_os == 'ios':
304 print('iOS must specify --arch. Probably arm64 or x86-64.')
306 if args.target_os not in ('android', 'mac', 'win'):
307 executable_info = subprocess.check_output(
308 ['file', os.path.join(args.out_dir, args.target_name)], text=True)
309 if 'ARM aarch64' in executable_info:
313 elif args.target_os == 'android':
314 args.arch = gn_args.get('target_cpu', 'detect')
316 mb_cmd = [sys.executable, 'tools/mb/mb.py', 'isolate']
318 mb_cmd.append('--no-build')
319 if args.isolate_map_file:
320 mb_cmd += ['--isolate-map-file', args.isolate_map_file]
321 mb_cmd += ['//' + args.out_dir, args.target_name]
322 subprocess.check_call(mb_cmd, shell=os.name == 'nt')
324 print('If you get authentication errors, follow:')
326 ' https://chromium.googlesource.com/chromium/src/+/HEAD/docs/workflow/debugging-with-swarming.md#authenticating'
329 print('Uploading to isolate server, this can take a while...')
330 isolate = os.path.join(args.out_dir, args.target_name + '.isolate')
331 archive_json = os.path.join(args.out_dir, args.target_name + '.archive.json')
332 subprocess.check_output([
333 'tools/luci-go/isolate', 'archive', '-cas-instance', 'chromium-swarm',
334 '-isolate', isolate, '-dump-json', archive_json
336 with open(archive_json) as f:
337 cas_digest = json.load(f).get(args.target_name)
340 sys.executable, 'tools/mb/mb.py', 'get-swarming-command', '--as-list'
343 mb_cmd.append('--no-build')
344 if args.isolate_map_file:
345 mb_cmd += ['--isolate-map-file', args.isolate_map_file]
346 mb_cmd += ['//' + args.out_dir, args.target_name]
347 mb_output = subprocess.check_output(mb_cmd, shell=os.name == 'nt')
348 swarming_cmd = json.loads(mb_output)
350 if os.path.isdir(args.results):
351 shutil.rmtree(args.results)
352 os.makedirs(args.results)
355 print('Triggering %d tasks...' % args.copies)
356 # Use dummy since threadpools give better exception messages
357 # than process pools do, and threads work fine for what we're doing.
358 pool = multiprocessing.dummy.Pool()
359 spawn_args = [(i, args, cas_digest, swarming_cmd)
360 for i in range(args.copies)]
361 spawn_results = pool.imap_unordered(_Spawn, spawn_args)
364 collect_results = pool.imap_unordered(_Collect, spawn_results)
365 for result in collect_results:
366 exit_codes.append(result)
367 successes = sum(1 for x in exit_codes if x == 0)
368 errors = sum(1 for x in exit_codes if x == INTERNAL_ERROR_EXIT_CODE)
369 failures = len(exit_codes) - successes - errors
370 clear_to_eol = '\033[K'
372 '\r[%d/%d] collected: '
373 '%d successes, %d failures, %d bot errors...%s' %
374 (len(exit_codes), args.copies, successes, failures, errors,
380 print('Results logs collected into', os.path.abspath(args.results) + '.')
387 if __name__ == '__main__':