2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Run Performance Test Bisect Tool
8 This script is used by a try bot to run the bisect script with the parameters
9 specified in the bisect config file. It checks out a copy of the depot in
10 a subdirectory 'bisect' of the working directory provided, annd runs the
21 from auto_bisect import bisect_perf_regression
22 from auto_bisect import bisect_utils
23 from auto_bisect import math_utils
25 CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
26 CROS_IP_ENV = 'BISECT_CROS_IP'
28 SCRIPT_DIR = os.path.dirname(__file__)
29 SRC_DIR = os.path.join(SCRIPT_DIR, os.path.pardir)
30 BISECT_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'auto_bisect', 'bisect.cfg')
31 RUN_TEST_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'run-perf-test.cfg')
32 WEBKIT_RUN_TEST_CONFIG_PATH = os.path.join(
33 SRC_DIR, 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg')
34 BISECT_SCRIPT_DIR = os.path.join(SCRIPT_DIR, 'auto_bisect')
39 def __init__(self, path_to_goma):
40 self._abs_path_to_goma = None
41 self._abs_path_to_goma_file = None
44 self._abs_path_to_goma = os.path.abspath(path_to_goma)
45 filename = 'goma_ctl.bat' if os.name == 'nt' else 'goma_ctl.sh'
46 self._abs_path_to_goma_file = os.path.join(self._abs_path_to_goma, filename)
49 if self._HasGomaPath():
53 def __exit__(self, *_):
54 if self._HasGomaPath():
57 def _HasGomaPath(self):
58 return bool(self._abs_path_to_goma)
60 def _SetupEnvVars(self):
62 os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
64 os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
67 os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma,
70 def _SetupAndStart(self):
71 """Sets up goma and launches it.
74 path_to_goma: Path to goma directory.
77 True if successful."""
80 # Sometimes goma is lingering around if something went bad on a previous
81 # run. Stop it before starting a new process. Can ignore the return code
82 # since it will return an error if it wasn't running.
85 if subprocess.call([self._abs_path_to_goma_file, 'start']):
86 raise RuntimeError('Goma failed to start.')
89 subprocess.call([self._abs_path_to_goma_file, 'stop'])
92 def _LoadConfigFile(config_file_path):
93 """Attempts to load the specified config file as a module
94 and grab the global config dict.
97 config_file_path: Path to the config file.
100 If successful, returns the config dict loaded from the file. If no
101 such dictionary could be loaded, returns the empty dictionary.
105 execfile(config_file_path, local_vars)
106 return local_vars['config']
109 traceback.print_exc()
114 def _ValidateConfigFile(config_contents, valid_parameters):
115 """Validates the config file contents, checking whether all values are
119 config_contents: A config dictionary.
120 valid_parameters: A list of parameters to check for.
125 for parameter in valid_parameters:
126 if parameter not in config_contents:
128 value = config_contents[parameter]
129 if not value or type(value) is not str:
134 def _ValidatePerfConfigFile(config_contents):
135 """Validates the perf config file contents.
137 This is used when we're doing a perf try job, rather than a bisect.
138 The config file is called run-perf-test.cfg by default.
140 The parameters checked are the required parameters; any additional optional
141 parameters won't be checked and validation will still pass.
144 config_contents: A config dictionary.
155 return _ValidateConfigFile(config_contents, valid_parameters)
158 def _ValidateBisectConfigFile(config_contents):
159 """Validates the bisect config file contents.
161 The parameters checked are the required parameters; any additional optional
162 parameters won't be checked and validation will still pass.
165 config_contents: A config dictionary.
179 return _ValidateConfigFile(config_contents, valid_params)
182 def _OutputFailedResults(text_to_print):
183 bisect_utils.OutputAnnotationStepStart('Results - Failed')
187 bisect_utils.OutputAnnotationStepClosed()
190 def _CreateBisectOptionsFromConfig(config):
191 print config['command']
193 opts_dict['command'] = config['command']
194 opts_dict['metric'] = config.get('metric')
196 if config['repeat_count']:
197 opts_dict['repeat_test_count'] = int(config['repeat_count'])
199 if config['truncate_percent']:
200 opts_dict['truncate_percent'] = int(config['truncate_percent'])
202 if config['max_time_minutes']:
203 opts_dict['max_time_minutes'] = int(config['max_time_minutes'])
205 if config.has_key('use_goma'):
206 opts_dict['use_goma'] = config['use_goma']
207 if config.has_key('goma_dir'):
208 opts_dict['goma_dir'] = config['goma_dir']
210 opts_dict['build_preference'] = 'ninja'
211 opts_dict['output_buildbot_annotations'] = True
213 if '--browser=cros' in config['command']:
214 opts_dict['target_platform'] = 'cros'
216 if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
217 opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV]
218 opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV]
220 raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or'
221 'BISECT_CROS_BOARD undefined.')
222 elif 'android' in config['command']:
223 if 'android-chrome-shell' in config['command']:
224 opts_dict['target_platform'] = 'android'
225 elif 'android-chrome' in config['command']:
226 opts_dict['target_platform'] = 'android-chrome'
228 opts_dict['target_platform'] = 'android'
230 return bisect_perf_regression.BisectOptions.FromDict(opts_dict)
233 def _RunPerformanceTest(config):
234 """Runs a performance test with and without the current patch.
237 config: Contents of the config file, a dictionary.
239 Attempts to build and run the current revision with and without the
240 current patch, with the parameters passed in.
242 # Bisect script expects to be run from the src directory
245 bisect_utils.OutputAnnotationStepStart('Building With Patch')
247 opts = _CreateBisectOptionsFromConfig(config)
248 b = bisect_perf_regression.BisectPerformanceMetrics(None, opts)
250 if bisect_utils.RunGClient(['runhooks']):
251 raise RuntimeError('Failed to run gclient runhooks')
253 if not b.BuildCurrentRevision('chromium'):
254 raise RuntimeError('Patched version failed to build.')
256 bisect_utils.OutputAnnotationStepClosed()
257 bisect_utils.OutputAnnotationStepStart('Running With Patch')
259 results_with_patch = b.RunPerformanceTestAndParseResults(
260 opts.command, opts.metric, reset_on_first_run=True, results_label='Patch')
262 if results_with_patch[1]:
263 raise RuntimeError('Patched version failed to run performance test.')
265 bisect_utils.OutputAnnotationStepClosed()
267 bisect_utils.OutputAnnotationStepStart('Reverting Patch')
268 # TODO: When this is re-written to recipes, this should use bot_update's
269 # revert mechanism to fully revert the client. But for now, since we know that
270 # the perf try bot currently only supports src/ and src/third_party/WebKit, we
271 # simply reset those two directories.
272 bisect_utils.CheckRunGit(['reset', '--hard'])
273 bisect_utils.CheckRunGit(['reset', '--hard'],
274 os.path.join('third_party', 'WebKit'))
275 bisect_utils.OutputAnnotationStepClosed()
277 bisect_utils.OutputAnnotationStepStart('Building Without Patch')
279 if bisect_utils.RunGClient(['runhooks']):
280 raise RuntimeError('Failed to run gclient runhooks')
282 if not b.BuildCurrentRevision('chromium'):
283 raise RuntimeError('Unpatched version failed to build.')
285 bisect_utils.OutputAnnotationStepClosed()
286 bisect_utils.OutputAnnotationStepStart('Running Without Patch')
288 results_without_patch = b.RunPerformanceTestAndParseResults(
289 opts.command, opts.metric, upload_on_last_run=True, results_label='ToT')
291 if results_without_patch[1]:
292 raise RuntimeError('Unpatched version failed to run performance test.')
294 # Find the link to the cloud stored results file.
295 output = results_without_patch[2]
296 cloud_file_link = [t for t in output.splitlines()
297 if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
299 # What we're getting here is basically "View online at http://..." so parse
300 # out just the URL portion.
301 cloud_file_link = cloud_file_link[0]
302 cloud_file_link = [t for t in cloud_file_link.split(' ')
303 if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
304 assert cloud_file_link, 'Couldn\'t parse URL from output.'
305 cloud_file_link = cloud_file_link[0]
309 # Calculate the % difference in the means of the 2 runs.
310 percent_diff_in_means = None
312 if (results_with_patch[0].has_key('mean') and
313 results_with_patch[0].has_key('values')):
314 percent_diff_in_means = (results_with_patch[0]['mean'] /
315 max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
316 std_err = math_utils.PooledStandardError(
317 [results_with_patch[0]['values'], results_without_patch[0]['values']])
319 bisect_utils.OutputAnnotationStepClosed()
320 if percent_diff_in_means is not None and std_err is not None:
321 bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
322 (percent_diff_in_means, std_err))
323 print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
324 'Std. Error'.center(20, ' '))
325 print ' %s %s %s' % ('Patch'.center(10, ' '),
326 ('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
327 ('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
328 print ' %s %s %s' % ('No Patch'.center(10, ' '),
329 ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
330 ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
332 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
333 bisect_utils.OutputAnnotationStepClosed()
334 elif cloud_file_link:
335 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
338 def _SetupAndRunPerformanceTest(config, path_to_goma):
339 """Attempts to build and run the current revision with and without the
340 current patch, with the parameters passed in.
343 config: The config read from run-perf-test.cfg.
344 path_to_goma: Path to goma directory.
347 An exit code: 0 on success, otherwise 1.
349 if platform.release() == 'XP':
350 print 'Windows XP is not supported for perf try jobs because it lacks '
351 print 'goma support. Please refer to crbug.com/330900.'
354 with Goma(path_to_goma) as _:
355 config['use_goma'] = bool(path_to_goma)
356 if config['use_goma']:
357 config['goma_dir'] = os.path.abspath(path_to_goma)
358 _RunPerformanceTest(config)
360 except RuntimeError, e:
361 bisect_utils.OutputAnnotationStepClosed()
362 _OutputFailedResults('Error: %s' % e.message)
366 def _RunBisectionScript(
367 config, working_directory, path_to_goma, path_to_extra_src, dry_run):
368 """Attempts to execute the bisect script with the given parameters.
371 config: A dict containing the parameters to pass to the script.
372 working_directory: A working directory to provide to the bisect script,
373 where it will store it's own copy of the depot.
374 path_to_goma: Path to goma directory.
375 path_to_extra_src: Path to extra source file.
376 dry_run: Do a dry run, skipping sync, build, and performance testing steps.
379 An exit status code: 0 on success, otherwise 1.
381 _PrintConfigStep(config)
383 cmd = ['python', os.path.join(BISECT_SCRIPT_DIR, 'bisect_perf_regression.py'),
384 '-c', config['command'],
385 '-g', config['good_revision'],
386 '-b', config['bad_revision'],
387 '-m', config['metric'],
388 '--working_directory', working_directory,
389 '--output_buildbot_annotations']
391 if config.get('metric'):
392 cmd.extend(['-m', config['metric']])
394 if config['repeat_count']:
395 cmd.extend(['-r', config['repeat_count']])
397 if config['truncate_percent']:
398 cmd.extend(['-t', config['truncate_percent']])
400 if config['max_time_minutes']:
401 cmd.extend(['--max_time_minutes', config['max_time_minutes']])
403 if config.has_key('bisect_mode'):
404 cmd.extend(['--bisect_mode', config['bisect_mode']])
406 cmd.extend(['--build_preference', 'ninja'])
408 if '--browser=cros' in config['command']:
409 cmd.extend(['--target_platform', 'cros'])
411 if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
412 cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]])
413 cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]])
415 print ('Error: Cros build selected, but BISECT_CROS_IP or'
416 'BISECT_CROS_BOARD undefined.\n')
419 if 'android' in config['command']:
420 if 'android-chrome-shell' in config['command']:
421 cmd.extend(['--target_platform', 'android'])
422 elif 'android-chrome' in config['command']:
423 cmd.extend(['--target_platform', 'android-chrome'])
425 cmd.extend(['--target_platform', 'android'])
428 # For Windows XP platforms, goma service is not supported.
429 # Moreover we don't compile chrome when gs_bucket flag is set instead
430 # use builds archives, therefore ignore goma service for Windows XP.
431 # See http://crbug.com/330900.
432 if config.get('gs_bucket') and platform.release() == 'XP':
433 print ('Goma doesn\'t have a win32 binary, therefore it is not supported '
434 'on Windows XP platform. Please refer to crbug.com/330900.')
436 cmd.append('--use_goma')
438 if path_to_extra_src:
439 cmd.extend(['--extra_src', path_to_extra_src])
441 # These flags are used to download build archives from cloud storage if
442 # available, otherwise will post a try_job_http request to build it on the
444 if config.get('gs_bucket'):
445 if config.get('builder_host') and config.get('builder_port'):
446 cmd.extend(['--gs_bucket', config['gs_bucket'],
447 '--builder_host', config['builder_host'],
448 '--builder_port', config['builder_port']
451 print ('Error: Specified gs_bucket, but missing builder_host or '
452 'builder_port information in config.')
456 cmd.extend(['--debug_ignore_build', '--debug_ignore_sync',
457 '--debug_ignore_perf_test'])
458 cmd = [str(c) for c in cmd]
460 with Goma(path_to_goma) as _:
461 return_code = subprocess.call(cmd)
464 print ('Error: bisect_perf_regression.py returned with error %d\n'
470 def _PrintConfigStep(config):
471 """Prints out the given config, along with Buildbot annotations."""
472 bisect_utils.OutputAnnotationStepStart('Config')
474 for k, v in config.iteritems():
475 print ' %s : %s' % (k, v)
477 bisect_utils.OutputAnnotationStepClosed()
481 """Returns the options parser for run-bisect-perf-regression.py."""
482 usage = ('%prog [options] [-- chromium-options]\n'
483 'Used by a try bot to run the bisection script using the parameters'
484 ' provided in the auto_bisect/bisect.cfg file.')
485 parser = optparse.OptionParser(usage=usage)
486 parser.add_option('-w', '--working_directory',
488 help='A working directory to supply to the bisection '
489 'script, which will use it as the location to checkout '
490 'a copy of the chromium depot.')
491 parser.add_option('-p', '--path_to_goma',
493 help='Path to goma directory. If this is supplied, goma '
494 'builds will be enabled.')
495 parser.add_option('--path_to_config',
497 help='Path to the config file to use. If this is supplied, '
498 'the bisect script will use this to override the default '
499 'config file path. The script will attempt to load it '
500 'as a bisect config first, then a perf config.')
501 parser.add_option('--extra_src',
503 help='Path to extra source file. If this is supplied, '
504 'bisect script will use this to override default behavior.')
505 parser.add_option('--dry_run',
507 help='The script will perform the full bisect, but '
508 'without syncing, building, or running the performance '
514 """Entry point for run-bisect-perf-regression.py.
516 Reads the config file, and then tries to either bisect a regression or
517 just run a performance test, depending on the particular config parameters
518 specified in the config file.
520 parser = _OptionParser()
521 opts, _ = parser.parse_args()
523 # Use the default config file path unless one was specified.
524 config_path = BISECT_CONFIG_PATH
525 if opts.path_to_config:
526 config_path = opts.path_to_config
527 config = _LoadConfigFile(config_path)
529 # Check if the config is valid for running bisect job.
530 config_is_valid = _ValidateBisectConfigFile(config)
532 if config and config_is_valid:
533 if not opts.working_directory:
534 print 'Error: missing required parameter: --working_directory\n'
538 return _RunBisectionScript(
539 config, opts.working_directory, opts.path_to_goma, opts.extra_src,
542 # If it wasn't valid for running a bisect, then maybe the user wanted
543 # to run a perf test instead of a bisect job. Try reading any possible
544 # perf test config files.
545 perf_cfg_files = [RUN_TEST_CONFIG_PATH, WEBKIT_RUN_TEST_CONFIG_PATH]
546 for current_perf_cfg_file in perf_cfg_files:
547 if opts.path_to_config:
548 path_to_perf_cfg = opts.path_to_config
550 path_to_perf_cfg = os.path.join(
551 os.path.abspath(os.path.dirname(sys.argv[0])),
552 current_perf_cfg_file)
554 config = _LoadConfigFile(path_to_perf_cfg)
555 config_is_valid = _ValidatePerfConfigFile(config)
557 if config and config_is_valid:
558 return _SetupAndRunPerformanceTest(config, opts.path_to_goma)
560 print ('Error: Could not load config file. Double check your changes to '
561 'auto_bisect/bisect.cfg or run-perf-test.cfg for syntax errors.\n')
565 if __name__ == '__main__':