2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Run Performance Test Bisect Tool
8 This script is used by a trybot to run the src/tools/bisect-perf-regression.py
9 script with the parameters specified in run-bisect-perf-regression.cfg. It will
10 check out a copy of the depot in a subdirectory 'bisect' of the working
11 directory provided, and run the bisect-perf-regression.py script there.
23 bisect = imp.load_source('bisect-perf-regression',
24 os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])),
25 'bisect-perf-regression.py'))
28 CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
29 CROS_IP_ENV = 'BISECT_CROS_IP'
34 def __init__(self, path_to_goma):
35 self._abs_path_to_goma = None
36 self._abs_path_to_goma_file = None
38 self._abs_path_to_goma = os.path.abspath(path_to_goma)
39 self._abs_path_to_goma_file = self._GetExecutablePath(
40 self._abs_path_to_goma)
43 if self._HasGOMAPath():
47 def __exit__(self, *_):
48 if self._HasGOMAPath():
51 def _HasGOMAPath(self):
52 return bool(self._abs_path_to_goma)
54 def _GetExecutablePath(self, path_to_goma):
56 return os.path.join(path_to_goma, 'goma_ctl.bat')
58 return os.path.join(path_to_goma, 'goma_ctl.sh')
60 def _SetupEnvVars(self):
62 os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
64 os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
67 os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma,
70 def _SetupAndStart(self):
71 """Sets up GOMA and launches it.
74 path_to_goma: Path to goma directory.
77 True if successful."""
80 # Sometimes goma is lingering around if something went bad on a previous
81 # run. Stop it before starting a new process. Can ignore the return code
82 # since it will return an error if it wasn't running.
85 if subprocess.call([self._abs_path_to_goma_file, 'start']):
86 raise RuntimeError('GOMA failed to start.')
89 subprocess.call([self._abs_path_to_goma_file, 'stop'])
93 def _LoadConfigFile(path_to_file):
94 """Attempts to load the specified config file as a module
95 and grab the global config dict.
98 path_to_file: Path to the file.
101 The config dict which should be formatted as follows:
102 {'command': string, 'good_revision': string, 'bad_revision': string
103 'metric': string, etc...}.
104 Returns None on failure.
108 execfile(path_to_file, local_vars)
110 return local_vars['config']
113 traceback.print_exc()
118 def _OutputFailedResults(text_to_print):
119 bisect_utils.OutputAnnotationStepStart('Results - Failed')
123 bisect_utils.OutputAnnotationStepClosed()
126 def _CreateBisectOptionsFromConfig(config):
128 opts_dict['command'] = config['command']
129 opts_dict['metric'] = config['metric']
131 if config['repeat_count']:
132 opts_dict['repeat_test_count'] = int(config['repeat_count'])
134 if config['truncate_percent']:
135 opts_dict['truncate_percent'] = int(config['truncate_percent'])
137 if config['max_time_minutes']:
138 opts_dict['max_time_minutes'] = int(config['max_time_minutes'])
140 if config.has_key('use_goma'):
141 opts_dict['use_goma'] = config['use_goma']
143 opts_dict['build_preference'] = 'ninja'
144 opts_dict['output_buildbot_annotations'] = True
146 if '--browser=cros' in config['command']:
147 opts_dict['target_platform'] = 'cros'
149 if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
150 opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV]
151 opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV]
153 raise RuntimeError('Cros build selected, but BISECT_CROS_IP or'
154 'BISECT_CROS_BOARD undefined.')
155 elif 'android' in config['command']:
156 opts_dict['target_platform'] = 'android'
158 return bisect.BisectOptions.FromDict(opts_dict)
161 def _RunPerformanceTest(config, path_to_file):
162 # Bisect script expects to be run from src
163 os.chdir(os.path.join(path_to_file, '..'))
165 bisect_utils.OutputAnnotationStepStart('Building With Patch')
167 opts = _CreateBisectOptionsFromConfig(config)
168 b = bisect.BisectPerformanceMetrics(None, opts)
170 if bisect_utils.RunGClient(['runhooks']):
171 raise RuntimeError('Failed to run gclient runhooks')
173 if not b.BuildCurrentRevision('chromium'):
174 raise RuntimeError('Patched version failed to build.')
176 bisect_utils.OutputAnnotationStepClosed()
177 bisect_utils.OutputAnnotationStepStart('Running With Patch')
179 results_with_patch = b.RunPerformanceTestAndParseResults(
180 opts.command, opts.metric, reset_on_first_run=True, results_label='Patch')
182 if results_with_patch[1]:
183 raise RuntimeError('Patched version failed to run performance test.')
185 bisect_utils.OutputAnnotationStepClosed()
187 bisect_utils.OutputAnnotationStepStart('Reverting Patch')
188 if bisect_utils.RunGClient(['revert']):
189 raise RuntimeError('Failed to run gclient runhooks')
190 bisect_utils.OutputAnnotationStepClosed()
192 bisect_utils.OutputAnnotationStepStart('Building Without Patch')
194 if bisect_utils.RunGClient(['runhooks']):
195 raise RuntimeError('Failed to run gclient runhooks')
197 if not b.BuildCurrentRevision('chromium'):
198 raise RuntimeError('Unpatched version failed to build.')
200 bisect_utils.OutputAnnotationStepClosed()
201 bisect_utils.OutputAnnotationStepStart('Running Without Patch')
203 results_without_patch = b.RunPerformanceTestAndParseResults(
204 opts.command, opts.metric, upload_on_last_run=True, results_label='ToT')
206 if results_without_patch[1]:
207 raise RuntimeError('Unpatched version failed to run performance test.')
209 # Find the link to the cloud stored results file.
210 output = results_without_patch[2]
211 cloud_file_link = [t for t in output.splitlines()
212 if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
214 # What we're getting here is basically "View online at http://..." so parse
215 # out just the url portion.
216 cloud_file_link = cloud_file_link[0]
217 cloud_file_link = [t for t in cloud_file_link.split(' ')
218 if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
219 assert cloud_file_link, "Couldn't parse url from output."
220 cloud_file_link = cloud_file_link[0]
224 # Calculate the % difference in the means of the 2 runs.
225 percent_diff_in_means = (results_with_patch[0]['mean'] /
226 max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
227 std_err = bisect.CalculatePooledStandardError(
228 [results_with_patch[0]['values'], results_without_patch[0]['values']])
230 bisect_utils.OutputAnnotationStepClosed()
231 bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
232 (percent_diff_in_means, std_err))
233 print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
234 'Std. Error'.center(20, ' '))
235 print ' %s %s %s' % ('Patch'.center(10, ' '),
236 ('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
237 ('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
238 print ' %s %s %s' % ('No Patch'.center(10, ' '),
239 ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
240 ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
242 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
243 bisect_utils.OutputAnnotationStepClosed()
246 def _SetupAndRunPerformanceTest(config, path_to_file, path_to_goma):
247 """Attempts to build and run the current revision with and without the
248 current patch, with the parameters passed in.
251 config: The config read from run-perf-test.cfg.
252 path_to_file: Path to the bisect-perf-regression.py script.
253 path_to_goma: Path to goma directory.
256 0 on success, otherwise 1.
259 with Goma(path_to_goma) as goma:
260 config['use_goma'] = bool(path_to_goma)
261 _RunPerformanceTest(config, path_to_file)
263 except RuntimeError, e:
264 bisect_utils.OutputAnnotationStepClosed()
265 _OutputFailedResults('Error: %s' % e.message)
269 def _RunBisectionScript(config, working_directory, path_to_file, path_to_goma,
271 """Attempts to execute src/tools/bisect-perf-regression.py with the parameters
275 config: A dict containing the parameters to pass to the script.
276 working_directory: A working directory to provide to the
277 bisect-perf-regression.py script, where it will store it's own copy of
279 path_to_file: Path to the bisect-perf-regression.py script.
280 path_to_goma: Path to goma directory.
281 dry_run: Do a dry run, skipping sync, build, and performance testing steps.
284 0 on success, otherwise 1.
286 bisect_utils.OutputAnnotationStepStart('Config')
288 for k, v in config.iteritems():
289 print ' %s : %s' % (k, v)
291 bisect_utils.OutputAnnotationStepClosed()
293 cmd = ['python', os.path.join(path_to_file, 'bisect-perf-regression.py'),
294 '-c', config['command'],
295 '-g', config['good_revision'],
296 '-b', config['bad_revision'],
297 '-m', config['metric'],
298 '--working_directory', working_directory,
299 '--output_buildbot_annotations']
301 if config['repeat_count']:
302 cmd.extend(['-r', config['repeat_count']])
304 if config['truncate_percent']:
305 cmd.extend(['-t', config['truncate_percent']])
307 if config['max_time_minutes']:
308 cmd.extend(['--max_time_minutes', config['max_time_minutes']])
310 cmd.extend(['--build_preference', 'ninja'])
312 if '--browser=cros' in config['command']:
313 cmd.extend(['--target_platform', 'cros'])
315 if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
316 cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]])
317 cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]])
319 print 'Error: Cros build selected, but BISECT_CROS_IP or'\
320 'BISECT_CROS_BOARD undefined.'
324 if 'android' in config['command']:
325 cmd.extend(['--target_platform', 'android'])
328 cmd.append('--use_goma')
331 cmd.extend(['--debug_ignore_build', '--debug_ignore_sync',
332 '--debug_ignore_perf_test'])
333 cmd = [str(c) for c in cmd]
335 with Goma(path_to_goma) as goma:
336 return_code = subprocess.call(cmd)
339 print 'Error: bisect-perf-regression.py returned with error %d' %\
348 usage = ('%prog [options] [-- chromium-options]\n'
349 'Used by a trybot to run the bisection script using the parameters'
350 ' provided in the run-bisect-perf-regression.cfg file.')
352 parser = optparse.OptionParser(usage=usage)
353 parser.add_option('-w', '--working_directory',
355 help='A working directory to supply to the bisection '
356 'script, which will use it as the location to checkout '
357 'a copy of the chromium depot.')
358 parser.add_option('-p', '--path_to_goma',
360 help='Path to goma directory. If this is supplied, goma '
361 'builds will be enabled.')
362 parser.add_option('--dry_run',
364 help='The script will perform the full bisect, but '
365 'without syncing, building, or running the performance '
367 (opts, args) = parser.parse_args()
369 path_to_current_directory = os.path.abspath(os.path.dirname(sys.argv[0]))
370 path_to_bisect_cfg = os.path.join(path_to_current_directory,
371 'run-bisect-perf-regression.cfg')
373 config = _LoadConfigFile(path_to_bisect_cfg)
375 # Check if the config is empty
376 config_has_values = [v for v in config.values() if v]
378 if config and config_has_values:
379 if not opts.working_directory:
380 print 'Error: missing required parameter: --working_directory'
385 return _RunBisectionScript(config, opts.working_directory,
386 path_to_current_directory, opts.path_to_goma, opts.dry_run)
388 perf_cfg_files = ['run-perf-test.cfg', os.path.join('..', 'third_party',
389 'WebKit', 'Tools', 'run-perf-test.cfg')]
391 for current_perf_cfg_file in perf_cfg_files:
392 path_to_perf_cfg = os.path.join(
393 os.path.abspath(os.path.dirname(sys.argv[0])), current_perf_cfg_file)
395 config = _LoadConfigFile(path_to_perf_cfg)
396 config_has_values = [v for v in config.values() if v]
398 if config and config_has_values:
399 return _SetupAndRunPerformanceTest(config, path_to_current_directory,
402 print 'Error: Could not load config file. Double check your changes to '\
403 'run-bisect-perf-regression.cfg/run-perf-test.cfg for syntax errors.'
408 if __name__ == '__main__':