1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
13 from telemetry import decorators
14 from telemetry.core import browser_finder
15 from telemetry.core import browser_info
16 from telemetry.core import exceptions
17 from telemetry.core import util
18 from telemetry.core import wpr_modes
19 from telemetry.core.platform.profiler import profiler_finder
20 from telemetry.page import page_filter
21 from telemetry.page import page_test
22 from telemetry.page.actions import navigate
23 from telemetry.page.actions import page_action
24 from telemetry.results import results_options
25 from telemetry.util import cloud_storage
26 from telemetry.util import exception_formatter
27 from telemetry.value import failure
28 from telemetry.value import skip
31 class _RunState(object):
35 self._append_to_existing_wpr = False
36 self._last_archive_path = None
37 self._first_browser = True
38 self.profiler_dir = None
40 def StartBrowserIfNeeded(self, test, page_set, page, possible_browser,
41 credentials_path, archive_path, finder_options):
42 started_browser = not self.browser
45 test.CustomizeBrowserOptionsForSinglePage(page, finder_options)
46 possible_browser.SetReplayArchivePath(archive_path,
47 self._append_to_existing_wpr,
48 page_set.make_javascript_deterministic)
49 possible_browser.SetCredentialsPath(credentials_path)
50 self._last_archive_path = page.archive_path
52 test.WillStartBrowser(possible_browser.platform)
53 self.browser = possible_browser.Create()
54 test.DidStartBrowser(self.browser)
56 if self._first_browser:
57 self._first_browser = False
58 self.browser.credentials.WarnIfMissingCredentials(page_set)
59 logging.info('OS: %s %s',
60 self.browser.platform.GetOSName(),
61 self.browser.platform.GetOSVersionName())
62 if self.browser.supports_system_info:
63 system_info = self.browser.GetSystemInfo()
64 if system_info.model_name:
65 logging.info('Model: %s', system_info.model_name)
67 for i, device in enumerate(system_info.gpu.devices):
68 logging.info('GPU device %d: %s', i, device)
69 if system_info.gpu.aux_attributes:
70 logging.info('GPU Attributes:')
71 for k, v in sorted(system_info.gpu.aux_attributes.iteritems()):
72 logging.info(' %-20s: %s', k, v)
73 if system_info.gpu.feature_status:
74 logging.info('Feature Status:')
75 for k, v in sorted(system_info.gpu.feature_status.iteritems()):
76 logging.info(' %-20s: %s', k, v)
77 if system_info.gpu.driver_bug_workarounds:
78 logging.info('Driver Bug Workarounds:')
79 for workaround in system_info.gpu.driver_bug_workarounds:
80 logging.info(' %s', workaround)
82 logging.info('No GPU devices')
84 logging.warning('System info not supported')
86 # Set up WPR path if it changed.
87 if page.archive_path and self._last_archive_path != page.archive_path:
88 self.browser.SetReplayArchivePath(
90 self._append_to_existing_wpr,
91 page_set.make_javascript_deterministic)
92 self._last_archive_path = page.archive_path
94 if self.browser.supports_tab_control and test.close_tabs_before_run:
95 # Create a tab if there's none.
96 if len(self.browser.tabs) == 0:
97 self.browser.tabs.New()
99 # Ensure only one tab is open, unless the test is a multi-tab test.
100 if not test.is_multi_tab_test:
101 while len(self.browser.tabs) > 1:
102 self.browser.tabs[-1].Close()
104 # Must wait for tab to commit otherwise it can commit after the next
105 # navigation has begun and RenderFrameHostManager::DidNavigateMainFrame()
106 # will cancel the next navigation because it's pending. This manifests as
107 # the first navigation in a PageSet freezing indefinitly because the
108 # navigation was silently cancelled when |self.browser.tabs[0]| was
109 # committed. Only do this when we just started the browser, otherwise
110 # there are cases where previous pages in a PageSet never complete
111 # loading so we'll wait forever.
113 self.browser.tabs[0].WaitForDocumentReadyStateToBeComplete()
115 def StopBrowser(self):
120 # Restarting the state will also restart the wpr server. If we're
121 # recording, we need to continue adding into the same wpr archive,
123 self._append_to_existing_wpr = True
125 def StartProfiling(self, page, finder_options):
126 if not self.profiler_dir:
127 self.profiler_dir = tempfile.mkdtemp()
128 output_file = os.path.join(self.profiler_dir, page.file_safe_name)
129 is_repeating = (finder_options.page_repeat != 1 or
130 finder_options.pageset_repeat != 1)
132 output_file = util.GetSequentialFileName(output_file)
133 self.browser.platform.profiling_controller.Start(
134 finder_options.profiler, output_file)
136 def StopProfiling(self):
138 self.browser.platform.profiling_controller.Stop()
141 class PageState(object):
142 def __init__(self, page, tab):
146 self._did_login = False
148 def PreparePage(self, test=None):
149 if self.page.is_file:
150 self.tab.browser.SetHTTPServerDirectories(
151 self.page.page_set.serving_dirs | set([self.page.serving_dir]))
153 if self.page.credentials:
154 if not self.tab.browser.credentials.LoginNeeded(
155 self.tab, self.page.credentials):
156 raise page_test.Failure('Login as ' + self.page.credentials + ' failed')
157 self._did_login = True
160 if test.clear_cache_before_each_run:
161 self.tab.ClearCache(force=True)
163 def ImplicitPageNavigation(self, test=None):
164 """Executes the implicit navigation that occurs for every page iteration.
166 This function will be called once per page before any actions are executed.
169 test.WillNavigateToPage(self.page, self.tab)
170 test.RunNavigateSteps(self.page, self.tab)
171 test.DidNavigateToPage(self.page, self.tab)
173 i = navigate.NavigateAction()
174 i.RunAction(self.page, self.tab, None)
176 def CleanUpPage(self, test):
177 test.CleanUpAfterPage(self.page, self.tab)
178 if self.page.credentials and self._did_login:
179 self.tab.browser.credentials.LoginNoLongerNeeded(
180 self.tab, self.page.credentials)
183 def AddCommandLineArgs(parser):
184 page_filter.PageFilter.AddCommandLineArgs(parser)
185 results_options.AddResultsOptions(parser)
188 group = optparse.OptionGroup(parser, 'Page set ordering and repeat options')
189 group.add_option('--pageset-shuffle', action='store_true',
190 dest='pageset_shuffle',
191 help='Shuffle the order of pages within a pageset.')
192 group.add_option('--pageset-shuffle-order-file',
193 dest='pageset_shuffle_order_file', default=None,
194 help='Filename of an output of a previously run test on the current '
195 'pageset. The tests will run in the same order again, overriding '
196 'what is specified by --page-repeat and --pageset-repeat.')
197 group.add_option('--page-repeat', default=1, type='int',
198 help='Number of times to repeat each individual page '
199 'before proceeding with the next page in the pageset.')
200 group.add_option('--pageset-repeat', default=1, type='int',
201 help='Number of times to repeat the entire pageset.')
202 group.add_option('--max-failures', default=None, type='int',
203 help='Maximum number of test failures before aborting '
204 'the run. Defaults to the number specified by the '
206 parser.add_option_group(group)
209 group = optparse.OptionGroup(parser, 'Web Page Replay options')
210 group.add_option('--use-live-sites',
211 dest='use_live_sites', action='store_true',
212 help='Run against live sites and ignore the Web Page Replay archives.')
213 parser.add_option_group(group)
215 parser.add_option('-d', '--also-run-disabled-tests',
216 dest='run_disabled_tests',
217 action='store_true', default=False,
218 help='Ignore @Disabled and @Enabled restrictions.')
220 def ProcessCommandLineArgs(parser, args):
221 page_filter.PageFilter.ProcessCommandLineArgs(parser, args)
224 if args.pageset_shuffle_order_file and not args.pageset_shuffle:
225 parser.error('--pageset-shuffle-order-file requires --pageset-shuffle.')
227 if args.page_repeat < 1:
228 parser.error('--page-repeat must be a positive integer.')
229 if args.pageset_repeat < 1:
230 parser.error('--pageset-repeat must be a positive integer.')
233 def _PrepareAndRunPage(test, page_set, expectations, finder_options,
234 browser_options, page, credentials_path,
235 possible_browser, results, state):
236 if finder_options.use_live_sites:
237 browser_options.wpr_mode = wpr_modes.WPR_OFF
238 elif browser_options.wpr_mode != wpr_modes.WPR_RECORD:
239 browser_options.wpr_mode = (
241 if page.archive_path and os.path.isfile(page.archive_path)
242 else wpr_modes.WPR_OFF)
244 max_attempts = test.attempts
246 while attempt_num < max_attempts:
249 results.WillAttemptPageRun(attempt_num, max_attempts)
251 if test.RestartBrowserBeforeEachPage() or page.startup_url:
253 # If we are restarting the browser for each page customize the per page
254 # options for just the current page before starting the browser.
255 state.StartBrowserIfNeeded(test, page_set, page, possible_browser,
256 credentials_path, page.archive_path,
258 if not page.CanRunOnBrowser(browser_info.BrowserInfo(state.browser)):
259 logging.info('Skip test for page %s because browser is not supported.'
263 expectation = expectations.GetExpectationForPage(state.browser, page)
265 _WaitForThermalThrottlingIfNeeded(state.browser.platform)
267 if finder_options.profiler:
268 state.StartProfiling(page, finder_options)
271 _RunPage(test, page, state, expectation, results)
272 _CheckThermalThrottling(state.browser.platform)
273 except exceptions.TabCrashException as e:
274 if test.is_multi_tab_test:
275 logging.error('Aborting multi-tab test after tab %s crashed',
278 logging.warning(str(e))
281 if finder_options.profiler:
282 state.StopProfiling()
284 if (test.StopBrowserAfterPage(state.browser, page)):
288 except exceptions.BrowserGoneException as e:
290 if attempt_num == max_attempts:
291 logging.error('Aborting after too many retries')
293 if test.is_multi_tab_test:
294 logging.error('Aborting multi-tab test after browser crashed')
296 logging.warning(str(e))
300 def _UpdateCredentials(page_set):
301 # Attempt to download the credentials file.
302 if page_set.credentials_path:
304 cloud_storage.GetIfChanged(
305 os.path.join(page_set.base_dir, page_set.credentials_path))
306 except (cloud_storage.CredentialsError, cloud_storage.PermissionError,
307 cloud_storage.CloudStorageError) as e:
308 logging.warning('Cannot retrieve credential file %s due to cloud storage '
309 'error %s', page_set.credentials_path, str(e))
313 def _UpdatePageSetArchivesIfChanged(page_set):
314 # Scan every serving directory for .sha1 files
315 # and download them from Cloud Storage. Assume all data is public.
316 all_serving_dirs = page_set.serving_dirs.copy()
317 # Add individual page dirs to all serving dirs.
318 for page in page_set:
320 all_serving_dirs.add(page.serving_dir)
321 # Scan all serving dirs.
322 for serving_dir in all_serving_dirs:
323 if os.path.splitdrive(serving_dir)[1] == '/':
324 raise ValueError('Trying to serve root directory from HTTP server.')
325 for dirpath, _, filenames in os.walk(serving_dir):
326 for filename in filenames:
327 path, extension = os.path.splitext(
328 os.path.join(dirpath, filename))
329 if extension != '.sha1':
331 cloud_storage.GetIfChanged(path, page_set.bucket)
334 def Run(test, page_set, expectations, finder_options, results):
335 """Runs a given test against a given page_set with the given options."""
336 test.ValidatePageSet(page_set)
338 # Create a possible_browser with the given options.
340 possible_browser = browser_finder.FindBrowser(finder_options)
341 except browser_finder.BrowserTypeRequiredException, e:
342 sys.stderr.write(str(e) + '\n')
344 if not possible_browser:
346 'No browser found. Available browsers:\n%s\n' %
347 '\n'.join(browser_finder.GetAllAvailableBrowserTypes(finder_options)))
350 browser_options = possible_browser.finder_options.browser_options
351 browser_options.browser_type = possible_browser.browser_type
352 test.CustomizeBrowserOptions(browser_options)
354 if (not decorators.IsEnabled(test, possible_browser) and
355 not finder_options.run_disabled_tests):
356 logging.warning('You are trying to run a disabled test.')
357 logging.warning('Pass --also-run-disabled-tests to squelch this message.')
360 if possible_browser.IsRemote():
361 possible_browser.RunRemote()
364 # Reorder page set based on options.
365 pages = _ShuffleAndFilterPageSet(page_set, finder_options)
367 if not finder_options.use_live_sites:
368 _UpdateCredentials(page_set)
369 if browser_options.wpr_mode != wpr_modes.WPR_RECORD:
370 _UpdatePageSetArchivesIfChanged(page_set)
371 pages = _CheckArchives(page_set, pages, results)
373 # Verify credentials path.
374 credentials_path = None
375 if page_set.credentials_path:
376 credentials_path = os.path.join(os.path.dirname(page_set.file_path),
377 page_set.credentials_path)
378 if not os.path.exists(credentials_path):
379 credentials_path = None
382 browser_options.browser_user_agent_type = page_set.user_agent_type or None
384 if finder_options.profiler:
385 profiler_class = profiler_finder.FindProfiler(finder_options.profiler)
386 profiler_class.CustomizeBrowserOptions(browser_options.browser_type,
389 for page in list(pages):
390 if not test.CanRunForPage(page):
391 results.WillRunPage(page)
392 logging.debug('Skipping test: it cannot run for %s', page.url)
393 results.AddValue(skip.SkipValue(page, 'Test cannot run'))
394 results.DidRunPage(page)
401 pages_with_discarded_first_result = set()
402 max_failures = finder_options.max_failures # command-line gets priority
403 if max_failures is None:
404 max_failures = test.max_failures # may be None
407 test.WillRunTest(finder_options)
408 for _ in xrange(finder_options.pageset_repeat):
412 for _ in xrange(finder_options.page_repeat):
413 results.WillRunPage(page)
416 test, page_set, expectations, finder_options, browser_options,
417 page, credentials_path, possible_browser, results, state)
419 discard_run = (test.discard_first_result and
420 page not in pages_with_discarded_first_result)
422 pages_with_discarded_first_result.add(page)
423 results.DidRunPage(page, discard_run=discard_run)
424 if max_failures is not None and len(results.failures) > max_failures:
425 logging.error('Too many failures. Aborting.')
428 test.DidRunTest(state.browser, results)
432 def _ShuffleAndFilterPageSet(page_set, finder_options):
433 if finder_options.pageset_shuffle_order_file:
434 return page_set.ReorderPageSet(finder_options.pageset_shuffle_order_file)
435 pages = [page for page in page_set.pages[:]
436 if not page.disabled and page_filter.PageFilter.IsSelected(page)]
437 if finder_options.pageset_shuffle:
438 random.shuffle(pages)
442 def _CheckArchives(page_set, pages, results):
443 """Returns a subset of pages that are local or have WPR archives.
445 Logs warnings if any are missing.
447 # Warn of any problems with the entire page set.
448 if any(not p.is_local for p in pages):
449 if not page_set.archive_data_file:
450 logging.warning('The page set is missing an "archive_data_file" '
451 'property. Skipping any live sites. To include them, '
452 'pass the flag --use-live-sites.')
453 if not page_set.wpr_archive_info:
454 logging.warning('The archive info file is missing. '
455 'To fix this, either add svn-internal to your '
456 '.gclient using http://goto/read-src-internal, '
457 'or create a new archive using record_wpr.')
459 # Warn of any problems with individual pages and return valid pages.
460 pages_missing_archive_path = []
461 pages_missing_archive_data = []
464 if not page.is_local and not page.archive_path:
465 pages_missing_archive_path.append(page)
466 elif not page.is_local and not os.path.isfile(page.archive_path):
467 pages_missing_archive_data.append(page)
469 valid_pages.append(page)
470 if pages_missing_archive_path:
471 logging.warning('The page set archives for some pages do not exist. '
472 'Skipping those pages. To fix this, record those pages '
473 'using record_wpr. To ignore this warning and run '
474 'against live sites, pass the flag --use-live-sites.')
475 if pages_missing_archive_data:
476 logging.warning('The page set archives for some pages are missing. '
477 'Someone forgot to check them in, or they were deleted. '
478 'Skipping those pages. To fix this, record those pages '
479 'using record_wpr. To ignore this warning and run '
480 'against live sites, pass the flag --use-live-sites.')
481 for page in pages_missing_archive_path + pages_missing_archive_data:
482 results.WillRunPage(page)
483 results.AddValue(failure.FailureValue.FromMessage(
484 page, 'Page set archive doesn\'t exist.'))
485 results.DidRunPage(page)
489 def _RunPage(test, page, state, expectation, results):
490 if expectation == 'skip':
491 logging.debug('Skipping test: Skip expectation for %s', page.url)
492 results.AddValue(skip.SkipValue(page, 'Skipped by test expectations'))
495 page_state = PageState(page, test.TabForPage(page, state.browser))
498 if expectation == 'fail':
499 msg = 'Expected exception while running %s' % page.url
501 msg = 'Exception while running %s' % page.url
502 results.AddValue(failure.FailureValue(page, sys.exc_info()))
503 exception_formatter.PrintFormattedException(msg=msg)
506 page_state.PreparePage(test)
507 page_state.ImplicitPageNavigation(test)
508 test.RunPage(page, page_state.tab, results)
509 util.CloseConnections(page_state.tab)
510 except page_test.TestNotSupportedOnPlatformFailure:
512 except page_test.Failure:
513 if expectation == 'fail':
514 exception_formatter.PrintFormattedException(
515 msg='Expected failure while running %s' % page.url)
517 exception_formatter.PrintFormattedException(
518 msg='Failure while running %s' % page.url)
519 results.AddValue(failure.FailureValue(page, sys.exc_info()))
520 except (util.TimeoutException, exceptions.LoginException,
521 exceptions.ProfilingException):
523 except (exceptions.TabCrashException, exceptions.BrowserGoneException):
525 # Run() catches these exceptions to relaunch the tab/browser, so re-raise.
527 except page_action.PageActionNotSupported as e:
528 results.AddValue(skip.SkipValue(page, 'Unsupported page action: %s' % e))
530 exception_formatter.PrintFormattedException(
531 msg='Unhandled exception while running %s' % page.url)
532 results.AddValue(failure.FailureValue(page, sys.exc_info()))
534 if expectation == 'fail':
535 logging.warning('%s was expected to fail, but passed.\n', page.url)
537 page_state.CleanUpPage(test)
540 def _WaitForThermalThrottlingIfNeeded(platform):
541 if not platform.CanMonitorThermalThrottling():
543 thermal_throttling_retry = 0
544 while (platform.IsThermallyThrottled() and
545 thermal_throttling_retry < 3):
546 logging.warning('Thermally throttled, waiting (%d)...',
547 thermal_throttling_retry)
548 thermal_throttling_retry += 1
549 time.sleep(thermal_throttling_retry * 2)
551 if thermal_throttling_retry and platform.IsThermallyThrottled():
552 logging.warning('Device is thermally throttled before running '
553 'performance tests, results will vary.')
556 def _CheckThermalThrottling(platform):
557 if not platform.CanMonitorThermalThrottling():
559 if platform.HasBeenThermallyThrottled():
560 logging.warning('Device has been thermally throttled during '
561 'performance tests, results will vary.')