src/tools/telemetry/telemetry/page/page_runner.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6 import optparse
   7 import os
   8 import random
   9 import sys
  10 import tempfile
  11 import time
  12
  13 from telemetry import decorators
  14 from telemetry.core import browser_finder
  15 from telemetry.core import browser_info
  16 from telemetry.core import exceptions
  17 from telemetry.core import util
  18 from telemetry.core import wpr_modes
  19 from telemetry.core.platform.profiler import profiler_finder
  20 from telemetry.page import page_filter
  21 from telemetry.page import page_test
  22 from telemetry.page.actions import navigate
  23 from telemetry.page.actions import page_action
  24 from telemetry.results import results_options
  25 from telemetry.util import cloud_storage
  26 from telemetry.util import exception_formatter
  27 from telemetry.value import failure
  28 from telemetry.value import skip
  29
  30
  31 class _RunState(object):
  32   def __init__(self):
  33     self.browser = None
  34
  35     self._append_to_existing_wpr = False
  36     self._last_archive_path = None
  37     self._first_browser = True
  38     self.profiler_dir = None
  39
  40   def StartBrowserIfNeeded(self, test, page_set, page, possible_browser,
  41                            credentials_path, archive_path, finder_options):
  42     started_browser = not self.browser
  43     # Create a browser.
  44     if not self.browser:
  45       test.CustomizeBrowserOptionsForSinglePage(page, finder_options)
  46       possible_browser.SetReplayArchivePath(archive_path,
  47                                         self._append_to_existing_wpr,
  48                                         page_set.make_javascript_deterministic)
  49       possible_browser.SetCredentialsPath(credentials_path)
  50       self._last_archive_path = page.archive_path
  51
  52       test.WillStartBrowser(possible_browser.platform)
  53       self.browser = possible_browser.Create()
  54       test.DidStartBrowser(self.browser)
  55
  56       if self._first_browser:
  57         self._first_browser = False
  58         self.browser.credentials.WarnIfMissingCredentials(page_set)
  59         logging.info('OS: %s %s',
  60                      self.browser.platform.GetOSName(),
  61                      self.browser.platform.GetOSVersionName())
  62         if self.browser.supports_system_info:
  63           system_info = self.browser.GetSystemInfo()
  64           if system_info.model_name:
  65             logging.info('Model: %s', system_info.model_name)
  66           if system_info.gpu:
  67             for i, device in enumerate(system_info.gpu.devices):
  68               logging.info('GPU device %d: %s', i, device)
  69             if system_info.gpu.aux_attributes:
  70               logging.info('GPU Attributes:')
  71               for k, v in sorted(system_info.gpu.aux_attributes.iteritems()):
  72                 logging.info('  %-20s: %s', k, v)
  73             if system_info.gpu.feature_status:
  74               logging.info('Feature Status:')
  75               for k, v in sorted(system_info.gpu.feature_status.iteritems()):
  76                 logging.info('  %-20s: %s', k, v)
  77             if system_info.gpu.driver_bug_workarounds:
  78               logging.info('Driver Bug Workarounds:')
  79               for workaround in system_info.gpu.driver_bug_workarounds:
  80                 logging.info('  %s', workaround)
  81           else:
  82             logging.info('No GPU devices')
  83         else:
  84           logging.warning('System info not supported')
  85     else:
  86       # Set up WPR path if it changed.
  87       if page.archive_path and self._last_archive_path != page.archive_path:
  88         self.browser.SetReplayArchivePath(
  89             page.archive_path,
  90             self._append_to_existing_wpr,
  91             page_set.make_javascript_deterministic)
  92         self._last_archive_path = page.archive_path
  93
  94     if self.browser.supports_tab_control and test.close_tabs_before_run:
  95       # Create a tab if there's none.
  96       if len(self.browser.tabs) == 0:
  97         self.browser.tabs.New()
  98
  99       # Ensure only one tab is open, unless the test is a multi-tab test.
 100       if not test.is_multi_tab_test:
 101         while len(self.browser.tabs) > 1:
 102           self.browser.tabs[-1].Close()
 103
 104       # Must wait for tab to commit otherwise it can commit after the next
 105       # navigation has begun and RenderFrameHostManager::DidNavigateMainFrame()
 106       # will cancel the next navigation because it's pending. This manifests as
 107       # the first navigation in a PageSet freezing indefinitly because the
 108       # navigation was silently cancelled when |self.browser.tabs[0]| was
 109       # committed. Only do this when we just started the browser, otherwise
 110       # there are cases where previous pages in a PageSet never complete
 111       # loading so we'll wait forever.
 112       if started_browser:
 113         self.browser.tabs[0].WaitForDocumentReadyStateToBeComplete()
 114
 115   def StopBrowser(self):
 116     if self.browser:
 117       self.browser.Close()
 118       self.browser = None
 119
 120       # Restarting the state will also restart the wpr server. If we're
 121       # recording, we need to continue adding into the same wpr archive,
 122       # not overwrite it.
 123       self._append_to_existing_wpr = True
 124
 125   def StartProfiling(self, page, finder_options):
 126     if not self.profiler_dir:
 127       self.profiler_dir = tempfile.mkdtemp()
 128     output_file = os.path.join(self.profiler_dir, page.file_safe_name)
 129     is_repeating = (finder_options.page_repeat != 1 or
 130                     finder_options.pageset_repeat != 1)
 131     if is_repeating:
 132       output_file = util.GetSequentialFileName(output_file)
 133     self.browser.platform.profiling_controller.Start(
 134         finder_options.profiler, output_file)
 135
 136   def StopProfiling(self):
 137     if self.browser:
 138       self.browser.platform.profiling_controller.Stop()
 139
 140
 141 class PageState(object):
 142   def __init__(self, page, tab):
 143     self.page = page
 144     self.tab = tab
 145
 146     self._did_login = False
 147
 148   def PreparePage(self, test=None):
 149     if self.page.is_file:
 150       self.tab.browser.SetHTTPServerDirectories(
 151           self.page.page_set.serving_dirs | set([self.page.serving_dir]))
 152
 153     if self.page.credentials:
 154       if not self.tab.browser.credentials.LoginNeeded(
 155           self.tab, self.page.credentials):
 156         raise page_test.Failure('Login as ' + self.page.credentials + ' failed')
 157       self._did_login = True
 158
 159     if test:
 160       if test.clear_cache_before_each_run:
 161         self.tab.ClearCache(force=True)
 162
 163   def ImplicitPageNavigation(self, test=None):
 164     """Executes the implicit navigation that occurs for every page iteration.
 165
 166     This function will be called once per page before any actions are executed.
 167     """
 168     if test:
 169       test.WillNavigateToPage(self.page, self.tab)
 170       test.RunNavigateSteps(self.page, self.tab)
 171       test.DidNavigateToPage(self.page, self.tab)
 172     else:
 173       i = navigate.NavigateAction()
 174       i.RunAction(self.page, self.tab, None)
 175
 176   def CleanUpPage(self, test):
 177     test.CleanUpAfterPage(self.page, self.tab)
 178     if self.page.credentials and self._did_login:
 179       self.tab.browser.credentials.LoginNoLongerNeeded(
 180           self.tab, self.page.credentials)
 181
 182
 183 def AddCommandLineArgs(parser):
 184   page_filter.PageFilter.AddCommandLineArgs(parser)
 185   results_options.AddResultsOptions(parser)
 186
 187   # Page set options
 188   group = optparse.OptionGroup(parser, 'Page set ordering and repeat options')
 189   group.add_option('--pageset-shuffle', action='store_true',
 190       dest='pageset_shuffle',
 191       help='Shuffle the order of pages within a pageset.')
 192   group.add_option('--pageset-shuffle-order-file',
 193       dest='pageset_shuffle_order_file', default=None,
 194       help='Filename of an output of a previously run test on the current '
 195       'pageset. The tests will run in the same order again, overriding '
 196       'what is specified by --page-repeat and --pageset-repeat.')
 197   group.add_option('--page-repeat', default=1, type='int',
 198                    help='Number of times to repeat each individual page '
 199                    'before proceeding with the next page in the pageset.')
 200   group.add_option('--pageset-repeat', default=1, type='int',
 201                    help='Number of times to repeat the entire pageset.')
 202   group.add_option('--max-failures', default=None, type='int',
 203                    help='Maximum number of test failures before aborting '
 204                    'the run. Defaults to the number specified by the '
 205                    'PageTest.')
 206   parser.add_option_group(group)
 207
 208   # WPR options
 209   group = optparse.OptionGroup(parser, 'Web Page Replay options')
 210   group.add_option('--use-live-sites',
 211       dest='use_live_sites', action='store_true',
 212       help='Run against live sites and ignore the Web Page Replay archives.')
 213   parser.add_option_group(group)
 214
 215   parser.add_option('-d', '--also-run-disabled-tests',
 216                     dest='run_disabled_tests',
 217                     action='store_true', default=False,
 218                     help='Ignore @Disabled and @Enabled restrictions.')
 219
 220 def ProcessCommandLineArgs(parser, args):
 221   page_filter.PageFilter.ProcessCommandLineArgs(parser, args)
 222
 223   # Page set options
 224   if args.pageset_shuffle_order_file and not args.pageset_shuffle:
 225     parser.error('--pageset-shuffle-order-file requires --pageset-shuffle.')
 226
 227   if args.page_repeat < 1:
 228     parser.error('--page-repeat must be a positive integer.')
 229   if args.pageset_repeat < 1:
 230     parser.error('--pageset-repeat must be a positive integer.')
 231
 232
 233 def _PrepareAndRunPage(test, page_set, expectations, finder_options,
 234                        browser_options, page, credentials_path,
 235                        possible_browser, results, state):
 236   if finder_options.use_live_sites:
 237     browser_options.wpr_mode = wpr_modes.WPR_OFF
 238   elif browser_options.wpr_mode != wpr_modes.WPR_RECORD:
 239     browser_options.wpr_mode = (
 240         wpr_modes.WPR_REPLAY
 241         if page.archive_path and os.path.isfile(page.archive_path)
 242         else wpr_modes.WPR_OFF)
 243
 244   max_attempts = test.attempts
 245   attempt_num = 0
 246   while attempt_num < max_attempts:
 247     attempt_num += 1
 248     try:
 249       results.WillAttemptPageRun(attempt_num, max_attempts)
 250
 251       if test.RestartBrowserBeforeEachPage() or page.startup_url:
 252         state.StopBrowser()
 253         # If we are restarting the browser for each page customize the per page
 254         # options for just the current page before starting the browser.
 255       state.StartBrowserIfNeeded(test, page_set, page, possible_browser,
 256                                  credentials_path, page.archive_path,
 257                                  finder_options)
 258       if not page.CanRunOnBrowser(browser_info.BrowserInfo(state.browser)):
 259         logging.info('Skip test for page %s because browser is not supported.'
 260                      % page.url)
 261         return
 262
 263       expectation = expectations.GetExpectationForPage(state.browser, page)
 264
 265       _WaitForThermalThrottlingIfNeeded(state.browser.platform)
 266
 267       if finder_options.profiler:
 268         state.StartProfiling(page, finder_options)
 269
 270       try:
 271         _RunPage(test, page, state, expectation, results)
 272         _CheckThermalThrottling(state.browser.platform)
 273       except exceptions.TabCrashException as e:
 274         if test.is_multi_tab_test:
 275           logging.error('Aborting multi-tab test after tab %s crashed',
 276                         page.url)
 277           raise
 278         logging.warning(str(e))
 279         state.StopBrowser()
 280
 281       if finder_options.profiler:
 282         state.StopProfiling()
 283
 284       if (test.StopBrowserAfterPage(state.browser, page)):
 285         state.StopBrowser()
 286
 287       return
 288     except exceptions.BrowserGoneException as e:
 289       state.StopBrowser()
 290       if attempt_num == max_attempts:
 291         logging.error('Aborting after too many retries')
 292         raise
 293       if test.is_multi_tab_test:
 294         logging.error('Aborting multi-tab test after browser crashed')
 295         raise
 296       logging.warning(str(e))
 297
 298
 299 @decorators.Cache
 300 def _UpdateCredentials(page_set):
 301   # Attempt to download the credentials file.
 302   if page_set.credentials_path:
 303     try:
 304       cloud_storage.GetIfChanged(
 305           os.path.join(page_set.base_dir, page_set.credentials_path))
 306     except (cloud_storage.CredentialsError, cloud_storage.PermissionError,
 307             cloud_storage.CloudStorageError) as e:
 308       logging.warning('Cannot retrieve credential file %s due to cloud storage '
 309                       'error %s', page_set.credentials_path, str(e))
 310
 311
 312 @decorators.Cache
 313 def _UpdatePageSetArchivesIfChanged(page_set):
 314   # Scan every serving directory for .sha1 files
 315   # and download them from Cloud Storage. Assume all data is public.
 316   all_serving_dirs = page_set.serving_dirs.copy()
 317   # Add individual page dirs to all serving dirs.
 318   for page in page_set:
 319     if page.is_file:
 320       all_serving_dirs.add(page.serving_dir)
 321   # Scan all serving dirs.
 322   for serving_dir in all_serving_dirs:
 323     if os.path.splitdrive(serving_dir)[1] == '/':
 324       raise ValueError('Trying to serve root directory from HTTP server.')
 325     for dirpath, _, filenames in os.walk(serving_dir):
 326       for filename in filenames:
 327         path, extension = os.path.splitext(
 328             os.path.join(dirpath, filename))
 329         if extension != '.sha1':
 330           continue
 331         cloud_storage.GetIfChanged(path, page_set.bucket)
 332
 333
 334 def Run(test, page_set, expectations, finder_options, results):
 335   """Runs a given test against a given page_set with the given options."""
 336   test.ValidatePageSet(page_set)
 337
 338   # Create a possible_browser with the given options.
 339   try:
 340     possible_browser = browser_finder.FindBrowser(finder_options)
 341   except browser_finder.BrowserTypeRequiredException, e:
 342     sys.stderr.write(str(e) + '\n')
 343     sys.exit(-1)
 344   if not possible_browser:
 345     sys.stderr.write(
 346         'No browser found. Available browsers:\n%s\n' %
 347         '\n'.join(browser_finder.GetAllAvailableBrowserTypes(finder_options)))
 348     sys.exit(-1)
 349
 350   browser_options = possible_browser.finder_options.browser_options
 351   browser_options.browser_type = possible_browser.browser_type
 352   test.CustomizeBrowserOptions(browser_options)
 353
 354   if (not decorators.IsEnabled(test, possible_browser) and
 355       not finder_options.run_disabled_tests):
 356     logging.warning('You are trying to run a disabled test.')
 357     logging.warning('Pass --also-run-disabled-tests to squelch this message.')
 358     return
 359
 360   if possible_browser.IsRemote():
 361     possible_browser.RunRemote()
 362     sys.exit(0)
 363
 364   # Reorder page set based on options.
 365   pages = _ShuffleAndFilterPageSet(page_set, finder_options)
 366
 367   if not finder_options.use_live_sites:
 368     _UpdateCredentials(page_set)
 369     if browser_options.wpr_mode != wpr_modes.WPR_RECORD:
 370       _UpdatePageSetArchivesIfChanged(page_set)
 371       pages = _CheckArchives(page_set, pages, results)
 372
 373   # Verify credentials path.
 374   credentials_path = None
 375   if page_set.credentials_path:
 376     credentials_path = os.path.join(os.path.dirname(page_set.file_path),
 377                                     page_set.credentials_path)
 378     if not os.path.exists(credentials_path):
 379       credentials_path = None
 380
 381   # Set up user agent.
 382   browser_options.browser_user_agent_type = page_set.user_agent_type or None
 383
 384   if finder_options.profiler:
 385     profiler_class = profiler_finder.FindProfiler(finder_options.profiler)
 386     profiler_class.CustomizeBrowserOptions(browser_options.browser_type,
 387                                            finder_options)
 388
 389   for page in list(pages):
 390     if not test.CanRunForPage(page):
 391       results.WillRunPage(page)
 392       logging.debug('Skipping test: it cannot run for %s', page.url)
 393       results.AddValue(skip.SkipValue(page, 'Test cannot run'))
 394       results.DidRunPage(page)
 395       pages.remove(page)
 396
 397   if not pages:
 398     return
 399
 400   state = _RunState()
 401   pages_with_discarded_first_result = set()
 402   max_failures = finder_options.max_failures  # command-line gets priority
 403   if max_failures is None:
 404     max_failures = test.max_failures  # may be None
 405
 406   try:
 407     test.WillRunTest(finder_options)
 408     for _ in xrange(finder_options.pageset_repeat):
 409       for page in pages:
 410         if test.IsExiting():
 411           break
 412         for _ in xrange(finder_options.page_repeat):
 413           results.WillRunPage(page)
 414           try:
 415             _PrepareAndRunPage(
 416                 test, page_set, expectations, finder_options, browser_options,
 417                 page, credentials_path, possible_browser, results, state)
 418           finally:
 419             discard_run = (test.discard_first_result and
 420                            page not in pages_with_discarded_first_result)
 421             if discard_run:
 422               pages_with_discarded_first_result.add(page)
 423             results.DidRunPage(page, discard_run=discard_run)
 424         if max_failures is not None and len(results.failures) > max_failures:
 425           logging.error('Too many failures. Aborting.')
 426           test.RequestExit()
 427   finally:
 428     test.DidRunTest(state.browser, results)
 429     state.StopBrowser()
 430
 431
 432 def _ShuffleAndFilterPageSet(page_set, finder_options):
 433   if finder_options.pageset_shuffle_order_file:
 434     return page_set.ReorderPageSet(finder_options.pageset_shuffle_order_file)
 435   pages = [page for page in page_set.pages[:]
 436            if not page.disabled and page_filter.PageFilter.IsSelected(page)]
 437   if finder_options.pageset_shuffle:
 438     random.shuffle(pages)
 439   return pages
 440
 441
 442 def _CheckArchives(page_set, pages, results):
 443   """Returns a subset of pages that are local or have WPR archives.
 444
 445   Logs warnings if any are missing.
 446   """
 447   # Warn of any problems with the entire page set.
 448   if any(not p.is_local for p in pages):
 449     if not page_set.archive_data_file:
 450       logging.warning('The page set is missing an "archive_data_file" '
 451                       'property. Skipping any live sites. To include them, '
 452                       'pass the flag --use-live-sites.')
 453     if not page_set.wpr_archive_info:
 454       logging.warning('The archive info file is missing. '
 455                       'To fix this, either add svn-internal to your '
 456                       '.gclient using http://goto/read-src-internal, '
 457                       'or create a new archive using record_wpr.')
 458
 459   # Warn of any problems with individual pages and return valid pages.
 460   pages_missing_archive_path = []
 461   pages_missing_archive_data = []
 462   valid_pages = []
 463   for page in pages:
 464     if not page.is_local and not page.archive_path:
 465       pages_missing_archive_path.append(page)
 466     elif not page.is_local and not os.path.isfile(page.archive_path):
 467       pages_missing_archive_data.append(page)
 468     else:
 469       valid_pages.append(page)
 470   if pages_missing_archive_path:
 471     logging.warning('The page set archives for some pages do not exist. '
 472                     'Skipping those pages. To fix this, record those pages '
 473                     'using record_wpr. To ignore this warning and run '
 474                     'against live sites, pass the flag --use-live-sites.')
 475   if pages_missing_archive_data:
 476     logging.warning('The page set archives for some pages are missing. '
 477                     'Someone forgot to check them in, or they were deleted. '
 478                     'Skipping those pages. To fix this, record those pages '
 479                     'using record_wpr. To ignore this warning and run '
 480                     'against live sites, pass the flag --use-live-sites.')
 481   for page in pages_missing_archive_path + pages_missing_archive_data:
 482     results.WillRunPage(page)
 483     results.AddValue(failure.FailureValue.FromMessage(
 484         page, 'Page set archive doesn\'t exist.'))
 485     results.DidRunPage(page)
 486   return valid_pages
 487
 488
 489 def _RunPage(test, page, state, expectation, results):
 490   if expectation == 'skip':
 491     logging.debug('Skipping test: Skip expectation for %s', page.url)
 492     results.AddValue(skip.SkipValue(page, 'Skipped by test expectations'))
 493     return
 494
 495   page_state = PageState(page, test.TabForPage(page, state.browser))
 496
 497   def ProcessError():
 498     if expectation == 'fail':
 499       msg = 'Expected exception while running %s' % page.url
 500     else:
 501       msg = 'Exception while running %s' % page.url
 502       results.AddValue(failure.FailureValue(page, sys.exc_info()))
 503     exception_formatter.PrintFormattedException(msg=msg)
 504
 505   try:
 506     page_state.PreparePage(test)
 507     page_state.ImplicitPageNavigation(test)
 508     test.RunPage(page, page_state.tab, results)
 509     util.CloseConnections(page_state.tab)
 510   except page_test.TestNotSupportedOnPlatformFailure:
 511     raise
 512   except page_test.Failure:
 513     if expectation == 'fail':
 514       exception_formatter.PrintFormattedException(
 515           msg='Expected failure while running %s' % page.url)
 516     else:
 517       exception_formatter.PrintFormattedException(
 518           msg='Failure while running %s' % page.url)
 519       results.AddValue(failure.FailureValue(page, sys.exc_info()))
 520   except (util.TimeoutException, exceptions.LoginException,
 521           exceptions.ProfilingException):
 522     ProcessError()
 523   except (exceptions.TabCrashException, exceptions.BrowserGoneException):
 524     ProcessError()
 525     # Run() catches these exceptions to relaunch the tab/browser, so re-raise.
 526     raise
 527   except page_action.PageActionNotSupported as e:
 528     results.AddValue(skip.SkipValue(page, 'Unsupported page action: %s' % e))
 529   except Exception:
 530     exception_formatter.PrintFormattedException(
 531         msg='Unhandled exception while running %s' % page.url)
 532     results.AddValue(failure.FailureValue(page, sys.exc_info()))
 533   else:
 534     if expectation == 'fail':
 535       logging.warning('%s was expected to fail, but passed.\n', page.url)
 536   finally:
 537     page_state.CleanUpPage(test)
 538
 539
 540 def _WaitForThermalThrottlingIfNeeded(platform):
 541   if not platform.CanMonitorThermalThrottling():
 542     return
 543   thermal_throttling_retry = 0
 544   while (platform.IsThermallyThrottled() and
 545          thermal_throttling_retry < 3):
 546     logging.warning('Thermally throttled, waiting (%d)...',
 547                     thermal_throttling_retry)
 548     thermal_throttling_retry += 1
 549     time.sleep(thermal_throttling_retry * 2)
 550
 551   if thermal_throttling_retry and platform.IsThermallyThrottled():
 552     logging.warning('Device is thermally throttled before running '
 553                     'performance tests, results will vary.')
 554
 555
 556 def _CheckThermalThrottling(platform):
 557   if not platform.CanMonitorThermalThrottling():
 558     return
 559   if platform.HasBeenThermallyThrottled():
 560     logging.warning('Device has been thermally throttled during '
 561                     'performance tests, results will vary.')