src/tools/telemetry/telemetry/page/page_runner_unittest.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6 import os
   7 import tempfile
   8 import unittest
   9
  10 from telemetry import benchmark
  11 from telemetry import decorators
  12 from telemetry.core import browser_finder
  13 from telemetry.core import exceptions
  14 from telemetry.core import user_agent
  15 from telemetry.core import util
  16 from telemetry.page import page as page_module
  17 from telemetry.page import page_runner
  18 from telemetry.page import page_set
  19 from telemetry.page import page_test
  20 from telemetry.page import test_expectations
  21 from telemetry.results import results_options
  22 from telemetry.unittest import options_for_unittests
  23 from telemetry.value import scalar
  24 from telemetry.value import string
  25
  26
  27 SIMPLE_CREDENTIALS_STRING = """
  28 {
  29   "test": {
  30     "username": "example",
  31     "password": "asdf"
  32   }
  33 }
  34 """
  35
  36
  37 def SetUpPageRunnerArguments(options):
  38   parser = options.CreateParser()
  39   page_runner.AddCommandLineArgs(parser)
  40   options.MergeDefaultValues(parser.get_default_values())
  41   page_runner.ProcessCommandLineArgs(parser, options)
  42
  43 class EmptyMetadataForTest(benchmark.BenchmarkMetadata):
  44   def __init__(self):
  45     super(EmptyMetadataForTest, self).__init__('')
  46
  47 class StubCredentialsBackend(object):
  48   def __init__(self, login_return_value):
  49     self.did_get_login = False
  50     self.did_get_login_no_longer_needed = False
  51     self.login_return_value = login_return_value
  52
  53   @property
  54   def credentials_type(self):
  55     return 'test'
  56
  57   def LoginNeeded(self, *_):
  58     self.did_get_login = True
  59     return self.login_return_value
  60
  61   def LoginNoLongerNeeded(self, _):
  62     self.did_get_login_no_longer_needed = True
  63
  64
  65 def GetSuccessfulPageRuns(results):
  66   return [run for run in results.all_page_runs if run.ok or run.skipped]
  67
  68
  69 class PageRunnerTests(unittest.TestCase):
  70   # TODO(nduca): Move the basic "test failed, test succeeded" tests from
  71   # page_test_unittest to here.
  72
  73   def testHandlingOfCrashedTab(self):
  74     ps = page_set.PageSet()
  75     expectations = test_expectations.TestExpectations()
  76     page1 = page_module.Page('chrome://crash', ps)
  77     ps.pages.append(page1)
  78
  79     class Test(page_test.PageTest):
  80       def ValidatePage(self, *args):
  81         pass
  82
  83     options = options_for_unittests.GetCopy()
  84     options.output_format = 'none'
  85     options.suppress_gtest_report = True
  86     SetUpPageRunnerArguments(options)
  87     results = results_options.CreateResults(EmptyMetadataForTest(), options)
  88     page_runner.Run(Test(), ps, expectations, options, results)
  89     self.assertEquals(0, len(GetSuccessfulPageRuns(results)))
  90     self.assertEquals(1, len(results.failures))
  91
  92   def testHandlingOfTestThatRaisesWithNonFatalUnknownExceptions(self):
  93     ps = page_set.PageSet()
  94     expectations = test_expectations.TestExpectations()
  95     ps.pages.append(page_module.Page(
  96         'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
  97     ps.pages.append(page_module.Page(
  98         'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
  99
 100     class ExpectedException(Exception):
 101       pass
 102
 103     class Test(page_test.PageTest):
 104       def __init__(self, *args):
 105         super(Test, self).__init__(*args)
 106         self.run_count = 0
 107       def ValidatePage(self, *_):
 108         old_run_count = self.run_count
 109         self.run_count += 1
 110         if old_run_count == 0:
 111           raise ExpectedException()
 112
 113     options = options_for_unittests.GetCopy()
 114     options.output_format = 'none'
 115     options.suppress_gtest_report = True
 116     test = Test()
 117     SetUpPageRunnerArguments(options)
 118     results = results_options.CreateResults(EmptyMetadataForTest(), options)
 119     page_runner.Run(test, ps, expectations, options, results)
 120     self.assertEquals(2, test.run_count)
 121     self.assertEquals(1, len(GetSuccessfulPageRuns(results)))
 122     self.assertEquals(1, len(results.failures))
 123
 124   def testHandlingOfCrashedTabWithExpectedFailure(self):
 125     ps = page_set.PageSet()
 126     expectations = test_expectations.TestExpectations()
 127     expectations.Fail('chrome://crash')
 128     page1 = page_module.Page('chrome://crash', ps)
 129     ps.pages.append(page1)
 130
 131     class Test(page_test.PageTest):
 132       def ValidatePage(self, *_):
 133         pass
 134
 135     options = options_for_unittests.GetCopy()
 136     options.output_format = 'none'
 137     options.suppress_gtest_report = True
 138     SetUpPageRunnerArguments(options)
 139     results = results_options.CreateResults(EmptyMetadataForTest(), options)
 140     page_runner.Run(Test(), ps, expectations, options, results)
 141     self.assertEquals(1, len(GetSuccessfulPageRuns(results)))
 142     self.assertEquals(0, len(results.failures))
 143
 144   def testRetryOnBrowserCrash(self):
 145     ps = page_set.PageSet()
 146     expectations = test_expectations.TestExpectations()
 147     ps.pages.append(page_module.Page(
 148         'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
 149
 150     class CrashyMeasurement(page_test.PageTest):
 151       has_crashed = False
 152       def ValidateAndMeasurePage(self, page, tab, results):
 153         # This value should be discarded on the first run when the
 154         # browser crashed.
 155         results.AddValue(
 156             string.StringValue(page, 'test', 't', str(self.has_crashed)))
 157         if not self.has_crashed:
 158           self.has_crashed = True
 159           raise exceptions.BrowserGoneException(tab.browser)
 160
 161     options = options_for_unittests.GetCopy()
 162     options.output_format = 'csv'
 163     options.suppress_gtest_report = True
 164
 165     SetUpPageRunnerArguments(options)
 166     results = results_options.CreateResults(EmptyMetadataForTest(), options)
 167     page_runner.Run(CrashyMeasurement(), ps, expectations, options, results)
 168
 169     self.assertEquals(1, len(GetSuccessfulPageRuns(results)))
 170     self.assertEquals(0, len(results.failures))
 171     self.assertEquals(1, len(results.all_page_specific_values))
 172     self.assertEquals(
 173         'True', results.all_page_specific_values[0].GetRepresentativeString())
 174
 175   @decorators.Disabled('xp')  # Flaky, http://crbug.com/390079.
 176   def testDiscardFirstResult(self):
 177     ps = page_set.PageSet()
 178     expectations = test_expectations.TestExpectations()
 179     ps.pages.append(page_module.Page(
 180         'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
 181     ps.pages.append(page_module.Page(
 182         'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
 183
 184     class Measurement(page_test.PageTest):
 185       @property
 186       def discard_first_result(self):
 187         return True
 188
 189       def ValidateAndMeasurePage(self, page, _, results):
 190         results.AddValue(string.StringValue(page, 'test', 't', page.url))
 191
 192     options = options_for_unittests.GetCopy()
 193     options.output_format = 'none'
 194     options.suppress_gtest_report = True
 195     options.reset_results = None
 196     options.upload_results = None
 197     options.results_label = None
 198
 199     options.page_repeat = 1
 200     options.pageset_repeat = 1
 201     SetUpPageRunnerArguments(options)
 202     results = results_options.CreateResults(EmptyMetadataForTest(), options)
 203     page_runner.Run(Measurement(), ps, expectations, options, results)
 204     self.assertEquals(0, len(GetSuccessfulPageRuns(results)))
 205     self.assertEquals(0, len(results.failures))
 206     self.assertEquals(0, len(results.all_page_specific_values))
 207
 208     options.page_repeat = 1
 209     options.pageset_repeat = 2
 210     SetUpPageRunnerArguments(options)
 211     results = results_options.CreateResults(EmptyMetadataForTest(), options)
 212     page_runner.Run(Measurement(), ps, expectations, options, results)
 213     self.assertEquals(2, len(GetSuccessfulPageRuns(results)))
 214     self.assertEquals(0, len(results.failures))
 215     self.assertEquals(2, len(results.all_page_specific_values))
 216
 217     options.page_repeat = 2
 218     options.pageset_repeat = 1
 219     SetUpPageRunnerArguments(options)
 220     results = results_options.CreateResults(EmptyMetadataForTest(), options)
 221     page_runner.Run(Measurement(), ps, expectations, options, results)
 222     self.assertEquals(2, len(GetSuccessfulPageRuns(results)))
 223     self.assertEquals(0, len(results.failures))
 224     self.assertEquals(2, len(results.all_page_specific_values))
 225
 226     options.output_format = 'html'
 227     options.suppress_gtest_report = True
 228     options.page_repeat = 1
 229     options.pageset_repeat = 1
 230     SetUpPageRunnerArguments(options)
 231     results = results_options.CreateResults(EmptyMetadataForTest(), options)
 232     page_runner.Run(Measurement(), ps, expectations, options, results)
 233     self.assertEquals(0, len(GetSuccessfulPageRuns(results)))
 234     self.assertEquals(0, len(results.failures))
 235     self.assertEquals(0, len(results.all_page_specific_values))
 236
 237   @decorators.Disabled('win')
 238   def testPagesetRepeat(self):
 239     ps = page_set.PageSet()
 240     expectations = test_expectations.TestExpectations()
 241     ps.pages.append(page_module.Page(
 242         'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
 243     ps.pages.append(page_module.Page(
 244         'file://green_rect.html', ps, base_dir=util.GetUnittestDataDir()))
 245
 246     class Measurement(page_test.PageTest):
 247       i = 0
 248       def ValidateAndMeasurePage(self, page, _, results):
 249         self.i += 1
 250         results.AddValue(scalar.ScalarValue(
 251             page, 'metric', 'unit', self.i))
 252
 253     output_file = tempfile.NamedTemporaryFile(delete=False).name
 254     try:
 255       options = options_for_unittests.GetCopy()
 256       options.output_format = 'buildbot'
 257       options.output_file = output_file
 258       options.suppress_gtest_report = True
 259       options.reset_results = None
 260       options.upload_results = None
 261       options.results_label = None
 262
 263       options.page_repeat = 1
 264       options.pageset_repeat = 2
 265       SetUpPageRunnerArguments(options)
 266       results = results_options.CreateResults(EmptyMetadataForTest(), options)
 267       page_runner.Run(Measurement(), ps, expectations, options, results)
 268       results.PrintSummary()
 269       self.assertEquals(4, len(GetSuccessfulPageRuns(results)))
 270       self.assertEquals(0, len(results.failures))
 271       with open(output_file) as f:
 272         stdout = f.read()
 273       self.assertIn('RESULT metric: blank.html= [1,3] unit', stdout)
 274       self.assertIn('RESULT metric: green_rect.html= [2,4] unit', stdout)
 275       self.assertIn('*RESULT metric: metric= [1,2,3,4] unit', stdout)
 276     finally:
 277       # TODO(chrishenry): This is a HACK!!1 Really, the right way to
 278       # do this is for page_runner (or output formatter) to close any
 279       # files it has opened.
 280       for formatter in results._output_formatters:  # pylint: disable=W0212
 281         formatter.output_stream.close()
 282       os.remove(output_file)
 283
 284   def testCredentialsWhenLoginFails(self):
 285     credentials_backend = StubCredentialsBackend(login_return_value=False)
 286     did_run = self.runCredentialsTest(credentials_backend)
 287     assert credentials_backend.did_get_login == True
 288     assert credentials_backend.did_get_login_no_longer_needed == False
 289     assert did_run == False
 290
 291   def testCredentialsWhenLoginSucceeds(self):
 292     credentials_backend = StubCredentialsBackend(login_return_value=True)
 293     did_run = self.runCredentialsTest(credentials_backend)
 294     assert credentials_backend.did_get_login == True
 295     assert credentials_backend.did_get_login_no_longer_needed == True
 296     assert did_run
 297
 298   def runCredentialsTest(self, credentials_backend):
 299     ps = page_set.PageSet()
 300     expectations = test_expectations.TestExpectations()
 301     page = page_module.Page(
 302         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
 303     page.credentials = "test"
 304     ps.pages.append(page)
 305
 306     did_run = [False]
 307
 308     try:
 309       with tempfile.NamedTemporaryFile(delete=False) as f:
 310         f.write(SIMPLE_CREDENTIALS_STRING)
 311         ps.credentials_path = f.name
 312
 313       class TestThatInstallsCredentialsBackend(page_test.PageTest):
 314         def __init__(self, credentials_backend):
 315           super(TestThatInstallsCredentialsBackend, self).__init__()
 316           self._credentials_backend = credentials_backend
 317
 318         def DidStartBrowser(self, browser):
 319           browser.credentials.AddBackend(self._credentials_backend)
 320
 321         def ValidatePage(self, *_):
 322           did_run[0] = True
 323
 324       test = TestThatInstallsCredentialsBackend(credentials_backend)
 325       options = options_for_unittests.GetCopy()
 326       options.output_format = 'none'
 327       options.suppress_gtest_report = True
 328       SetUpPageRunnerArguments(options)
 329       results = results_options.CreateResults(EmptyMetadataForTest(), options)
 330       page_runner.Run(test, ps, expectations, options, results)
 331     finally:
 332       os.remove(f.name)
 333
 334     return did_run[0]
 335
 336   def testUserAgent(self):
 337     ps = page_set.PageSet()
 338     expectations = test_expectations.TestExpectations()
 339     page = page_module.Page(
 340         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
 341     ps.pages.append(page)
 342     ps.user_agent_type = 'tablet'
 343
 344     class TestUserAgent(page_test.PageTest):
 345       def ValidatePage(self, _1, tab, _2):
 346         actual_user_agent = tab.EvaluateJavaScript('window.navigator.userAgent')
 347         expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet']
 348         assert actual_user_agent.strip() == expected_user_agent
 349
 350         # This is so we can check later that the test actually made it into this
 351         # function. Previously it was timing out before even getting here, which
 352         # should fail, but since it skipped all the asserts, it slipped by.
 353         self.hasRun = True # pylint: disable=W0201
 354
 355     test = TestUserAgent()
 356     options = options_for_unittests.GetCopy()
 357     options.output_format = 'none'
 358     options.suppress_gtest_report = True
 359     SetUpPageRunnerArguments(options)
 360     results = results_options.CreateResults(EmptyMetadataForTest(), options)
 361     page_runner.Run(test, ps, expectations, options, results)
 362
 363     self.assertTrue(hasattr(test, 'hasRun') and test.hasRun)
 364
 365   # Ensure that page_runner forces exactly 1 tab before running a page.
 366   @decorators.Enabled('has tabs')
 367   def testOneTab(self):
 368     ps = page_set.PageSet()
 369     expectations = test_expectations.TestExpectations()
 370     page = page_module.Page(
 371         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
 372     ps.pages.append(page)
 373
 374     class TestOneTab(page_test.PageTest):
 375       def __init__(self):
 376         super(TestOneTab, self).__init__()
 377         self._browser = None
 378
 379       def DidStartBrowser(self, browser):
 380         self._browser = browser
 381         self._browser.tabs.New()
 382
 383       def ValidatePage(self, *_):
 384         assert len(self._browser.tabs) == 1
 385
 386     test = TestOneTab()
 387     options = options_for_unittests.GetCopy()
 388     options.output_format = 'none'
 389     options.suppress_gtest_report = True
 390     SetUpPageRunnerArguments(options)
 391     results = results_options.CreateResults(EmptyMetadataForTest(), options)
 392     page_runner.Run(test, ps, expectations, options, results)
 393
 394   # Ensure that page_runner allows the test to customize the browser before it
 395   # launches.
 396   def testBrowserBeforeLaunch(self):
 397     ps = page_set.PageSet()
 398     expectations = test_expectations.TestExpectations()
 399     page = page_module.Page(
 400         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
 401     ps.pages.append(page)
 402
 403     class TestBeforeLaunch(page_test.PageTest):
 404       def __init__(self):
 405         super(TestBeforeLaunch, self).__init__()
 406         self._did_call_will_start = False
 407         self._did_call_did_start = False
 408
 409       def WillStartBrowser(self, browser):
 410         self._did_call_will_start = True
 411         # TODO(simonjam): Test that the profile is available.
 412
 413       def DidStartBrowser(self, browser):
 414         assert self._did_call_will_start
 415         self._did_call_did_start = True
 416
 417       def ValidatePage(self, *_):
 418         assert self._did_call_did_start
 419
 420     test = TestBeforeLaunch()
 421     options = options_for_unittests.GetCopy()
 422     options.output_format = 'none'
 423     options.suppress_gtest_report = True
 424     SetUpPageRunnerArguments(options)
 425     results = results_options.CreateResults(EmptyMetadataForTest(), options)
 426     page_runner.Run(test, ps, expectations, options, results)
 427
 428   def testRunPageWithStartupUrl(self):
 429     ps = page_set.PageSet()
 430     expectations = test_expectations.TestExpectations()
 431     expectations = test_expectations.TestExpectations()
 432     page = page_module.Page(
 433         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
 434     page.startup_url = 'about:blank'
 435     ps.pages.append(page)
 436
 437     class Measurement(page_test.PageTest):
 438       def __init__(self):
 439         super(Measurement, self).__init__()
 440         self.browser_restarted = False
 441
 442       def CustomizeBrowserOptionsForSinglePage(self, ps, options):
 443         self.browser_restarted = True
 444         super(Measurement, self).CustomizeBrowserOptionsForSinglePage(ps,
 445                                                                       options)
 446       def ValidateAndMeasurePage(self, page, tab, results):
 447         pass
 448
 449     options = options_for_unittests.GetCopy()
 450     options.page_repeat = 2
 451     options.output_format = 'none'
 452     options.suppress_gtest_report = True
 453     if not browser_finder.FindBrowser(options):
 454       return
 455     test = Measurement()
 456     SetUpPageRunnerArguments(options)
 457     results = results_options.CreateResults(EmptyMetadataForTest(), options)
 458     page_runner.Run(test, ps, expectations, options, results)
 459     self.assertEquals('about:blank', options.browser_options.startup_url)
 460     self.assertTrue(test.browser_restarted)
 461
 462   # Ensure that page_runner calls cleanUp when a page run fails.
 463   def testCleanUpPage(self):
 464     ps = page_set.PageSet()
 465     expectations = test_expectations.TestExpectations()
 466     page = page_module.Page(
 467         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
 468     ps.pages.append(page)
 469
 470     class Test(page_test.PageTest):
 471       def __init__(self):
 472         super(Test, self).__init__()
 473         self.did_call_clean_up = False
 474
 475       def ValidatePage(self, *_):
 476         raise exceptions.IntentionalException
 477
 478       def CleanUpAfterPage(self, page, tab):
 479         self.did_call_clean_up = True
 480
 481
 482     test = Test()
 483     options = options_for_unittests.GetCopy()
 484     options.output_format = 'none'
 485     options.suppress_gtest_report = True
 486     SetUpPageRunnerArguments(options)
 487     results = results_options.CreateResults(EmptyMetadataForTest(), options)
 488     page_runner.Run(test, ps, expectations, options, results)
 489     assert test.did_call_clean_up
 490
 491   # Ensure skipping the test if page cannot be run on the browser
 492   def testPageCannotRunOnBrowser(self):
 493     ps = page_set.PageSet()
 494     expectations = test_expectations.TestExpectations()
 495
 496     class PageThatCannotRunOnBrowser(page_module.Page):
 497
 498       def __init__(self):
 499         super(PageThatCannotRunOnBrowser, self).__init__(
 500             url='file://blank.html', page_set=ps,
 501             base_dir=util.GetUnittestDataDir())
 502
 503       def CanRunOnBrowser(self, _):
 504         return False
 505
 506       def ValidatePage(self, _):
 507         pass
 508
 509     class Test(page_test.PageTest):
 510       def __init__(self, *args, **kwargs):
 511         super(Test, self).__init__(*args, **kwargs)
 512         self.will_navigate_to_page_called = False
 513
 514       def ValidatePage(self, *args):
 515         pass
 516
 517       def WillNavigateToPage(self, _1, _2):
 518         self.will_navigate_to_page_called = True
 519
 520     test = Test()
 521     options = options_for_unittests.GetCopy()
 522     options.output_format = 'none'
 523     options.suppress_gtest_report = True
 524     SetUpPageRunnerArguments(options)
 525     results = results_options.CreateResults(EmptyMetadataForTest(), options)
 526     page_runner.Run(test, ps, expectations, options, results)
 527     self.assertFalse(test.will_navigate_to_page_called)
 528     self.assertEquals(0, len(GetSuccessfulPageRuns(results)))
 529     self.assertEquals(0, len(results.failures))
 530
 531   def TestUseLiveSitesFlag(self, options, expect_from_archive):
 532     ps = page_set.PageSet(
 533       file_path=util.GetUnittestDataDir(),
 534       archive_data_file='data/archive_blank.json')
 535     ps.pages.append(page_module.Page(
 536       'file://blank.html', ps, base_dir=ps.base_dir))
 537     expectations = test_expectations.TestExpectations()
 538
 539     class ArchiveTest(page_test.PageTest):
 540       def __init__(self):
 541         super(ArchiveTest, self).__init__()
 542         self.is_page_from_archive = False
 543         self.archive_path_exist = True
 544
 545       def WillNavigateToPage(self, page, tab):
 546         self.archive_path_exist = (page.archive_path
 547                                    and os.path.isfile(page.archive_path))
 548         self.is_page_from_archive = (
 549           tab.browser._wpr_server is not None) # pylint: disable=W0212
 550
 551       def ValidateAndMeasurePage(self, _, __, results):
 552         pass
 553
 554     test = ArchiveTest()
 555     results = results_options.CreateResults(EmptyMetadataForTest(), options)
 556     try:
 557       page_runner.Run(test, ps, expectations, options, results)
 558       if expect_from_archive and not test.archive_path_exist:
 559         logging.warning('archive path did not exist, asserting that page '
 560                         'is from archive is skipped.')
 561         return
 562       self.assertEquals(expect_from_archive, test.is_page_from_archive)
 563     finally:
 564       for p in ps:
 565         if os.path.isfile(p.archive_path):
 566           os.remove(p.archive_path)
 567
 568
 569   def testUseLiveSitesFlagSet(self):
 570     options = options_for_unittests.GetCopy()
 571     options.output_format = 'none'
 572     options.suppress_gtest_report = True
 573     options.use_live_sites = True
 574     SetUpPageRunnerArguments(options)
 575     self.TestUseLiveSitesFlag(options, expect_from_archive=False)
 576
 577   def testUseLiveSitesFlagUnset(self):
 578     options = options_for_unittests.GetCopy()
 579     options.output_format = 'none'
 580     options.suppress_gtest_report = True
 581     SetUpPageRunnerArguments(options)
 582     self.TestUseLiveSitesFlag(options, expect_from_archive=True)
 583
 584   def testMaxFailuresOptionIsRespected(self):
 585     class TestPage(page_module.Page):
 586       def __init__(self, *args, **kwargs):
 587         super(TestPage, self).__init__(*args, **kwargs)
 588         self.was_run = False
 589
 590       def RunNavigateSteps(self, action_runner):
 591         self.was_run = True
 592         raise Exception('Test exception')
 593
 594     class Test(page_test.PageTest):
 595       def ValidatePage(self, *args):
 596         pass
 597
 598     ps = page_set.PageSet()
 599     expectations = test_expectations.TestExpectations()
 600     page1 = TestPage(
 601         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
 602     ps.pages.append(page1)
 603     page2 = TestPage(
 604         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
 605     ps.pages.append(page2)
 606     page3 = TestPage(
 607         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
 608     ps.pages.append(page3)
 609     page4 = TestPage(
 610         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
 611     ps.pages.append(page4)
 612     page5 = TestPage(
 613         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
 614     ps.pages.append(page5)
 615
 616     options = options_for_unittests.GetCopy()
 617     options.output_format = 'none'
 618     options.suppress_gtest_report = True
 619     SetUpPageRunnerArguments(options)
 620     results = results_options.CreateResults(EmptyMetadataForTest(), options)
 621     page_runner.Run(Test(max_failures=2), ps, expectations, options, results)
 622     self.assertEquals(0, len(GetSuccessfulPageRuns(results)))
 623     # Runs up to max_failures+1 failing tests before stopping, since
 624     # every tests after max_failures failures have been encountered
 625     # may all be passing.
 626     self.assertEquals(3, len(results.failures))
 627     self.assertTrue(page1.was_run)
 628     self.assertTrue(page2.was_run)
 629     self.assertTrue(page3.was_run)
 630     self.assertFalse(page4.was_run)
 631     self.assertFalse(page5.was_run)