src/tools/telemetry/telemetry/page/page_runner_unittest.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 import logging
   5 import os
   6 import tempfile
   7 import unittest
   8
   9 from telemetry.core import user_agent
  10 from telemetry.core import util
  11 from telemetry.page import page as page_module
  12 from telemetry.page import page_measurement
  13 from telemetry.page import page_set
  14 from telemetry.page import page_test
  15 from telemetry.page import page_runner
  16 from telemetry.unittest import options_for_unittests
  17 from telemetry.page import test_expectations
  18
  19 SIMPLE_CREDENTIALS_STRING = """
  20 {
  21   "test": {
  22     "username": "example",
  23     "password": "asdf"
  24   }
  25 }
  26 """
  27 class StubCredentialsBackend(object):
  28   def __init__(self, login_return_value):
  29     self.did_get_login = False
  30     self.did_get_login_no_longer_needed = False
  31     self.login_return_value = login_return_value
  32
  33   @property
  34   def credentials_type(self): # pylint: disable=R0201
  35     return 'test'
  36
  37   def LoginNeeded(self, tab, config): # pylint: disable=W0613
  38     self.did_get_login = True
  39     return self.login_return_value
  40
  41   def LoginNoLongerNeeded(self, tab): # pylint: disable=W0613
  42     self.did_get_login_no_longer_needed = True
  43
  44 class PageRunnerTests(unittest.TestCase):
  45   # TODO(nduca): Move the basic "test failed, test succeeded" tests from
  46   # page_measurement_unittest to here.
  47
  48   def testHandlingOfCrashedTab(self):
  49     ps = page_set.PageSet()
  50     expectations = test_expectations.TestExpectations()
  51     page1 = page_module.Page('chrome://crash', ps)
  52     ps.pages.append(page1)
  53
  54     class Test(page_test.PageTest):
  55       def RunTest(self, *args):
  56         pass
  57
  58     options = options_for_unittests.GetCopy()
  59     options.output_format = 'none'
  60     results = page_runner.Run(Test('RunTest'), ps, expectations, options)
  61     self.assertEquals(0, len(results.successes))
  62     self.assertEquals(0, len(results.failures))
  63     self.assertEquals(1, len(results.errors))
  64
  65   def testHandlingOfCrashedTabWithExpectedFailure(self):
  66     ps = page_set.PageSet()
  67     expectations = test_expectations.TestExpectations()
  68     expectations.Fail('chrome://crash')
  69     page1 = page_module.Page('chrome://crash', ps)
  70     ps.pages.append(page1)
  71
  72     class Test(page_test.PageTest):
  73       def RunTest(self, *args):
  74         pass
  75
  76     options = options_for_unittests.GetCopy()
  77     options.output_format = 'none'
  78     results = page_runner.Run(Test('RunTest'), ps, expectations, options)
  79     self.assertEquals(1, len(results.successes))
  80     self.assertEquals(0, len(results.failures))
  81     self.assertEquals(0, len(results.errors))
  82
  83   def testDiscardFirstResult(self):
  84     ps = page_set.PageSet()
  85     expectations = test_expectations.TestExpectations()
  86     ps.pages.append(page_module.Page(
  87         'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
  88     ps.pages.append(page_module.Page(
  89         'file://blank.html', ps, base_dir=util.GetUnittestDataDir()))
  90
  91     class Measurement(page_measurement.PageMeasurement):
  92       @property
  93       def discard_first_result(self):
  94         return True
  95       def MeasurePage(self, *args):
  96         pass
  97
  98     options = options_for_unittests.GetCopy()
  99     options.output_format = 'none'
 100     options.reset_results = None
 101     options.upload_results = None
 102     options.results_label = None
 103
 104     options.repeat_options.page_repeat_iters = 1
 105     options.repeat_options.pageset_repeat_iters = 1
 106     results = page_runner.Run(Measurement(), ps, expectations, options)
 107     self.assertEquals(0, len(results.successes))
 108     self.assertEquals(0, len(results.failures))
 109
 110     options.repeat_options.page_repeat_iters = 1
 111     options.repeat_options.pageset_repeat_iters = 2
 112     results = page_runner.Run(Measurement(), ps, expectations, options)
 113     self.assertEquals(2, len(results.successes))
 114     self.assertEquals(0, len(results.failures))
 115
 116     options.repeat_options.page_repeat_iters = 2
 117     options.repeat_options.pageset_repeat_iters = 1
 118     results = page_runner.Run(Measurement(), ps, expectations, options)
 119     self.assertEquals(2, len(results.successes))
 120     self.assertEquals(0, len(results.failures))
 121
 122     options.output_format = 'html'
 123     options.repeat_options.page_repeat_iters = 1
 124     options.repeat_options.pageset_repeat_iters = 1
 125     results = page_runner.Run(Measurement(), ps, expectations, options)
 126     self.assertEquals(0, len(results.successes))
 127     self.assertEquals(0, len(results.failures))
 128
 129   def testCredentialsWhenLoginFails(self):
 130     credentials_backend = StubCredentialsBackend(login_return_value=False)
 131     did_run = self.runCredentialsTest(credentials_backend)
 132     assert credentials_backend.did_get_login == True
 133     assert credentials_backend.did_get_login_no_longer_needed == False
 134     assert did_run == False
 135
 136   def testCredentialsWhenLoginSucceeds(self):
 137     credentials_backend = StubCredentialsBackend(login_return_value=True)
 138     did_run = self.runCredentialsTest(credentials_backend)
 139     assert credentials_backend.did_get_login == True
 140     assert credentials_backend.did_get_login_no_longer_needed == True
 141     assert did_run
 142
 143   def runCredentialsTest(self, # pylint: disable=R0201
 144                          credentials_backend):
 145     ps = page_set.PageSet()
 146     expectations = test_expectations.TestExpectations()
 147     page = page_module.Page(
 148         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
 149     page.credentials = "test"
 150     ps.pages.append(page)
 151
 152     did_run = [False]
 153
 154     try:
 155       with tempfile.NamedTemporaryFile(delete=False) as f:
 156         f.write(SIMPLE_CREDENTIALS_STRING)
 157         ps.credentials_path = f.name
 158
 159       class TestThatInstallsCredentialsBackend(page_test.PageTest):
 160         def __init__(self, credentials_backend):
 161           super(TestThatInstallsCredentialsBackend, self).__init__('RunTest')
 162           self._credentials_backend = credentials_backend
 163
 164         def DidStartBrowser(self, browser):
 165           browser.credentials.AddBackend(self._credentials_backend)
 166
 167         def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201
 168           did_run[0] = True
 169
 170       test = TestThatInstallsCredentialsBackend(credentials_backend)
 171       options = options_for_unittests.GetCopy()
 172       options.output_format = 'none'
 173       page_runner.Run(test, ps, expectations, options)
 174     finally:
 175       os.remove(f.name)
 176
 177     return did_run[0]
 178
 179   def testUserAgent(self):
 180     ps = page_set.PageSet()
 181     expectations = test_expectations.TestExpectations()
 182     page = page_module.Page(
 183         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
 184     ps.pages.append(page)
 185     ps.user_agent_type = 'tablet'
 186
 187     class TestUserAgent(page_test.PageTest):
 188       def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201
 189         actual_user_agent = tab.EvaluateJavaScript('window.navigator.userAgent')
 190         expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet']
 191         assert actual_user_agent.strip() == expected_user_agent
 192
 193         # This is so we can check later that the test actually made it into this
 194         # function. Previously it was timing out before even getting here, which
 195         # should fail, but since it skipped all the asserts, it slipped by.
 196         self.hasRun = True # pylint: disable=W0201
 197
 198     test = TestUserAgent('RunTest')
 199     options = options_for_unittests.GetCopy()
 200     options.output_format = 'none'
 201     page_runner.Run(test, ps, expectations, options)
 202
 203     self.assertTrue(hasattr(test, 'hasRun') and test.hasRun)
 204
 205   # Ensure that page_runner forces exactly 1 tab before running a page.
 206   def testOneTab(self):
 207     ps = page_set.PageSet()
 208     expectations = test_expectations.TestExpectations()
 209     page = page_module.Page(
 210         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
 211     ps.pages.append(page)
 212
 213     class TestOneTab(page_test.PageTest):
 214       def __init__(self,
 215                    test_method_name,
 216                    action_name_to_run='',
 217                    needs_browser_restart_after_each_run=False):
 218         super(TestOneTab, self).__init__(test_method_name, action_name_to_run,
 219                                          needs_browser_restart_after_each_run)
 220         self._browser = None
 221
 222       def DidStartBrowser(self, browser):
 223         self._browser = browser
 224         if self._browser.supports_tab_control:
 225           self._browser.tabs.New()
 226
 227       def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201
 228         if not self._browser.supports_tab_control:
 229           logging.warning('Browser does not support tab control, skipping test')
 230           return
 231         assert len(self._browser.tabs) == 1
 232
 233     test = TestOneTab('RunTest')
 234     options = options_for_unittests.GetCopy()
 235     options.output_format = 'none'
 236     page_runner.Run(test, ps, expectations, options)
 237
 238   # Ensure that page_runner allows the test to customize the browser before it
 239   # launches.
 240   def testBrowserBeforeLaunch(self):
 241     ps = page_set.PageSet()
 242     expectations = test_expectations.TestExpectations()
 243     page = page_module.Page(
 244         'file://blank.html', ps, base_dir=util.GetUnittestDataDir())
 245     ps.pages.append(page)
 246
 247     class TestBeforeLaunch(page_test.PageTest):
 248       def __init__(self,
 249                    test_method_name,
 250                    action_name_to_run=''):
 251         super(TestBeforeLaunch, self).__init__(
 252             test_method_name, action_name_to_run, False)
 253         self._did_call_will_start = False
 254         self._did_call_did_start = False
 255
 256       def WillStartBrowser(self, browser):
 257         self._did_call_will_start = True
 258         # TODO(simonjam): Test that the profile is available.
 259
 260       def DidStartBrowser(self, browser):
 261         assert self._did_call_will_start
 262         self._did_call_did_start = True
 263
 264       def RunTest(self, page, tab, results): # pylint: disable=W0613,R0201
 265         assert self._did_call_did_start
 266
 267     test = TestBeforeLaunch('RunTest')
 268     options = options_for_unittests.GetCopy()
 269     options.output_format = 'none'
 270     page_runner.Run(test, ps, expectations, options)