src/tools/telemetry/telemetry/page/test_expectations_unittest.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 import unittest
   5
   6 from telemetry.core import system_info
   7 from telemetry.page import page as page_module
   8 from telemetry.page import page_set
   9 from telemetry.page import test_expectations
  10
  11 VENDOR_NVIDIA = 0x10DE
  12 VENDOR_AMD = 0x1002
  13 VENDOR_INTEL = 0x8086
  14
  15 VENDOR_STRING_IMAGINATION = 'Imagination Technologies'
  16 DEVICE_STRING_SGX = 'PowerVR SGX 554'
  17
  18 class StubPlatform(object):
  19   def __init__(self, os_name, os_version_name=None):
  20     self.os_name = os_name
  21     self.os_version_name = os_version_name
  22
  23   def GetOSName(self):
  24     return self.os_name
  25
  26   def GetOSVersionName(self):
  27     return self.os_version_name
  28
  29 class StubBrowser(object):
  30   def __init__(self, platform, gpu, device, vendor_string, device_string):
  31     self.platform = platform
  32     self.system_info = system_info.SystemInfo.FromDict({
  33       'model_name': '',
  34       'gpu': {
  35         'devices': [
  36           { 'vendor_id': gpu, 'device_id': device,
  37             'vendor_string': vendor_string, 'device_string': device_string },
  38         ]
  39       }
  40     })
  41
  42   @property
  43   def supports_system_info(self):
  44     return False if not self.system_info else True
  45
  46   def GetSystemInfo(self):
  47     return self.system_info
  48
  49 class SampleTestExpectations(test_expectations.TestExpectations):
  50   def SetExpectations(self):
  51     self.Fail('page1.html', ['win', 'mac'], bug=123)
  52     self.Fail('page2.html', ['vista'], bug=123)
  53     self.Fail('page3.html', bug=123)
  54     self.Fail('page4.*', bug=123)
  55     self.Fail('http://test.com/page5.html', bug=123)
  56     self.Fail('page6.html', ['nvidia', 'intel'], bug=123)
  57     self.Fail('page7.html', [('nvidia', 0x1001), ('nvidia', 0x1002)], bug=123)
  58     self.Fail('page8.html', ['win', 'intel', ('amd', 0x1001)], bug=123)
  59     self.Fail('page9.html', ['imagination'])
  60     self.Fail('page10.html', [('imagination', 'PowerVR SGX 554')])
  61     self.Fail('Pages.page_11')
  62     self.Fail('page12.html', ['mountainlion'])
  63     self.Fail('page13.html', ['mavericks'])
  64     self.Fail('page14.html', ['yosemite'])
  65
  66 class TestExpectationsTest(unittest.TestCase):
  67   def setUp(self):
  68     self.expectations = SampleTestExpectations()
  69
  70   def assertExpectationEquals(self, expected, page, platform='', gpu=0,
  71       device=0, vendor_string='', device_string=''):
  72     result = self.expectations.GetExpectationForPage(StubBrowser(platform, gpu,
  73         device, vendor_string, device_string), page)
  74     self.assertEquals(expected, result)
  75
  76   # Pages with no expectations should always return 'pass'
  77   def testNoExpectations(self):
  78     ps = page_set.PageSet()
  79     page = page_module.Page('http://test.com/page0.html', ps)
  80     self.assertExpectationEquals('pass', page, StubPlatform('win'))
  81
  82   # Pages with expectations for an OS should only return them when running on
  83   # that OS
  84   def testOSExpectations(self):
  85     ps = page_set.PageSet()
  86     page = page_module.Page('http://test.com/page1.html', ps)
  87     self.assertExpectationEquals('fail', page, StubPlatform('win'))
  88     self.assertExpectationEquals('fail', page, StubPlatform('mac'))
  89     self.assertExpectationEquals('pass', page, StubPlatform('linux'))
  90
  91   # Pages with expectations for an OS version should only return them when
  92   # running on that OS version
  93   def testOSVersionExpectations(self):
  94     ps = page_set.PageSet()
  95     page = page_module.Page('http://test.com/page2.html', ps)
  96     self.assertExpectationEquals('fail', page, StubPlatform('win', 'vista'))
  97     self.assertExpectationEquals('pass', page, StubPlatform('win', 'win7'))
  98
  99   # Pages with non-conditional expectations should always return that
 100   # expectation regardless of OS or OS version
 101   def testConditionlessExpectations(self):
 102     ps = page_set.PageSet()
 103     page = page_module.Page('http://test.com/page3.html', ps)
 104     self.assertExpectationEquals('fail', page, StubPlatform('win'))
 105     self.assertExpectationEquals('fail', page, StubPlatform('mac', 'lion'))
 106     self.assertExpectationEquals('fail', page, StubPlatform('linux'))
 107
 108   # Expectations with wildcard characters should return for matching patterns
 109   def testWildcardExpectations(self):
 110     ps = page_set.PageSet()
 111     page = page_module.Page('http://test.com/page4.html', ps)
 112     page_js = page_module.Page('http://test.com/page4.html', ps)
 113     self.assertExpectationEquals('fail', page, StubPlatform('win'))
 114     self.assertExpectationEquals('fail', page_js, StubPlatform('win'))
 115
 116   # Expectations with absolute paths should match the entire path
 117   def testAbsoluteExpectations(self):
 118     ps = page_set.PageSet()
 119     page = page_module.Page('http://test.com/page5.html', ps)
 120     page_org = page_module.Page('http://test.org/page5.html', ps)
 121     page_https = page_module.Page('https://test.com/page5.html', ps)
 122     self.assertExpectationEquals('fail', page, StubPlatform('win'))
 123     self.assertExpectationEquals('pass', page_org, StubPlatform('win'))
 124     self.assertExpectationEquals('pass', page_https, StubPlatform('win'))
 125
 126   # Pages with expectations for a GPU should only return them when running with
 127   # that GPU
 128   def testGpuExpectations(self):
 129     ps = page_set.PageSet()
 130     page = page_module.Page('http://test.com/page6.html', ps)
 131     self.assertExpectationEquals('fail', page, gpu=VENDOR_NVIDIA)
 132     self.assertExpectationEquals('fail', page, gpu=VENDOR_INTEL)
 133     self.assertExpectationEquals('pass', page, gpu=VENDOR_AMD)
 134
 135   # Pages with expectations for a GPU should only return them when running with
 136   # that GPU
 137   def testGpuDeviceIdExpectations(self):
 138     ps = page_set.PageSet()
 139     page = page_module.Page('http://test.com/page7.html', ps)
 140     self.assertExpectationEquals('fail', page, gpu=VENDOR_NVIDIA, device=0x1001)
 141     self.assertExpectationEquals('fail', page, gpu=VENDOR_NVIDIA, device=0x1002)
 142     self.assertExpectationEquals('pass', page, gpu=VENDOR_NVIDIA, device=0x1003)
 143     self.assertExpectationEquals('pass', page, gpu=VENDOR_AMD, device=0x1001)
 144
 145   # Pages with multiple expectations should only return them when all criteria
 146   # is met
 147   def testMultipleExpectations(self):
 148     ps = page_set.PageSet()
 149     page = page_module.Page('http://test.com/page8.html', ps)
 150     self.assertExpectationEquals('fail', page,
 151         StubPlatform('win'), VENDOR_AMD, 0x1001)
 152     self.assertExpectationEquals('fail', page,
 153         StubPlatform('win'), VENDOR_INTEL, 0x1002)
 154     self.assertExpectationEquals('pass', page,
 155         StubPlatform('win'), VENDOR_NVIDIA, 0x1001)
 156     self.assertExpectationEquals('pass', page,
 157         StubPlatform('mac'), VENDOR_AMD, 0x1001)
 158     self.assertExpectationEquals('pass', page,
 159         StubPlatform('win'), VENDOR_AMD, 0x1002)
 160
 161   # Pages with expectations based on GPU vendor string.
 162   def testGpuVendorStringExpectations(self):
 163     ps = page_set.PageSet()
 164     page = page_module.Page('http://test.com/page9.html', ps)
 165     self.assertExpectationEquals('fail', page,
 166                                  vendor_string=VENDOR_STRING_IMAGINATION,
 167                                  device_string=DEVICE_STRING_SGX)
 168     self.assertExpectationEquals('fail', page,
 169                                  vendor_string=VENDOR_STRING_IMAGINATION,
 170                                  device_string='Triangle Monster 3000')
 171     self.assertExpectationEquals('pass', page,
 172                                  vendor_string='Acme',
 173                                  device_string=DEVICE_STRING_SGX)
 174
 175   # Pages with expectations based on GPU vendor and renderer string pairs.
 176   def testGpuDeviceStringExpectations(self):
 177     ps = page_set.PageSet()
 178     page = page_module.Page('http://test.com/page10.html', ps)
 179     self.assertExpectationEquals('fail', page,
 180                                  vendor_string=VENDOR_STRING_IMAGINATION,
 181                                  device_string=DEVICE_STRING_SGX)
 182     self.assertExpectationEquals('pass', page,
 183                                  vendor_string=VENDOR_STRING_IMAGINATION,
 184                                  device_string='Triangle Monster 3000')
 185     self.assertExpectationEquals('pass', page,
 186                                  vendor_string='Acme',
 187                                  device_string=DEVICE_STRING_SGX)
 188
 189   # Expectations can be set against page names as well as urls
 190   def testPageNameExpectations(self):
 191     ps = page_set.PageSet()
 192     page = page_module.Page('http://test.com/page11.html', ps,
 193                             name='Pages.page_11')
 194     self.assertExpectationEquals('fail', page)
 195
 196   # Verify version-specific Mac expectations.
 197   def testMacVersionExpectations(self):
 198     ps = page_set.PageSet()
 199     page = page_module.Page('http://test.com/page12.html', ps)
 200     self.assertExpectationEquals('fail', page,
 201                                  StubPlatform('mac', 'mountainlion'))
 202     self.assertExpectationEquals('pass', page,
 203                                  StubPlatform('mac', 'mavericks'))
 204     self.assertExpectationEquals('pass', page,
 205                                  StubPlatform('mac', 'yosemite'))
 206     ps = page_set.PageSet()
 207     page = page_module.Page('http://test.com/page13.html', ps)
 208     self.assertExpectationEquals('pass', page,
 209                                  StubPlatform('mac', 'mountainlion'))
 210     self.assertExpectationEquals('fail', page,
 211                                  StubPlatform('mac', 'mavericks'))
 212     self.assertExpectationEquals('pass', page,
 213                                  StubPlatform('mac', 'yosemite'))
 214     ps = page_set.PageSet()
 215     page = page_module.Page('http://test.com/page14.html', ps)
 216     self.assertExpectationEquals('pass', page,
 217                                  StubPlatform('mac', 'mountainlion'))
 218     self.assertExpectationEquals('pass', page,
 219                                  StubPlatform('mac', 'mavericks'))
 220     self.assertExpectationEquals('fail', page,
 221                                  StubPlatform('mac', 'yosemite'))