src/tools/telemetry/telemetry/web_perf/timeline_based_measurement_unittest.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import os
   6 import unittest
   7
   8 from telemetry import benchmark
   9 from telemetry.core import platform
  10 from telemetry.core import wpr_modes
  11 from telemetry.page import page as page_module
  12 from telemetry.page import page_set
  13 from telemetry.results import page_test_results
  14 from telemetry.timeline import model as model_module
  15 from telemetry.timeline import async_slice
  16 from telemetry.unittest import options_for_unittests
  17 from telemetry.unittest import page_test_test_case
  18 from telemetry.value import scalar
  19 from telemetry.web_perf import timeline_based_measurement as tbm_module
  20 from telemetry.web_perf import timeline_interaction_record as tir_module
  21 from telemetry.web_perf.metrics import timeline_based_metric
  22
  23
  24 class FakeFastMetric(timeline_based_metric.TimelineBasedMetric):
  25
  26   def AddResults(self, model, renderer_thread, interaction_records, results):
  27     results.AddValue(scalar.ScalarValue(
  28         results.current_page, 'FakeFastMetric', 'ms', 1))
  29     results.AddValue(scalar.ScalarValue(
  30         results.current_page, 'FastMetricRecords', 'count',
  31         len(interaction_records)))
  32
  33
  34 class FakeSmoothMetric(timeline_based_metric.TimelineBasedMetric):
  35
  36   def AddResults(self, model, renderer_thread, interaction_records, results):
  37     results.AddValue(scalar.ScalarValue(
  38         results.current_page, 'FakeSmoothMetric', 'ms', 1))
  39     results.AddValue(scalar.ScalarValue(
  40         results.current_page, 'SmoothMetricRecords', 'count',
  41         len(interaction_records)))
  42
  43
  44 class FakeLoadingMetric(timeline_based_metric.TimelineBasedMetric):
  45
  46   def AddResults(self, model, renderer_thread, interaction_records, results):
  47     results.AddValue(scalar.ScalarValue(
  48         results.current_page, 'FakeLoadingMetric', 'ms', 2))
  49     results.AddValue(scalar.ScalarValue(
  50         results.current_page, 'LoadingMetricRecords', 'count',
  51         len(interaction_records)))
  52
  53
  54 def GetMetricFromMetricType(metric_type):
  55   if metric_type == tir_module.IS_FAST:
  56     return FakeFastMetric()
  57   if metric_type == tir_module.IS_SMOOTH:
  58     return FakeSmoothMetric()
  59   if metric_type == tir_module.IS_RESPONSIVE:
  60     return FakeLoadingMetric()
  61   raise Exception('Unrecognized metric type: %s' % metric_type)
  62
  63
  64 class TimelineBasedMetricTestData(object):
  65
  66   def __init__(self):
  67     self._model = model_module.TimelineModel()
  68     renderer_process = self._model.GetOrCreateProcess(1)
  69     self._renderer_thread = renderer_process.GetOrCreateThread(2)
  70     self._renderer_thread.name = 'CrRendererMain'
  71     self._results = page_test_results.PageTestResults()
  72     self._metric = None
  73     self._ps = None
  74
  75   @property
  76   def results(self):
  77     return self._results
  78
  79   @property
  80   def metric(self):
  81     return self._metric
  82
  83   def AddInteraction(self, marker='', ts=0, duration=5):
  84     self._renderer_thread.async_slices.append(async_slice.AsyncSlice(
  85         'category', marker, timestamp=ts, duration=duration,
  86         start_thread=self._renderer_thread, end_thread=self._renderer_thread,
  87         thread_start=ts, thread_duration=duration))
  88
  89   def FinalizeImport(self):
  90     self._model.FinalizeImport()
  91     self._metric = tbm_module._TimelineBasedMetrics(  # pylint: disable=W0212
  92         self._model, self._renderer_thread, GetMetricFromMetricType)
  93     self._ps = page_set.PageSet(file_path=os.path.dirname(__file__))
  94     self._ps.AddPageWithDefaultRunNavigate('http://www.bar.com/')
  95     self._results.WillRunPage(self._ps.pages[0])
  96
  97   def AddResults(self):
  98     self._metric.AddResults(self._results)
  99     self._results.DidRunPage(self._ps.pages[0])
 100
 101
 102 class TimelineBasedMetricsTests(unittest.TestCase):
 103
 104   def testFindTimelineInteractionRecords(self):
 105     d = TimelineBasedMetricTestData()
 106     d.AddInteraction(ts=0, duration=20,
 107                      marker='Interaction.LogicalName1/is_smooth')
 108     d.AddInteraction(ts=25, duration=5,
 109                      marker='Interaction.LogicalName2/is_responsive')
 110     d.AddInteraction(ts=50, duration=15,
 111                      marker='Interaction.LogicalName3/is_fast')
 112     d.FinalizeImport()
 113     interactions = d.metric.FindTimelineInteractionRecords()
 114     self.assertEquals(3, len(interactions))
 115     self.assertTrue(interactions[0].is_smooth)
 116     self.assertEquals(0, interactions[0].start)
 117     self.assertEquals(20, interactions[0].end)
 118
 119     self.assertTrue(interactions[1].is_responsive)
 120     self.assertEquals(25, interactions[1].start)
 121     self.assertEquals(30, interactions[1].end)
 122
 123     self.assertTrue(interactions[2].is_fast)
 124     self.assertEquals(50, interactions[2].start)
 125     self.assertEquals(65, interactions[2].end)
 126
 127   def testAddResults(self):
 128     d = TimelineBasedMetricTestData()
 129     d.AddInteraction(ts=0, duration=20,
 130                      marker='Interaction.LogicalName1/is_smooth')
 131     d.AddInteraction(ts=25, duration=5,
 132                      marker='Interaction.LogicalName2/is_responsive')
 133     d.AddInteraction(ts=50, duration=15,
 134                      marker='Interaction.LogicalName3/is_fast')
 135     d.FinalizeImport()
 136     d.AddResults()
 137     self.assertEquals(1, len(d.results.FindAllPageSpecificValuesNamed(
 138         'LogicalName1-FakeSmoothMetric')))
 139     self.assertEquals(1, len(d.results.FindAllPageSpecificValuesNamed(
 140         'LogicalName2-FakeLoadingMetric')))
 141     self.assertEquals(1, len(d.results.FindAllPageSpecificValuesNamed(
 142         'LogicalName3-FakeFastMetric')))
 143
 144   def testNoInteractions(self):
 145     d = TimelineBasedMetricTestData()
 146     d.FinalizeImport()
 147     self.assertRaises(tbm_module.InvalidInteractions, d.AddResults)
 148
 149   def testDuplicateUnrepeatableInteractions(self):
 150     d = TimelineBasedMetricTestData()
 151     d.AddInteraction(ts=10, duration=5,
 152                      marker='Interaction.LogicalName/is_smooth')
 153     d.AddInteraction(ts=20, duration=5,
 154                      marker='Interaction.LogicalName/is_smooth')
 155     d.FinalizeImport()
 156     self.assertRaises(tbm_module.InvalidInteractions, d.AddResults)
 157
 158   def testDuplicateRepeatableInteractions(self):
 159     d = TimelineBasedMetricTestData()
 160     d.AddInteraction(ts=10, duration=5,
 161                      marker='Interaction.LogicalName/is_smooth,repeatable')
 162     d.AddInteraction(ts=20, duration=5,
 163                      marker='Interaction.LogicalName/is_smooth,repeatable')
 164     d.FinalizeImport()
 165     d.AddResults()
 166     self.assertEquals(1, len(d.results.pages_that_succeeded))
 167
 168   def testDuplicateRepeatableInteractionsWithDifferentMetrics(self):
 169     d = TimelineBasedMetricTestData()
 170
 171     responsive_marker = 'Interaction.LogicalName/is_responsive,repeatable'
 172     d.AddInteraction(ts=10, duration=5, marker=responsive_marker)
 173     smooth_marker = 'Interaction.LogicalName/is_smooth,repeatable'
 174     d.AddInteraction(ts=20, duration=5, marker=smooth_marker)
 175     d.FinalizeImport()
 176     self.assertRaises(tbm_module.InvalidInteractions, d.AddResults)
 177
 178
 179 class TestTimelinebasedMeasurementPage(page_module.Page):
 180
 181   def __init__(self, ps, base_dir, trigger_animation=False,
 182                trigger_jank=False, trigger_slow=False):
 183     super(TestTimelinebasedMeasurementPage, self).__init__(
 184         'file://interaction_enabled_page.html', ps, base_dir)
 185     self._trigger_animation = trigger_animation
 186     self._trigger_jank = trigger_jank
 187     self._trigger_slow = trigger_slow
 188
 189   def RunPageInteractions(self, action_runner):
 190     if self._trigger_animation:
 191       action_runner.TapElement('#animating-button')
 192       action_runner.WaitForJavaScriptCondition('window.animationDone')
 193     if self._trigger_jank:
 194       action_runner.TapElement('#jank-button')
 195       action_runner.WaitForJavaScriptCondition('window.jankScriptDone')
 196     if self._trigger_slow:
 197       action_runner.TapElement('#slow-button')
 198       action_runner.WaitForJavaScriptCondition('window.slowScriptDone')
 199
 200
 201 class TimelineBasedMeasurementTest(page_test_test_case.PageTestTestCase):
 202
 203   def setUp(self):
 204     self._options = options_for_unittests.GetCopy()
 205     self._options.browser_options.wpr_mode = wpr_modes.WPR_OFF
 206
 207   def testSmoothnessTimelineBasedMeasurementForSmoke(self):
 208     ps = self.CreateEmptyPageSet()
 209     ps.AddPage(TestTimelinebasedMeasurementPage(
 210         ps, ps.base_dir, trigger_animation=True))
 211
 212     measurement = tbm_module.TimelineBasedMeasurement()
 213     results = self.RunMeasurement(measurement, ps,
 214                                   options=self._options)
 215
 216     self.assertEquals(0, len(results.failures))
 217     v = results.FindAllPageSpecificValuesNamed(
 218         'CenterAnimation-frame_time_discrepancy')
 219     self.assertEquals(len(v), 1)
 220     v = results.FindAllPageSpecificValuesNamed(
 221         'DrawerAnimation-frame_time_discrepancy')
 222     self.assertEquals(len(v), 1)
 223
 224   def testFastTimelineBasedMeasurementForSmoke(self):
 225     ps = self.CreateEmptyPageSet()
 226     ps.AddPage(TestTimelinebasedMeasurementPage(
 227         ps, ps.base_dir, trigger_slow=True))
 228
 229     measurement = tbm_module.TimelineBasedMeasurement()
 230     results = self.RunMeasurement(measurement, ps, options=self._options)
 231
 232     self.assertEquals([], results.failures)
 233     expected_names = set([
 234         'SlowThreadJsRun-fast-duration',
 235         'SlowThreadJsRun-fast-idle_time',
 236         'SlowThreadJsRun-fast-incremental_marking',
 237         'SlowThreadJsRun-fast-incremental_marking_outside_idle',
 238         'SlowThreadJsRun-fast-mark_compactor',
 239         'SlowThreadJsRun-fast-mark_compactor_outside_idle',
 240         'SlowThreadJsRun-fast-scavenger',
 241         'SlowThreadJsRun-fast-scavenger_outside_idle',
 242         'SlowThreadJsRun-fast-total_garbage_collection',
 243         'SlowThreadJsRun-fast-total_garbage_collection_outside_idle',
 244         'trace',
 245         ])
 246     if platform.GetHostPlatform().GetOSName() != 'win':
 247       # CPU metric is only supported non-Windows platforms.
 248       expected_names.add('SlowThreadJsRun-fast-cpu_time')
 249     self.assertEquals(
 250         expected_names, set(v.name for v in results.all_page_specific_values))
 251
 252     # In interaction_enabled_page.html, the "slow" interaction executes
 253     # a loop with window.performance.now() to wait 200ms.
 254     # fast-duration measures wall time so its value should be at least 200ms.
 255     v = results.FindAllPageSpecificValuesNamed('SlowThreadJsRun-fast-duration')
 256     self.assertGreaterEqual(v[0].value, 200.0)
 257
 258   # Disabled since mainthread_jank metric is not supported on windows platform.
 259   @benchmark.Disabled('win')
 260   def testMainthreadJankTimelineBasedMeasurement(self):
 261     ps = self.CreateEmptyPageSet()
 262     ps.AddPage(TestTimelinebasedMeasurementPage(
 263         ps, ps.base_dir, trigger_jank=True))
 264
 265     measurement = tbm_module.TimelineBasedMeasurement()
 266     results = self.RunMeasurement(measurement, ps,
 267                                   options=self._options)
 268     self.assertEquals(0, len(results.failures))
 269
 270     # In interaction_enabled_page.html, we create a jank loop based on
 271     # window.performance.now() (basically loop for x milliseconds).
 272     # Since window.performance.now() uses wall-time instead of thread time,
 273     # we only assert the biggest jank > 50ms here to account for the fact
 274     # that the browser may deschedule during the jank loop.
 275     v = results.FindAllPageSpecificValuesNamed(
 276         'JankThreadJSRun-responsive-biggest_jank_thread_time')
 277     self.assertGreaterEqual(v[0].value, 50)
 278
 279     v = results.FindAllPageSpecificValuesNamed(
 280         'JankThreadJSRun-responsive-total_big_jank_thread_time')
 281     self.assertGreaterEqual(v[0].value, 50)