src/tools/telemetry/telemetry/page/buildbot_page_measurement_results_unittest.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4 import os
   5 import unittest
   6
   7 from telemetry.page import buildbot_page_measurement_results
   8 from telemetry.page import page_set
   9 from telemetry.page import perf_tests_helper
  10 from telemetry.value import list_of_scalar_values
  11 from telemetry.value import scalar
  12
  13 def _MakePageSet():
  14   ps = page_set.PageSet(file_path=os.path.dirname(__file__))
  15   ps.AddPageWithDefaultRunNavigate('http://www.foo.com/')
  16   ps.AddPageWithDefaultRunNavigate('http://www.bar.com/')
  17   ps.AddPageWithDefaultRunNavigate('http://www.baz.com/')
  18   return ps
  19
  20 class SummarySavingPageMeasurementResults(
  21     buildbot_page_measurement_results.BuildbotPageMeasurementResults):
  22   def __init__(self, trace_tag=''):
  23     super(SummarySavingPageMeasurementResults, self).__init__(
  24         None, trace_tag=trace_tag)
  25     self.results = []
  26
  27   def _PrintPerfResult(self, *args):
  28     res = perf_tests_helper.PrintPerfResult(*args, print_to_stdout=False)
  29     self.results.append(res)
  30
  31 class BuildbotPageMeasurementResultsTest(unittest.TestCase):
  32   def assertEquals(self, ex, res):
  33     # This helps diagnose result mismatches.
  34     if ex != res and isinstance(ex, list):
  35       def CleanList(l):
  36         res = []
  37         for x in l:
  38           x = x.split('\n')
  39           res.extend(x)
  40         return res
  41       ex = CleanList(ex)
  42       res = CleanList(res)
  43       max_len = max(len(ex), len(res))
  44       max_width  = max([len(x) for x in ex + res])
  45       max_width = max(10, max_width)
  46       print "Lists differ!"
  47       print '%*s | %*s' % (max_width, 'expected', max_width, 'result')
  48       for i in range(max_len):
  49         if i < len(ex):
  50           e = ex[i]
  51         else:
  52           e = ''
  53         if i < len(res):
  54           r = res[i]
  55         else:
  56           r = ''
  57         if e != r:
  58           sep = '*'
  59         else:
  60           sep = '|'
  61         print '%*s %s %*s' % (max_width, e, sep, max_width, r)
  62       print ""
  63     super(BuildbotPageMeasurementResultsTest, self).assertEquals(ex, res)
  64
  65   def test_basic_summary(self):
  66     test_page_set = _MakePageSet()
  67
  68     measurement_results = SummarySavingPageMeasurementResults()
  69     measurement_results.WillMeasurePage(test_page_set.pages[0])
  70     measurement_results.Add('a', 'seconds', 3)
  71     measurement_results.DidMeasurePage()
  72     measurement_results.AddSuccess(test_page_set.pages[0])
  73
  74     measurement_results.WillMeasurePage(test_page_set.pages[1])
  75     measurement_results.Add('a', 'seconds', 7)
  76     measurement_results.DidMeasurePage()
  77     measurement_results.AddSuccess(test_page_set.pages[1])
  78
  79     measurement_results.PrintSummary()
  80     expected = ['RESULT a_by_url: http___www.bar.com_= 7 seconds',
  81                 'RESULT a_by_url: http___www.foo.com_= 3 seconds',
  82                 '*RESULT a: a= [3,7] seconds\nAvg a: 5.000000seconds\n' +
  83                     'Sd  a: 2.828427seconds',
  84                 'RESULT telemetry_page_measurement_results: ' +
  85                     'num_failed= 0 count',
  86                 'RESULT telemetry_page_measurement_results: ' +
  87                     'num_errored= 0 count']
  88     self.assertEquals(expected, measurement_results.results)
  89
  90   def test_basic_summary_with_only_one_page(self):
  91     test_page_set = _MakePageSet()
  92
  93     measurement_results = SummarySavingPageMeasurementResults()
  94     measurement_results.WillMeasurePage(test_page_set.pages[0])
  95     measurement_results.Add('a', 'seconds', 3)
  96     measurement_results.DidMeasurePage()
  97     measurement_results.AddSuccess(test_page_set.pages[0])
  98
  99     measurement_results.PrintSummary()
 100     expected = ['*RESULT a: a= 3 seconds',
 101                 'RESULT telemetry_page_measurement_results: ' +
 102                     'num_failed= 0 count',
 103                 'RESULT telemetry_page_measurement_results: ' +
 104                     'num_errored= 0 count']
 105     self.assertEquals(expected, measurement_results.results)
 106
 107   def test_basic_summary_nonuniform_results(self):
 108     test_page_set = _MakePageSet()
 109
 110     measurement_results = SummarySavingPageMeasurementResults()
 111     measurement_results.WillMeasurePage(test_page_set.pages[0])
 112     measurement_results.Add('a', 'seconds', 3)
 113     measurement_results.Add('b', 'seconds', 10)
 114     measurement_results.DidMeasurePage()
 115     measurement_results.AddSuccess(test_page_set.pages[0])
 116
 117     measurement_results.WillMeasurePage(test_page_set.pages[1])
 118     measurement_results.Add('a', 'seconds', 3)
 119     measurement_results.Add('b', 'seconds', 10)
 120     measurement_results.DidMeasurePage()
 121     measurement_results.AddSuccess(test_page_set.pages[1])
 122
 123     measurement_results.WillMeasurePage(test_page_set.pages[2])
 124     measurement_results.Add('a', 'seconds', 7)
 125     # Note, page[2] does not report a 'b' metric.
 126     measurement_results.DidMeasurePage()
 127     measurement_results.AddSuccess(test_page_set.pages[2])
 128
 129     measurement_results.PrintSummary()
 130     expected = ['RESULT a_by_url: http___www.bar.com_= 3 seconds',
 131                 'RESULT a_by_url: http___www.baz.com_= 7 seconds',
 132                 'RESULT a_by_url: http___www.foo.com_= 3 seconds',
 133                 '*RESULT a: a= [3,3,7] seconds\nAvg a: 4.333333seconds\n' +
 134                     'Sd  a: 2.309401seconds',
 135                 'RESULT b_by_url: http___www.bar.com_= 10 seconds',
 136                 'RESULT b_by_url: http___www.foo.com_= 10 seconds',
 137                 '*RESULT b: b= [10,10] seconds\nAvg b: 10.000000seconds',
 138                 'RESULT telemetry_page_measurement_results: ' +
 139                     'num_failed= 0 count',
 140                 'RESULT telemetry_page_measurement_results: ' +
 141                     'num_errored= 0 count']
 142     self.assertEquals(expected, measurement_results.results)
 143
 144   def test_basic_summary_pass_and_fail_page(self):
 145     """If a page failed, only print summary for individual pages."""
 146     test_page_set = _MakePageSet()
 147
 148     measurement_results = SummarySavingPageMeasurementResults()
 149     measurement_results.WillMeasurePage(test_page_set.pages[0])
 150     measurement_results.Add('a', 'seconds', 3)
 151     measurement_results.DidMeasurePage()
 152     measurement_results.AddFailureMessage(test_page_set.pages[0], 'message')
 153
 154     measurement_results.WillMeasurePage(test_page_set.pages[1])
 155     measurement_results.Add('a', 'seconds', 7)
 156     measurement_results.DidMeasurePage()
 157     measurement_results.AddSuccess(test_page_set.pages[1])
 158
 159     measurement_results.PrintSummary()
 160     expected = ['RESULT a_by_url: http___www.bar.com_= 7 seconds',
 161                 'RESULT a_by_url: http___www.foo.com_= 3 seconds',
 162                 'RESULT telemetry_page_measurement_results: ' +
 163                     'num_failed= 1 count',
 164                 'RESULT telemetry_page_measurement_results: ' +
 165                     'num_errored= 0 count']
 166     self.assertEquals(expected, measurement_results.results)
 167
 168   def test_repeated_pageset_one_iteration_one_page_fails(self):
 169     """Page fails on one iteration, no averaged results should print."""
 170     test_page_set = _MakePageSet()
 171
 172     measurement_results = SummarySavingPageMeasurementResults()
 173     measurement_results.WillMeasurePage(test_page_set.pages[0])
 174     measurement_results.Add('a', 'seconds', 3)
 175     measurement_results.DidMeasurePage()
 176
 177     measurement_results.WillMeasurePage(test_page_set.pages[1])
 178     measurement_results.Add('a', 'seconds', 7)
 179     measurement_results.DidMeasurePage()
 180     measurement_results.AddFailureMessage(test_page_set.pages[1], 'message')
 181
 182     measurement_results.WillMeasurePage(test_page_set.pages[0])
 183     measurement_results.Add('a', 'seconds', 4)
 184     measurement_results.DidMeasurePage()
 185     measurement_results.AddSuccess(test_page_set.pages[0])
 186
 187     measurement_results.WillMeasurePage(test_page_set.pages[1])
 188     measurement_results.Add('a', 'seconds', 8)
 189     measurement_results.DidMeasurePage()
 190     measurement_results.AddSuccess(test_page_set.pages[1])
 191
 192     measurement_results.PrintSummary()
 193     expected = ['RESULT a_by_url: http___www.bar.com_= [7,8] seconds\n' +
 194                     'Avg a_by_url: 7.500000seconds\n' +
 195                     'Sd  a_by_url: 0.707107seconds',
 196                 'RESULT a_by_url: http___www.foo.com_= [3,4] seconds\n' +
 197                     'Avg a_by_url: 3.500000seconds\n' +
 198                     'Sd  a_by_url: 0.707107seconds',
 199                 'RESULT telemetry_page_measurement_results: ' +
 200                     'num_failed= 1 count',
 201                 'RESULT telemetry_page_measurement_results: ' +
 202                     'num_errored= 0 count']
 203     self.assertEquals(expected, measurement_results.results)
 204
 205   def test_repeated_pageset_one_iteration_one_page_error(self):
 206     """Page error on one iteration, no averaged results should print."""
 207     test_page_set = _MakePageSet()
 208
 209     measurement_results = SummarySavingPageMeasurementResults()
 210     measurement_results.WillMeasurePage(test_page_set.pages[0])
 211     measurement_results.Add('a', 'seconds', 3)
 212     measurement_results.DidMeasurePage()
 213     measurement_results.AddSuccess(test_page_set.pages[0])
 214
 215     measurement_results.WillMeasurePage(test_page_set.pages[1])
 216     measurement_results.Add('a', 'seconds', 7)
 217     measurement_results.DidMeasurePage()
 218     measurement_results.AddErrorMessage(test_page_set.pages[1], 'message')
 219
 220     measurement_results.WillMeasurePage(test_page_set.pages[0])
 221     measurement_results.Add('a', 'seconds', 4)
 222     measurement_results.DidMeasurePage()
 223     measurement_results.AddSuccess(test_page_set.pages[0])
 224
 225     measurement_results.WillMeasurePage(test_page_set.pages[1])
 226     measurement_results.Add('a', 'seconds', 8)
 227     measurement_results.DidMeasurePage()
 228     measurement_results.AddSuccess(test_page_set.pages[1])
 229
 230     measurement_results.PrintSummary()
 231     expected = ['RESULT a_by_url: http___www.bar.com_= [7,8] seconds\n' +
 232                     'Avg a_by_url: 7.500000seconds\n' +
 233                     'Sd  a_by_url: 0.707107seconds',
 234                 'RESULT a_by_url: http___www.foo.com_= [3,4] seconds\n' +
 235                     'Avg a_by_url: 3.500000seconds\n' +
 236                     'Sd  a_by_url: 0.707107seconds',
 237                 'RESULT telemetry_page_measurement_results: ' +
 238                     'num_failed= 0 count',
 239                 'RESULT telemetry_page_measurement_results: ' +
 240                     'num_errored= 1 count']
 241     self.assertEquals(expected, measurement_results.results)
 242
 243   def test_repeated_pageset(self):
 244     test_page_set = _MakePageSet()
 245
 246     measurement_results = SummarySavingPageMeasurementResults()
 247     measurement_results.WillMeasurePage(test_page_set.pages[0])
 248     measurement_results.Add('a', 'seconds', 3)
 249     measurement_results.DidMeasurePage()
 250     measurement_results.AddSuccess(test_page_set.pages[0])
 251
 252     measurement_results.WillMeasurePage(test_page_set.pages[1])
 253     measurement_results.Add('a', 'seconds', 7)
 254     measurement_results.DidMeasurePage()
 255     measurement_results.AddSuccess(test_page_set.pages[1])
 256
 257     measurement_results.WillMeasurePage(test_page_set.pages[0])
 258     measurement_results.Add('a', 'seconds', 4)
 259     measurement_results.DidMeasurePage()
 260     measurement_results.AddSuccess(test_page_set.pages[0])
 261
 262     measurement_results.WillMeasurePage(test_page_set.pages[1])
 263     measurement_results.Add('a', 'seconds', 8)
 264     measurement_results.DidMeasurePage()
 265     measurement_results.AddSuccess(test_page_set.pages[1])
 266
 267     measurement_results.PrintSummary()
 268     expected = ['RESULT a_by_url: http___www.bar.com_= [7,8] seconds\n' +
 269                     'Avg a_by_url: 7.500000seconds\n' +
 270                     'Sd  a_by_url: 0.707107seconds',
 271                 'RESULT a_by_url: http___www.foo.com_= [3,4] seconds\n' +
 272                     'Avg a_by_url: 3.500000seconds\n' +
 273                     'Sd  a_by_url: 0.707107seconds',
 274                 '*RESULT a: a= [3,7,4,8] seconds\n' +
 275                     'Avg a: 5.500000seconds\n' +
 276                     'Sd  a: 2.380476seconds',
 277                 'RESULT telemetry_page_measurement_results: ' +
 278                     'num_failed= 0 count',
 279                 'RESULT telemetry_page_measurement_results: ' +
 280                     'num_errored= 0 count'
 281                 ]
 282     self.assertEquals(expected, measurement_results.results)
 283
 284   def test_repeated_pages(self):
 285     test_page_set = _MakePageSet()
 286
 287     measurement_results = SummarySavingPageMeasurementResults()
 288     measurement_results.WillMeasurePage(test_page_set.pages[0])
 289     measurement_results.Add('a', 'seconds', 3)
 290     measurement_results.DidMeasurePage()
 291     measurement_results.AddSuccess(test_page_set.pages[0])
 292
 293     measurement_results.WillMeasurePage(test_page_set.pages[0])
 294     measurement_results.Add('a', 'seconds', 4)
 295     measurement_results.DidMeasurePage()
 296     measurement_results.AddSuccess(test_page_set.pages[0])
 297
 298     measurement_results.WillMeasurePage(test_page_set.pages[1])
 299     measurement_results.Add('a', 'seconds', 7)
 300     measurement_results.DidMeasurePage()
 301     measurement_results.AddSuccess(test_page_set.pages[1])
 302
 303     measurement_results.WillMeasurePage(test_page_set.pages[1])
 304     measurement_results.Add('a', 'seconds', 8)
 305     measurement_results.DidMeasurePage()
 306     measurement_results.AddSuccess(test_page_set.pages[1])
 307
 308     measurement_results.PrintSummary()
 309     expected = ['RESULT a_by_url: http___www.bar.com_= [7,8] seconds\n' +
 310                     'Avg a_by_url: 7.500000seconds\n' +
 311                     'Sd  a_by_url: 0.707107seconds',
 312                 'RESULT a_by_url: http___www.foo.com_= [3,4] seconds\n' +
 313                     'Avg a_by_url: 3.500000seconds\n' +
 314                     'Sd  a_by_url: 0.707107seconds',
 315                 '*RESULT a: a= [3,4,7,8] seconds\n' +
 316                     'Avg a: 5.500000seconds\n' +
 317                     'Sd  a: 2.380476seconds',
 318                 'RESULT telemetry_page_measurement_results: ' +
 319                     'num_failed= 0 count',
 320                 'RESULT telemetry_page_measurement_results: ' +
 321                     'num_errored= 0 count'
 322                 ]
 323     self.assertEquals(expected, measurement_results.results)
 324
 325   def test_overall_results_trace_tag(self):
 326     test_page_set = _MakePageSet()
 327
 328     measurement_results = SummarySavingPageMeasurementResults(trace_tag='_ref')
 329
 330     measurement_results.AddSummaryValue(
 331         scalar.ScalarValue(None, 'a', 'seconds', 1))
 332
 333     measurement_results.WillMeasurePage(test_page_set.pages[0])
 334     measurement_results.Add('b', 'seconds', 2)
 335     measurement_results.DidMeasurePage()
 336     measurement_results.AddSuccess(test_page_set.pages[0])
 337
 338     measurement_results.WillMeasurePage(test_page_set.pages[1])
 339     measurement_results.Add('b', 'seconds', 3)
 340     measurement_results.DidMeasurePage()
 341     measurement_results.AddSuccess(test_page_set.pages[1])
 342
 343     measurement_results.AddSummaryValue(
 344         scalar.ScalarValue(None, 'c', 'seconds', 4))
 345
 346     measurement_results.PrintSummary()
 347
 348     expected = [
 349       '*RESULT b: b_ref= [2,3] seconds\n' +
 350       'Avg b: 2.500000seconds\nSd  b: 0.707107seconds',
 351       '*RESULT a: a_ref= 1 seconds',
 352       '*RESULT c: c_ref= 4 seconds',
 353       'RESULT telemetry_page_measurement_results: num_failed= 0 count',
 354       'RESULT telemetry_page_measurement_results: num_errored= 0 count']
 355     self.assertEquals(expected, measurement_results.results)
 356
 357
 358   def test_overall_results_page_runs_twice(self):
 359     test_page_set = _MakePageSet()
 360
 361     measurement_results = SummarySavingPageMeasurementResults()
 362
 363     measurement_results.AddSummaryValue(
 364         scalar.ScalarValue(None, 'a', 'seconds', 1))
 365
 366     measurement_results.WillMeasurePage(test_page_set.pages[0])
 367     measurement_results.Add('b', 'seconds', 2)
 368     measurement_results.DidMeasurePage()
 369     measurement_results.AddSuccess(test_page_set.pages[0])
 370
 371     measurement_results.WillMeasurePage(test_page_set.pages[0])
 372     measurement_results.Add('b', 'seconds', 3)
 373     measurement_results.DidMeasurePage()
 374     measurement_results.AddSuccess(test_page_set.pages[0])
 375
 376     measurement_results.PrintSummary()
 377
 378     expected = [
 379         'RESULT b_by_url: http___www.foo.com_= [2,3] seconds\n' +
 380             'Avg b_by_url: 2.500000seconds\nSd  b_by_url: 0.707107seconds',
 381         '*RESULT b: b= [2,3] seconds\n' +
 382         'Avg b: 2.500000seconds\nSd  b: 0.707107seconds',
 383         '*RESULT a: a= 1 seconds',
 384         'RESULT telemetry_page_measurement_results: num_failed= 0 count',
 385         'RESULT telemetry_page_measurement_results: num_errored= 0 count']
 386     self.assertEquals(expected, measurement_results.results)
 387
 388   def test_unimportant_results(self):
 389     test_page_set = _MakePageSet()
 390
 391     measurement_results = SummarySavingPageMeasurementResults()
 392
 393     measurement_results.AddSummaryValue(
 394         scalar.ScalarValue(None, 'a', 'seconds', 1, important=False))
 395
 396     measurement_results.WillMeasurePage(test_page_set.pages[0])
 397     measurement_results.Add('b', 'seconds', 2, data_type='unimportant')
 398     measurement_results.DidMeasurePage()
 399     measurement_results.AddSuccess(test_page_set.pages[0])
 400
 401     measurement_results.WillMeasurePage(test_page_set.pages[1])
 402     measurement_results.Add('b', 'seconds', 3, data_type='unimportant')
 403     measurement_results.DidMeasurePage()
 404     measurement_results.AddSuccess(test_page_set.pages[1])
 405
 406     measurement_results.PrintSummary()
 407
 408     self.assertEquals(
 409         measurement_results.results,
 410         ['RESULT b_by_url: http___www.bar.com_= 3 seconds',
 411          'RESULT b_by_url: http___www.foo.com_= 2 seconds',
 412          'RESULT b: b= [2,3] seconds\n' +
 413             'Avg b: 2.500000seconds\nSd  b: 0.707107seconds',
 414          'RESULT a: a= 1 seconds',
 415          'RESULT telemetry_page_measurement_results: num_failed= 0 count',
 416          'RESULT telemetry_page_measurement_results: num_errored= 0 count'])
 417
 418   def test_list_value(self):
 419     test_page_set = _MakePageSet()
 420
 421     measurement_results = SummarySavingPageMeasurementResults()
 422
 423     measurement_results.AddSummaryValue(
 424         list_of_scalar_values.ListOfScalarValues(None, 'a', 'seconds', [1, 1]))
 425
 426     measurement_results.WillMeasurePage(test_page_set.pages[0])
 427     measurement_results.Add('b', 'seconds', [2, 2])
 428     measurement_results.DidMeasurePage()
 429     measurement_results.AddSuccess(test_page_set.pages[0])
 430
 431     measurement_results.WillMeasurePage(test_page_set.pages[1])
 432     measurement_results.Add('b', 'seconds', [3, 3])
 433     measurement_results.DidMeasurePage()
 434     measurement_results.AddSuccess(test_page_set.pages[1])
 435
 436     measurement_results.PrintSummary()
 437
 438     expected = [
 439       'RESULT b_by_url: http___www.bar.com_= [3,3] seconds\n' +
 440           'Avg b_by_url: 3.000000seconds',
 441       'RESULT b_by_url: http___www.foo.com_= [2,2] seconds\n' +
 442           'Avg b_by_url: 2.000000seconds',
 443       '*RESULT b: b= [2,2,3,3] seconds\nAvg b: 2.500000seconds\n' +
 444           'Sd  b: 0.577350seconds',
 445       '*RESULT a: a= [1,1] seconds\nAvg a: 1.000000seconds',
 446       'RESULT telemetry_page_measurement_results: num_failed= 0 count',
 447       'RESULT telemetry_page_measurement_results: num_errored= 0 count']
 448     self.assertEquals(expected, measurement_results.results)
 449
 450   def test_histogram(self):
 451     test_page_set = _MakePageSet()
 452
 453     measurement_results = SummarySavingPageMeasurementResults()
 454     measurement_results.WillMeasurePage(test_page_set.pages[0])
 455     measurement_results.Add('a', 'units',
 456                             '{"buckets": [{"low": 1, "high": 2, "count": 1}]}',
 457                             data_type='unimportant-histogram')
 458     measurement_results.DidMeasurePage()
 459     measurement_results.AddSuccess(test_page_set.pages[0])
 460
 461     measurement_results.WillMeasurePage(test_page_set.pages[1])
 462     measurement_results.Add('a', 'units',
 463                           '{"buckets": [{"low": 2, "high": 3, "count": 1}]}',
 464                           data_type='unimportant-histogram')
 465     measurement_results.DidMeasurePage()
 466     measurement_results.AddSuccess(test_page_set.pages[1])
 467
 468     measurement_results.PrintSummary()
 469
 470     expected = [
 471         'HISTOGRAM a_by_url: http___www.bar.com_= ' +
 472             '{"buckets": [{"low": 2, "high": 3, "count": 1}]} units\n' +
 473             'Avg a_by_url: 2.500000units',
 474         'HISTOGRAM a_by_url: http___www.foo.com_= ' +
 475             '{"buckets": [{"low": 1, "high": 2, "count": 1}]} units\n' +
 476             'Avg a_by_url: 1.500000units',
 477         'RESULT telemetry_page_measurement_results: num_failed= 0 count',
 478         'RESULT telemetry_page_measurement_results: num_errored= 0 count']
 479     self.assertEquals(expected, measurement_results.results)