src/third_party/WebKit/Tools/TestResultServer/model/jsonresults_unittest.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #     * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #     * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #     * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 try:
  30     import jsonresults
  31     from jsonresults import *
  32 except ImportError:
  33     print "ERROR: Add the TestResultServer, google_appengine and yaml/lib directories to your PYTHONPATH"
  34     raise
  35
  36 import json
  37 import logging
  38 import unittest
  39
  40 FULL_RESULT_EXAMPLE = """ADD_RESULTS({
  41     "seconds_since_epoch": 1368146629,
  42     "tests": {
  43         "media": {
  44             "encrypted-media": {
  45                 "encrypted-media-v2-events.html": {
  46                     "bugs": ["crbug.com/1234"],
  47                     "expected": "TIMEOUT",
  48                     "actual": "TIMEOUT",
  49                     "time": 6.0
  50                 },
  51                 "encrypted-media-v2-syntax.html": {
  52                     "expected": "TIMEOUT",
  53                     "actual": "TIMEOUT"
  54                 }
  55             },
  56             "progress-events-generated-correctly.html": {
  57                 "expected": "PASS FAIL IMAGE TIMEOUT CRASH MISSING",
  58                 "actual": "TIMEOUT",
  59                 "time": 6.0
  60             },
  61             "W3C": {
  62                 "audio": {
  63                     "src": {
  64                         "src_removal_does_not_trigger_loadstart.html": {
  65                             "expected": "PASS",
  66                             "actual": "PASS",
  67                             "time": 3.5
  68                         }
  69                     }
  70                 },
  71                 "video": {
  72                     "src": {
  73                         "src_removal_does_not_trigger_loadstart.html": {
  74                             "expected": "PASS",
  75                             "actual": "PASS",
  76                             "time": 1.1
  77                         },
  78                         "notrun.html": {
  79                             "expected": "NOTRUN",
  80                             "actual": "SKIP",
  81                             "time": 1.1
  82                         }
  83                     }
  84                 }
  85             },
  86             "unexpected-skip.html": {
  87                 "expected": "PASS",
  88                 "actual": "SKIP"
  89             },
  90             "unexpected-fail.html": {
  91                 "expected": "PASS",
  92                 "actual": "FAIL"
  93             },
  94             "flaky-failed.html": {
  95                 "expected": "PASS FAIL",
  96                 "actual": "FAIL"
  97             },
  98             "media-document-audio-repaint.html": {
  99                 "expected": "IMAGE",
 100                 "actual": "IMAGE",
 101                 "time": 0.1
 102             },
 103             "unexpected-leak.html": {
 104                 "expected": "PASS",
 105                 "actual": "LEAK"
 106             }
 107         }
 108     },
 109     "skipped": 2,
 110     "num_regressions": 0,
 111     "build_number": "3",
 112     "interrupted": false,
 113     "layout_tests_dir": "\/tmp\/cr\/src\/third_party\/WebKit\/LayoutTests",
 114     "version": 3,
 115     "builder_name": "Webkit",
 116     "num_passes": 10,
 117     "pixel_tests_enabled": true,
 118     "blink_revision": "1234",
 119     "has_pretty_patch": true,
 120     "fixable": 25,
 121     "num_flaky": 0,
 122     "num_failures_by_type": {
 123         "CRASH": 3,
 124         "MISSING": 0,
 125         "TEXT": 3,
 126         "IMAGE": 1,
 127         "PASS": 10,
 128         "SKIP": 2,
 129         "TIMEOUT": 16,
 130         "IMAGE+TEXT": 0,
 131         "FAIL": 2,
 132         "AUDIO": 0,
 133         "LEAK": 1
 134     },
 135     "has_wdiff": true,
 136     "chromium_revision": "5678"
 137 });"""
 138
 139 JSON_RESULTS_OLD_TEMPLATE = (
 140     '{"[BUILDER_NAME]":{'
 141     '"allFixableCount":[[TESTDATA_COUNT]],'
 142     '"blinkRevision":[[TESTDATA_WEBKITREVISION]],'
 143     '"buildNumbers":[[TESTDATA_BUILDNUMBERS]],'
 144     '"chromeRevision":[[TESTDATA_CHROMEREVISION]],'
 145     '"failure_map": %s,'
 146     '"fixableCount":[[TESTDATA_COUNT]],'
 147     '"fixableCounts":[[TESTDATA_COUNTS]],'
 148     '"secondsSinceEpoch":[[TESTDATA_TIMES]],'
 149     '"tests":{[TESTDATA_TESTS]}'
 150     '},'
 151     '"version":[VERSION]'
 152     '}') % json.dumps(CHAR_TO_FAILURE)
 153
 154 JSON_RESULTS_COUNTS = '{"' + '":[[TESTDATA_COUNT]],"'.join([char for char in CHAR_TO_FAILURE.values()]) + '":[[TESTDATA_COUNT]]}'
 155
 156 JSON_RESULTS_TEMPLATE = (
 157     '{"[BUILDER_NAME]":{'
 158     '"blinkRevision":[[TESTDATA_WEBKITREVISION]],'
 159     '"buildNumbers":[[TESTDATA_BUILDNUMBERS]],'
 160     '"chromeRevision":[[TESTDATA_CHROMEREVISION]],'
 161     '"failure_map": %s,'
 162     '"num_failures_by_type":%s,'
 163     '"secondsSinceEpoch":[[TESTDATA_TIMES]],'
 164     '"tests":{[TESTDATA_TESTS]}'
 165     '},'
 166     '"version":[VERSION]'
 167     '}') % (json.dumps(CHAR_TO_FAILURE), JSON_RESULTS_COUNTS)
 168
 169 JSON_RESULTS_COUNTS_TEMPLATE = '{"' + '":[TESTDATA],"'.join([char for char in CHAR_TO_FAILURE]) + '":[TESTDATA]}'
 170
 171 JSON_RESULTS_TEST_LIST_TEMPLATE = '{"Webkit":{"tests":{[TESTDATA_TESTS]}}}'
 172
 173
 174 class MockFile(object):
 175     def __init__(self, name='results.json', data=''):
 176         self.master = 'MockMasterName'
 177         self.builder = 'MockBuilderName'
 178         self.test_type = 'MockTestType'
 179         self.name = name
 180         self.data = data
 181
 182     def save(self, data):
 183         self.data = data
 184         return True
 185
 186
 187 class JsonResultsTest(unittest.TestCase):
 188     def setUp(self):
 189         self._builder = "Webkit"
 190         self.old_log_level = logging.root.level
 191         logging.root.setLevel(logging.ERROR)
 192
 193     def tearDown(self):
 194         logging.root.setLevel(self.old_log_level)
 195
 196     # Use this to get better error messages than just string compare gives.
 197     def assert_json_equal(self, a, b):
 198         self.maxDiff = None
 199         a = json.loads(a) if isinstance(a, str) else a
 200         b = json.loads(b) if isinstance(b, str) else b
 201         self.assertEqual(a, b)
 202
 203     def test_strip_prefix_suffix(self):
 204         json = "['contents']"
 205         self.assertEqual(JsonResults._strip_prefix_suffix("ADD_RESULTS(" + json + ");"), json)
 206         self.assertEqual(JsonResults._strip_prefix_suffix(json), json)
 207
 208     def _make_test_json(self, test_data, json_string=JSON_RESULTS_TEMPLATE, builder_name="Webkit"):
 209         if not test_data:
 210             return ""
 211
 212         builds = test_data["builds"]
 213         tests = test_data["tests"]
 214         if not builds or not tests:
 215             return ""
 216
 217         counts = []
 218         build_numbers = []
 219         webkit_revision = []
 220         chrome_revision = []
 221         times = []
 222         for build in builds:
 223             counts.append(JSON_RESULTS_COUNTS_TEMPLATE.replace("[TESTDATA]", build))
 224             build_numbers.append("1000%s" % build)
 225             webkit_revision.append("2000%s" % build)
 226             chrome_revision.append("3000%s" % build)
 227             times.append("100000%s000" % build)
 228
 229         json_string = json_string.replace("[BUILDER_NAME]", builder_name)
 230         json_string = json_string.replace("[TESTDATA_COUNTS]", ",".join(counts))
 231         json_string = json_string.replace("[TESTDATA_COUNT]", ",".join(builds))
 232         json_string = json_string.replace("[TESTDATA_BUILDNUMBERS]", ",".join(build_numbers))
 233         json_string = json_string.replace("[TESTDATA_WEBKITREVISION]", ",".join(webkit_revision))
 234         json_string = json_string.replace("[TESTDATA_CHROMEREVISION]", ",".join(chrome_revision))
 235         json_string = json_string.replace("[TESTDATA_TIMES]", ",".join(times))
 236
 237         version = str(test_data["version"]) if "version" in test_data else "4"
 238         json_string = json_string.replace("[VERSION]", version)
 239         json_string = json_string.replace("{[TESTDATA_TESTS]}", json.dumps(tests, separators=(',', ':'), sort_keys=True))
 240         return json_string
 241
 242     def _test_merge(self, aggregated_data, incremental_data, expected_data, max_builds=jsonresults.JSON_RESULTS_MAX_BUILDS):
 243         aggregated_results = self._make_test_json(aggregated_data, builder_name=self._builder)
 244         incremental_json, _ = JsonResults._get_incremental_json(self._builder, self._make_test_json(incremental_data, builder_name=self._builder), is_full_results_format=False)
 245         merged_results, status_code = JsonResults.merge(self._builder, aggregated_results, incremental_json, num_runs=max_builds, sort_keys=True)
 246
 247         if expected_data:
 248             expected_results = self._make_test_json(expected_data, builder_name=self._builder)
 249             self.assert_json_equal(merged_results, expected_results)
 250             self.assertEqual(status_code, 200)
 251         else:
 252             self.assertTrue(status_code != 200)
 253
 254     def _test_get_test_list(self, input_data, expected_data):
 255         input_results = self._make_test_json(input_data)
 256         expected_results = JSON_RESULTS_TEST_LIST_TEMPLATE.replace("{[TESTDATA_TESTS]}", json.dumps(expected_data, separators=(',', ':')))
 257         actual_results = JsonResults.get_test_list(self._builder, input_results)
 258         self.assert_json_equal(actual_results, expected_results)
 259
 260     def test_update_files_empty_aggregate_data(self):
 261         small_file = MockFile(name='results-small.json')
 262         large_file = MockFile(name='results.json')
 263
 264         incremental_data = {
 265             "builds": ["2", "1"],
 266             "tests": {
 267                 "001.html": {
 268                     "results": [[200, TEXT]],
 269                     "times": [[200, 0]],
 270                 }
 271             }
 272         }
 273         incremental_string = self._make_test_json(incremental_data, builder_name=small_file.builder)
 274
 275         self.assertTrue(JsonResults.update_files(small_file.builder, incremental_string, small_file, large_file, is_full_results_format=False))
 276         self.assert_json_equal(small_file.data, incremental_string)
 277         self.assert_json_equal(large_file.data, incremental_string)
 278
 279     def test_update_files_null_incremental_data(self):
 280         small_file = MockFile(name='results-small.json')
 281         large_file = MockFile(name='results.json')
 282
 283         aggregated_data = {
 284             "builds": ["2", "1"],
 285             "tests": {
 286                 "001.html": {
 287                     "results": [[200, TEXT]],
 288                     "times": [[200, 0]],
 289                 }
 290             }
 291         }
 292         aggregated_string = self._make_test_json(aggregated_data, builder_name=small_file.builder)
 293
 294         small_file.data = large_file.data = aggregated_string
 295
 296         incremental_string = ""
 297
 298         self.assertEqual(JsonResults.update_files(small_file.builder, incremental_string, small_file, large_file, is_full_results_format=False),
 299             ('No incremental JSON data to merge.', 403))
 300         self.assert_json_equal(small_file.data, aggregated_string)
 301         self.assert_json_equal(large_file.data, aggregated_string)
 302
 303     def test_update_files_empty_incremental_data(self):
 304         small_file = MockFile(name='results-small.json')
 305         large_file = MockFile(name='results.json')
 306
 307         aggregated_data = {
 308             "builds": ["2", "1"],
 309             "tests": {
 310                 "001.html": {
 311                     "results": [[200, TEXT]],
 312                     "times": [[200, 0]],
 313                 }
 314             }
 315         }
 316         aggregated_string = self._make_test_json(aggregated_data, builder_name=small_file.builder)
 317
 318         small_file.data = large_file.data = aggregated_string
 319
 320         incremental_data = {
 321             "builds": [],
 322             "tests": {}
 323         }
 324         incremental_string = self._make_test_json(incremental_data, builder_name=small_file.builder)
 325
 326         self.assertEqual(JsonResults.update_files(small_file.builder, incremental_string, small_file, large_file, is_full_results_format=False),
 327             ('No incremental JSON data to merge.', 403))
 328         self.assert_json_equal(small_file.data, aggregated_string)
 329         self.assert_json_equal(large_file.data, aggregated_string)
 330
 331     def test_merge_with_empty_aggregated_results(self):
 332         incremental_data = {
 333             "builds": ["2", "1"],
 334             "tests": {
 335                 "001.html": {
 336                     "results": [[200, TEXT]],
 337                     "times": [[200, 0]],
 338                 }
 339             }
 340         }
 341         incremental_results, _ = JsonResults._get_incremental_json(self._builder, self._make_test_json(incremental_data), is_full_results_format=False)
 342         aggregated_results = ""
 343         merged_results, _ = JsonResults.merge(self._builder, aggregated_results, incremental_results, num_runs=jsonresults.JSON_RESULTS_MAX_BUILDS, sort_keys=True)
 344         self.assert_json_equal(merged_results, incremental_results)
 345
 346     def test_failures_by_type_added(self):
 347         aggregated_results = self._make_test_json({
 348             "builds": ["2", "1"],
 349             "tests": {
 350                 "001.html": {
 351                     "results": [[100, TEXT], [100, FAIL]],
 352                     "times": [[200, 0]],
 353                 }
 354             }
 355         }, json_string=JSON_RESULTS_OLD_TEMPLATE)
 356         incremental_results = self._make_test_json({
 357             "builds": ["3"],
 358             "tests": {
 359                 "001.html": {
 360                     "results": [[1, TEXT]],
 361                     "times": [[1, 0]],
 362                 }
 363             }
 364         }, json_string=JSON_RESULTS_OLD_TEMPLATE)
 365         incremental_json, _ = JsonResults._get_incremental_json(self._builder, incremental_results, is_full_results_format=False)
 366         merged_results, _ = JsonResults.merge(self._builder, aggregated_results, incremental_json, num_runs=201, sort_keys=True)
 367         self.assert_json_equal(merged_results, self._make_test_json({
 368             "builds": ["3", "2", "1"],
 369             "tests": {
 370                 "001.html": {
 371                     "results": [[101, TEXT], [100, FAIL]],
 372                     "times": [[201, 0]],
 373                 }
 374             }
 375         }))
 376
 377     def test_merge_full_results_format(self):
 378         expected_incremental_results = {
 379             "Webkit": {
 380                 "blinkRevision": ["1234"],
 381                 "buildNumbers": ["3"],
 382                 "chromeRevision": ["5678"],
 383                 "failure_map": CHAR_TO_FAILURE,
 384                 "num_failures_by_type": {"AUDIO": [0], "CRASH": [3], "FAIL": [2], "IMAGE": [1], "IMAGE+TEXT": [0], "MISSING": [0], "PASS": [10], "SKIP": [2], "TEXT": [3], "TIMEOUT": [16], "LEAK": [1]},
 385                 "secondsSinceEpoch": [1368146629],
 386                 "tests": {
 387                     "media": {
 388                         "W3C": {
 389                             "audio": {
 390                                 "src": {
 391                                     "src_removal_does_not_trigger_loadstart.html": {
 392                                         "results": [[1, PASS]],
 393                                         "times": [[1, 4]],
 394                                     }
 395                                 }
 396                             }
 397                         },
 398                         "encrypted-media": {
 399                             "encrypted-media-v2-events.html": {
 400                                 "bugs": ["crbug.com/1234"],
 401                                 "expected": "TIMEOUT",
 402                                 "results": [[1, TIMEOUT]],
 403                                 "times": [[1, 6]],
 404                             },
 405                             "encrypted-media-v2-syntax.html": {
 406                                 "expected": "TIMEOUT",
 407                                 "results": [[1, TIMEOUT]],
 408                                 "times": [[1, 0]],
 409                             }
 410                         },
 411                         "media-document-audio-repaint.html": {
 412                             "expected": "IMAGE",
 413                             "results": [[1, IMAGE]],
 414                             "times": [[1, 0]],
 415                         },
 416                         "progress-events-generated-correctly.html": {
 417                             "expected": "PASS FAIL IMAGE TIMEOUT CRASH MISSING",
 418                             "results": [[1, TIMEOUT]],
 419                             "times": [[1, 6]],
 420                         },
 421                         "flaky-failed.html": {
 422                             "expected": "PASS FAIL",
 423                             "results": [[1, FAIL]],
 424                             "times": [[1, 0]],
 425                         },
 426                         "unexpected-fail.html": {
 427                             "results": [[1, FAIL]],
 428                             "times": [[1, 0]],
 429                         },
 430                         "unexpected-leak.html": {
 431                             "results": [[1, LEAK]],
 432                             "times": [[1, 0]],
 433                         },
 434                     }
 435                 }
 436             },
 437             "version": 4
 438         }
 439
 440         aggregated_results = ""
 441         incremental_json, _ = JsonResults._get_incremental_json(self._builder, FULL_RESULT_EXAMPLE, is_full_results_format=True)
 442         merged_results, _ = JsonResults.merge("Webkit", aggregated_results, incremental_json, num_runs=jsonresults.JSON_RESULTS_MAX_BUILDS, sort_keys=True)
 443         self.assert_json_equal(merged_results, expected_incremental_results)
 444
 445     def test_merge_empty_aggregated_results(self):
 446         # No existing aggregated results.
 447         # Merged results == new incremental results.
 448         self._test_merge(
 449             # Aggregated results
 450             None,
 451             # Incremental results
 452             {"builds": ["2", "1"],
 453              "tests": {"001.html": {
 454                            "results": [[200, TEXT]],
 455                            "times": [[200, 0]]}}},
 456             # Expected result
 457             {"builds": ["2", "1"],
 458              "tests": {"001.html": {
 459                            "results": [[200, TEXT]],
 460                            "times": [[200, 0]]}}})
 461
 462     def test_merge_duplicate_build_number(self):
 463         self._test_merge(
 464             # Aggregated results
 465             {"builds": ["2", "1"],
 466              "tests": {"001.html": {
 467                            "results": [[100, TEXT]],
 468                            "times": [[100, 0]]}}},
 469             # Incremental results
 470             {"builds": ["2"],
 471              "tests": {"001.html": {
 472                            "results": [[1, TEXT]],
 473                            "times": [[1, 0]]}}},
 474             # Expected results
 475             None)
 476
 477     def test_merge_incremental_single_test_single_run_same_result(self):
 478         # Incremental results has the latest build and same test results for
 479         # that run.
 480         # Insert the incremental results at the first place and sum number
 481         # of runs for TEXT (200 + 1) to get merged results.
 482         self._test_merge(
 483             # Aggregated results
 484             {"builds": ["2", "1"],
 485              "tests": {"001.html": {
 486                            "results": [[200, TEXT]],
 487                            "times": [[200, 0]]}}},
 488             # Incremental results
 489             {"builds": ["3"],
 490              "tests": {"001.html": {
 491                            "results": [[1, TEXT]],
 492                            "times": [[1, 0]]}}},
 493             # Expected results
 494             {"builds": ["3", "2", "1"],
 495              "tests": {"001.html": {
 496                            "results": [[201, TEXT]],
 497                            "times": [[201, 0]]}}})
 498
 499     def test_merge_single_test_single_run_different_result(self):
 500         # Incremental results has the latest build but different test results
 501         # for that run.
 502         # Insert the incremental results at the first place.
 503         self._test_merge(
 504             # Aggregated results
 505             {"builds": ["2", "1"],
 506              "tests": {"001.html": {
 507                            "results": [[200, TEXT]],
 508                            "times": [[200, 0]]}}},
 509             # Incremental results
 510             {"builds": ["3"],
 511              "tests": {"001.html": {
 512                            "results": [[1, IMAGE]],
 513                            "times": [[1, 1]]}}},
 514             # Expected results
 515             {"builds": ["3", "2", "1"],
 516              "tests": {"001.html": {
 517                            "results": [[1, IMAGE], [200, TEXT]],
 518                            "times": [[1, 1], [200, 0]]}}})
 519
 520     def test_merge_single_test_single_run_result_changed(self):
 521         # Incremental results has the latest build but results which differ from
 522         # the latest result (but are the same as an older result).
 523         self._test_merge(
 524             # Aggregated results
 525             {"builds": ["2", "1"],
 526              "tests": {"001.html": {
 527                            "results": [[200, TEXT], [10, IMAGE]],
 528                            "times": [[200, 0], [10, 1]]}}},
 529             # Incremental results
 530             {"builds": ["3"],
 531              "tests": {"001.html": {
 532                            "results": [[1, IMAGE]],
 533                            "times": [[1, 1]]}}},
 534             # Expected results
 535             {"builds": ["3", "2", "1"],
 536              "tests": {"001.html": {
 537                            "results": [[1, IMAGE], [200, TEXT], [10, IMAGE]],
 538                            "times": [[1, 1], [200, 0], [10, 1]]}}})
 539
 540     def test_merge_multiple_tests_single_run(self):
 541         # All tests have incremental updates.
 542         self._test_merge(
 543             # Aggregated results
 544             {"builds": ["2", "1"],
 545              "tests": {"001.html": {
 546                            "results": [[200, TEXT]],
 547                            "times": [[200, 0]]},
 548                        "002.html": {
 549                            "results": [[100, IMAGE]],
 550                            "times": [[100, 1]]}}},
 551             # Incremental results
 552             {"builds": ["3"],
 553              "tests": {"001.html": {
 554                            "results": [[1, TEXT]],
 555                            "times": [[1, 0]]},
 556                        "002.html": {
 557                            "results": [[1, IMAGE]],
 558                            "times": [[1, 1]]}}},
 559             # Expected results
 560             {"builds": ["3", "2", "1"],
 561              "tests": {"001.html": {
 562                            "results": [[201, TEXT]],
 563                            "times": [[201, 0]]},
 564                        "002.html": {
 565                            "results": [[101, IMAGE]],
 566                            "times": [[101, 1]]}}})
 567
 568     def test_merge_multiple_tests_single_run_one_no_result(self):
 569         self._test_merge(
 570             # Aggregated results
 571             {"builds": ["2", "1"],
 572              "tests": {"001.html": {
 573                            "results": [[200, TEXT]],
 574                            "times": [[200, 0]]},
 575                        "002.html": {
 576                            "results": [[100, IMAGE]],
 577                            "times": [[100, 1]]}}},
 578             # Incremental results
 579             {"builds": ["3"],
 580              "tests": {"002.html": {
 581                            "results": [[1, IMAGE]],
 582                            "times": [[1, 1]]}}},
 583             # Expected results
 584             {"builds": ["3", "2", "1"],
 585              "tests": {"001.html": {
 586                            "results": [[1, NO_DATA], [200, TEXT]],
 587                            "times": [[201, 0]]},
 588                        "002.html": {
 589                            "results": [[101, IMAGE]],
 590                            "times": [[101, 1]]}}})
 591
 592     def test_merge_single_test_multiple_runs(self):
 593         self._test_merge(
 594             # Aggregated results
 595             {"builds": ["2", "1"],
 596              "tests": {"001.html": {
 597                            "results": [[200, TEXT]],
 598                            "times": [[200, 0]]}}},
 599             # Incremental results
 600             {"builds": ["4", "3"],
 601              "tests": {"001.html": {
 602                            "results": [[2, IMAGE], [1, FAIL]],
 603                            "times": [[3, 2]]}}},
 604             # Expected results
 605             {"builds": ["4", "3", "2", "1"],
 606              "tests": {"001.html": {
 607                            "results": [[1, FAIL], [2, IMAGE], [200, TEXT]],
 608                            "times": [[3, 2], [200, 0]]}}})
 609
 610     def test_merge_multiple_tests_multiple_runs(self):
 611         self._test_merge(
 612             # Aggregated results
 613             {"builds": ["2", "1"],
 614              "tests": {"001.html": {
 615                            "results": [[200, TEXT]],
 616                            "times": [[200, 0]]},
 617                        "002.html": {
 618                            "results": [[10, IMAGE_PLUS_TEXT]],
 619                            "times": [[10, 0]]}}},
 620             # Incremental results
 621             {"builds": ["4", "3"],
 622              "tests": {"001.html": {
 623                            "results": [[2, IMAGE]],
 624                            "times": [[2, 2]]},
 625                        "002.html": {
 626                            "results": [[1, CRASH]],
 627                            "times": [[1, 1]]}}},
 628             # Expected results
 629             {"builds": ["4", "3", "2", "1"],
 630              "tests": {"001.html": {
 631                            "results": [[2, IMAGE], [200, TEXT]],
 632                            "times": [[2, 2], [200, 0]]},
 633                        "002.html": {
 634                            "results": [[1, CRASH], [10, IMAGE_PLUS_TEXT]],
 635                            "times": [[1, 1], [10, 0]]}}})
 636
 637     def test_merge_incremental_result_older_build(self):
 638         # Test the build in incremental results is older than the most recent
 639         # build in aggregated results.
 640         self._test_merge(
 641             # Aggregated results
 642             {"builds": ["3", "1"],
 643              "tests": {"001.html": {
 644                            "results": [[5, TEXT]],
 645                            "times": [[5, 0]]}}},
 646             # Incremental results
 647             {"builds": ["2"],
 648              "tests": {"001.html": {
 649                            "results": [[1, TEXT]],
 650                            "times": [[1, 0]]}}},
 651             # Expected no merge happens.
 652             {"builds": ["2", "3", "1"],
 653              "tests": {"001.html": {
 654                            "results": [[6, TEXT]],
 655                            "times": [[6, 0]]}}})
 656
 657     def test_merge_incremental_result_same_build(self):
 658         # Test the build in incremental results is same as the build in
 659         # aggregated results.
 660         self._test_merge(
 661             # Aggregated results
 662             {"builds": ["2", "1"],
 663              "tests": {"001.html": {
 664                            "results": [[5, TEXT]],
 665                            "times": [[5, 0]]}}},
 666             # Incremental results
 667             {"builds": ["3", "2"],
 668              "tests": {"001.html": {
 669                            "results": [[2, TEXT]],
 670                            "times": [[2, 0]]}}},
 671             # Expected no merge happens.
 672             {"builds": ["3", "2", "2", "1"],
 673              "tests": {"001.html": {
 674                            "results": [[7, TEXT]],
 675                            "times": [[7, 0]]}}})
 676
 677     def test_merge_remove_new_test(self):
 678         self._test_merge(
 679             # Aggregated results
 680             {"builds": ["2", "1"],
 681              "tests": {"001.html": {
 682                            "results": [[199, TEXT]],
 683                            "times": [[199, 0]]},
 684                        }},
 685             # Incremental results
 686             {"builds": ["3"],
 687              "tests": {"001.html": {
 688                            "results": [[1, TEXT]],
 689                            "times": [[1, 0]]},
 690                        "002.html": {
 691                            "results": [[1, PASS]],
 692                            "times": [[1, 0]]},
 693                        "notrun.html": {
 694                            "results": [[1, NOTRUN]],
 695                            "times": [[1, 0]]},
 696                        "003.html": {
 697                            "results": [[1, NO_DATA]],
 698                            "times": [[1, 0]]},
 699                         }},
 700             # Expected results
 701             {"builds": ["3", "2", "1"],
 702              "tests": {"001.html": {
 703                            "results": [[200, TEXT]],
 704                            "times": [[200, 0]]},
 705                        }},
 706             max_builds=200)
 707
 708     def test_merge_remove_test(self):
 709         self._test_merge(
 710             # Aggregated results
 711             {
 712                 "builds": ["2", "1"],
 713                 "tests": {
 714                     "directory": {
 715                         "directory": {
 716                             "001.html": {
 717                                 "results": [[200, PASS]],
 718                                 "times": [[200, 0]]
 719                             }
 720                         }
 721                     },
 722                     "002.html": {
 723                         "results": [[10, TEXT]],
 724                         "times": [[10, 0]]
 725                     },
 726                     "003.html": {
 727                         "results": [[190, PASS], [9, NO_DATA], [1, TEXT]],
 728                         "times": [[200, 0]]
 729                     },
 730                 }
 731             },
 732             # Incremental results
 733             {
 734                 "builds": ["3"],
 735                 "tests": {
 736                     "directory": {
 737                         "directory": {
 738                             "001.html": {
 739                                 "results": [[1, PASS]],
 740                                 "times": [[1, 0]]
 741                             }
 742                         }
 743                     },
 744                     "002.html": {
 745                         "results": [[1, PASS]],
 746                         "times": [[1, 0]]
 747                     },
 748                     "003.html": {
 749                         "results": [[1, PASS]],
 750                         "times": [[1, 0]]
 751                     },
 752                 }
 753             },
 754             # Expected results
 755             {
 756                 "builds": ["3", "2", "1"],
 757                 "tests": {
 758                     "002.html": {
 759                         "results": [[1, PASS], [10, TEXT]],
 760                         "times": [[11, 0]]
 761                     }
 762                 }
 763             },
 764             max_builds=200)
 765
 766     def test_merge_updates_expected(self):
 767         self._test_merge(
 768             # Aggregated results
 769             {
 770                 "builds": ["2", "1"],
 771                 "tests": {
 772                     "directory": {
 773                         "directory": {
 774                             "001.html": {
 775                                 "expected": "FAIL",
 776                                 "results": [[200, PASS]],
 777                                 "times": [[200, 0]]
 778                             }
 779                         }
 780                     },
 781                     "002.html": {
 782                         "bugs": ["crbug.com/1234"],
 783                         "expected": "FAIL",
 784                         "results": [[10, TEXT]],
 785                         "times": [[10, 0]]
 786                     },
 787                     "003.html": {
 788                         "expected": "FAIL",
 789                         "results": [[190, PASS], [9, NO_DATA], [1, TEXT]],
 790                         "times": [[200, 0]]
 791                     },
 792                     "004.html": {
 793                         "results": [[199, PASS], [1, TEXT]],
 794                         "times": [[200, 0]]
 795                     },
 796                 }
 797             },
 798             # Incremental results
 799             {
 800                 "builds": ["3"],
 801                 "tests": {
 802                     "002.html": {
 803                         "expected": "PASS",
 804                         "results": [[1, PASS]],
 805                         "times": [[1, 0]]
 806                     },
 807                     "003.html": {
 808                         "expected": "TIMEOUT",
 809                         "results": [[1, PASS]],
 810                         "times": [[1, 0]]
 811                     },
 812                     "004.html": {
 813                         "bugs": ["crbug.com/1234"],
 814                         "results": [[1, PASS]],
 815                         "times": [[1, 0]]
 816                     },
 817                 }
 818             },
 819             # Expected results
 820             {
 821                 "builds": ["3", "2", "1"],
 822                 "tests": {
 823                     "002.html": {
 824                         "results": [[1, PASS], [10, TEXT]],
 825                         "times": [[11, 0]]
 826                     },
 827                     "003.html": {
 828                         "expected": "TIMEOUT",
 829                         "results": [[191, PASS], [9, NO_DATA]],
 830                         "times": [[200, 0]]
 831                     },
 832                     "004.html": {
 833                         "bugs": ["crbug.com/1234"],
 834                         "results": [[200, PASS]],
 835                         "times": [[200, 0]]
 836                     },
 837                 }
 838             },
 839             max_builds=200)
 840
 841
 842     def test_merge_keep_test_with_all_pass_but_slow_time(self):
 843         self._test_merge(
 844             # Aggregated results
 845             {"builds": ["2", "1"],
 846              "tests": {"001.html": {
 847                            "results": [[200, PASS]],
 848                            "times": [[200, jsonresults.JSON_RESULTS_MIN_TIME]]},
 849                        "002.html": {
 850                            "results": [[10, TEXT]],
 851                            "times": [[10, 0]]}}},
 852             # Incremental results
 853             {"builds": ["3"],
 854              "tests": {"001.html": {
 855                            "results": [[1, PASS]],
 856                            "times": [[1, 1]]},
 857                        "002.html": {
 858                            "results": [[1, PASS]],
 859                            "times": [[1, 0]]}}},
 860             # Expected results
 861             {"builds": ["3", "2", "1"],
 862              "tests": {"001.html": {
 863                            "results": [[201, PASS]],
 864                            "times": [[1, 1], [200, jsonresults.JSON_RESULTS_MIN_TIME]]},
 865                        "002.html": {
 866                            "results": [[1, PASS], [10, TEXT]],
 867                            "times": [[11, 0]]}}})
 868
 869     def test_merge_pruning_slow_tests_for_debug_builders(self):
 870         self._builder = "MockBuilder(dbg)"
 871         self._test_merge(
 872             # Aggregated results
 873             {"builds": ["2", "1"],
 874              "tests": {"001.html": {
 875                            "results": [[200, PASS]],
 876                            "times": [[200, 3 * jsonresults.JSON_RESULTS_MIN_TIME]]},
 877                        "002.html": {
 878                            "results": [[10, TEXT]],
 879                            "times": [[10, 0]]}}},
 880             # Incremental results
 881             {"builds": ["3"],
 882              "tests": {"001.html": {
 883                            "results": [[1, PASS]],
 884                            "times": [[1, 1]]},
 885                        "002.html": {
 886                            "results": [[1, PASS]],
 887                            "times": [[1, 0]]},
 888                        "003.html": {
 889                            "results": [[1, PASS]],
 890                            "times": [[1, jsonresults.JSON_RESULTS_MIN_TIME]]}}},
 891             # Expected results
 892             {"builds": ["3", "2", "1"],
 893              "tests": {"001.html": {
 894                            "results": [[201, PASS]],
 895                            "times": [[1, 1], [200, 3 * jsonresults.JSON_RESULTS_MIN_TIME]]},
 896                        "002.html": {
 897                            "results": [[1, PASS], [10, TEXT]],
 898                            "times": [[11, 0]]}}})
 899
 900     def test_merge_prune_extra_results(self):
 901         # Remove items from test results and times that exceed the max number
 902         # of builds to track.
 903         max_builds = jsonresults.JSON_RESULTS_MAX_BUILDS
 904         self._test_merge(
 905             # Aggregated results
 906             {"builds": ["2", "1"],
 907              "tests": {"001.html": {
 908                            "results": [[max_builds, TEXT], [1, IMAGE]],
 909                            "times": [[max_builds, 0], [1, 1]]}}},
 910             # Incremental results
 911             {"builds": ["3"],
 912              "tests": {"001.html": {
 913                            "results": [[1, TIMEOUT]],
 914                            "times": [[1, 1]]}}},
 915             # Expected results
 916             {"builds": ["3", "2", "1"],
 917              "tests": {"001.html": {
 918                            "results": [[1, TIMEOUT], [max_builds, TEXT]],
 919                            "times": [[1, 1], [max_builds, 0]]}}})
 920
 921     def test_merge_prune_extra_results_small(self):
 922         # Remove items from test results and times that exceed the max number
 923         # of builds to track, using smaller threshold.
 924         max_builds = jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL
 925         self._test_merge(
 926             # Aggregated results
 927             {"builds": ["2", "1"],
 928              "tests": {"001.html": {
 929                            "results": [[max_builds, TEXT], [1, IMAGE]],
 930                            "times": [[max_builds, 0], [1, 1]]}}},
 931             # Incremental results
 932             {"builds": ["3"],
 933              "tests": {"001.html": {
 934                            "results": [[1, TIMEOUT]],
 935                            "times": [[1, 1]]}}},
 936             # Expected results
 937             {"builds": ["3", "2", "1"],
 938              "tests": {"001.html": {
 939                            "results": [[1, TIMEOUT], [max_builds, TEXT]],
 940                            "times": [[1, 1], [max_builds, 0]]}}},
 941             int(max_builds))
 942
 943     def test_merge_prune_extra_results_with_new_result_of_same_type(self):
 944         # Test that merging in a new result of the same type as the last result
 945         # causes old results to fall off.
 946         max_builds = jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL
 947         self._test_merge(
 948             # Aggregated results
 949             {"builds": ["2", "1"],
 950              "tests": {"001.html": {
 951                            "results": [[max_builds, TEXT], [1, NO_DATA]],
 952                            "times": [[max_builds, 0], [1, 1]]}}},
 953             # Incremental results
 954             {"builds": ["3"],
 955              "tests": {"001.html": {
 956                            "results": [[1, TEXT]],
 957                            "times": [[1, 0]]}}},
 958             # Expected results
 959             {"builds": ["3", "2", "1"],
 960              "tests": {"001.html": {
 961                            "results": [[max_builds, TEXT]],
 962                            "times": [[max_builds, 0]]}}},
 963             int(max_builds))
 964
 965     def test_merge_build_directory_hierarchy(self):
 966         self._test_merge(
 967             # Aggregated results
 968             {"builds": ["2", "1"],
 969              "tests": {"bar": {"baz": {
 970                            "003.html": {
 971                                 "results": [[25, TEXT]],
 972                                 "times": [[25, 0]]}}},
 973                        "foo": {
 974                            "001.html": {
 975                                 "results": [[50, TEXT]],
 976                                 "times": [[50, 0]]},
 977                            "002.html": {
 978                                 "results": [[100, IMAGE]],
 979                                 "times": [[100, 0]]}}},
 980               "version": 4},
 981             # Incremental results
 982             {"builds": ["3"],
 983              "tests": {"baz": {
 984                            "004.html": {
 985                                "results": [[1, IMAGE]],
 986                                "times": [[1, 0]]}},
 987                        "foo": {
 988                            "001.html": {
 989                                "results": [[1, TEXT]],
 990                                "times": [[1, 0]]},
 991                            "002.html": {
 992                                "results": [[1, IMAGE]],
 993                                "times": [[1, 0]]}}},
 994              "version": 4},
 995             # Expected results
 996             {"builds": ["3", "2", "1"],
 997              "tests": {"bar": {"baz": {
 998                            "003.html": {
 999                                "results": [[1, NO_DATA], [25, TEXT]],
1000                                "times": [[26, 0]]}}},
1001                        "baz": {
1002                            "004.html": {
1003                                "results": [[1, IMAGE]],
1004                                "times": [[1, 0]]}},
1005                        "foo": {
1006                            "001.html": {
1007                                "results": [[51, TEXT]],
1008                                "times": [[51, 0]]},
1009                            "002.html": {
1010                                "results": [[101, IMAGE]],
1011                                "times": [[101, 0]]}}},
1012              "version": 4})
1013
1014     # FIXME(aboxhall): Add some tests for xhtml/svg test results.
1015
1016     def test_get_test_name_list(self):
1017         # Get test name list only. Don't include non-test-list data and
1018         # of test result details.
1019         # FIXME: This also tests a temporary bug in the data where directory-level
1020         # results have a results and times values. Once that bug is fixed,
1021         # remove this test-case and assert we don't ever hit it.
1022         self._test_get_test_list(
1023             # Input results
1024             {"builds": ["3", "2", "1"],
1025              "tests": {"foo": {
1026                            "001.html": {
1027                                "results": [[200, PASS]],
1028                                "times": [[200, 0]]},
1029                            "results": [[1, NO_DATA]],
1030                            "times": [[1, 0]]},
1031                        "002.html": {
1032                            "results": [[10, TEXT]],
1033                            "times": [[10, 0]]}}},
1034             # Expected results
1035             {"foo": {"001.html": {}}, "002.html": {}})
1036
1037     def test_gtest(self):
1038         self._test_merge(
1039             # Aggregated results
1040             {"builds": ["2", "1"],
1041              "tests": {"foo.bar": {
1042                            "results": [[50, TEXT]],
1043                            "times": [[50, 0]]},
1044                        "foo.bar2": {
1045                            "results": [[100, IMAGE]],
1046                            "times": [[100, 0]]},
1047                        "test.failed": {
1048                            "results": [[5, FAIL]],
1049                            "times": [[5, 0]]},
1050                        },
1051              "version": 3},
1052             # Incremental results
1053             {"builds": ["3"],
1054              "tests": {"foo.bar2": {
1055                            "results": [[1, IMAGE]],
1056                            "times": [[1, 0]]},
1057                        "foo.bar3": {
1058                            "results": [[1, TEXT]],
1059                            "times": [[1, 0]]},
1060                        "test.failed": {
1061                            "results": [[5, FAIL]],
1062                            "times": [[5, 0]]},
1063                        },
1064              "version": 4},
1065             # Expected results
1066             {"builds": ["3", "2", "1"],
1067              "tests": {"foo.bar": {
1068                            "results": [[1, NO_DATA], [50, TEXT]],
1069                            "times": [[51, 0]]},
1070                        "foo.bar2": {
1071                            "results": [[101, IMAGE]],
1072                            "times": [[101, 0]]},
1073                        "foo.bar3": {
1074                            "results": [[1, TEXT]],
1075                            "times": [[1, 0]]},
1076                        "test.failed": {
1077                            "results": [[10, FAIL]],
1078                            "times": [[10, 0]]},
1079                        },
1080              "version": 4})
1081
1082 if __name__ == '__main__':
1083     unittest.main()