Tools/TestResultServer/model/jsonresults_unittest.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #     * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #     * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #     * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 try:
  30     import jsonresults
  31     from jsonresults import JsonResults
  32 except ImportError:
  33     print "ERROR: Add the TestResultServer, google_appengine and yaml/lib directories to your PYTHONPATH"
  34     raise
  35
  36 from django.utils import simplejson
  37
  38 import unittest
  39
  40
  41 JSON_RESULTS_TEMPLATE = (
  42     '{"Webkit":{'
  43     '"allFixableCount":[[TESTDATA_COUNT]],'
  44     '"buildNumbers":[[TESTDATA_BUILDNUMBERS]],'
  45     '"chromeRevision":[[TESTDATA_CHROMEREVISION]],'
  46     '"deferredCounts":[[TESTDATA_COUNTS]],'
  47     '"fixableCount":[[TESTDATA_COUNT]],'
  48     '"fixableCounts":[[TESTDATA_COUNTS]],'
  49     '"secondsSinceEpoch":[[TESTDATA_TIMES]],'
  50     '"tests":{[TESTDATA_TESTS]},'
  51     '"webkitRevision":[[TESTDATA_WEBKITREVISION]],'
  52     '"wontfixCounts":[[TESTDATA_COUNTS]]'
  53     '},'
  54     '"version":[VERSION]'
  55     '}')
  56
  57 JSON_RESULTS_COUNTS_TEMPLATE = (
  58     '{'
  59     '"C":[TESTDATA],'
  60     '"F":[TESTDATA],'
  61     '"I":[TESTDATA],'
  62     '"O":[TESTDATA],'
  63     '"P":[TESTDATA],'
  64     '"T":[TESTDATA],'
  65     '"X":[TESTDATA],'
  66     '"Z":[TESTDATA]}')
  67
  68 JSON_RESULTS_DIRECTORY_TEMPLATE = '[[TESTDATA_DIRECTORY]]:{[TESTDATA_DATA]}'
  69
  70 JSON_RESULTS_TESTS_TEMPLATE = (
  71     '[[TESTDATA_TEST_NAME]]:{'
  72     '"results":[[TESTDATA_TEST_RESULTS]],'
  73     '"times":[[TESTDATA_TEST_TIMES]]}')
  74
  75 JSON_RESULTS_TEST_LIST_TEMPLATE = (
  76     '{"Webkit":{"tests":{[TESTDATA_TESTS]}}}')
  77
  78
  79 class JsonResultsTest(unittest.TestCase):
  80     def setUp(self):
  81         self._builder = "Webkit"
  82
  83     def test_strip_prefix_suffix(self):
  84         json = "['contents']"
  85         self.assertEqual(JsonResults._strip_prefix_suffix("ADD_RESULTS(" + json + ");"), json)
  86         self.assertEqual(JsonResults._strip_prefix_suffix(json), json)
  87
  88     def _make_test_json(self, test_data):
  89         if not test_data:
  90             return ""
  91
  92         builds = test_data["builds"]
  93         tests = test_data["tests"]
  94         if not builds or not tests:
  95             return ""
  96
  97         json = JSON_RESULTS_TEMPLATE
  98
  99         counts = []
 100         build_numbers = []
 101         webkit_revision = []
 102         chrome_revision = []
 103         times = []
 104         for build in builds:
 105             counts.append(JSON_RESULTS_COUNTS_TEMPLATE.replace("[TESTDATA]", build))
 106             build_numbers.append("1000%s" % build)
 107             webkit_revision.append("2000%s" % build)
 108             chrome_revision.append("3000%s" % build)
 109             times.append("100000%s000" % build)
 110
 111         json = json.replace("[TESTDATA_COUNTS]", ",".join(counts))
 112         json = json.replace("[TESTDATA_COUNT]", ",".join(builds))
 113         json = json.replace("[TESTDATA_BUILDNUMBERS]", ",".join(build_numbers))
 114         json = json.replace("[TESTDATA_WEBKITREVISION]", ",".join(webkit_revision))
 115         json = json.replace("[TESTDATA_CHROMEREVISION]", ",".join(chrome_revision))
 116         json = json.replace("[TESTDATA_TIMES]", ",".join(times))
 117
 118         version = str(test_data["version"]) if "version" in test_data else "4"
 119         json = json.replace("[VERSION]", version)
 120         json = json.replace("{[TESTDATA_TESTS]}", simplejson.dumps(tests, separators=(',', ':'), sort_keys=True))
 121         return json
 122
 123     def _test_merge(self, aggregated_data, incremental_data, expected_data, max_builds=jsonresults.JSON_RESULTS_MAX_BUILDS):
 124         aggregated_results = self._make_test_json(aggregated_data)
 125         incremental_results = self._make_test_json(incremental_data)
 126         merged_results = JsonResults.merge(self._builder, aggregated_results, incremental_results, max_builds, sort_keys=True)
 127
 128         if expected_data:
 129             expected_results = self._make_test_json(expected_data)
 130             self.assertEquals(merged_results, expected_results)
 131         else:
 132             self.assertFalse(merged_results)
 133
 134     def _test_get_test_list(self, input_data, expected_data):
 135         input_results = self._make_test_json(input_data)
 136         expected_results = JSON_RESULTS_TEST_LIST_TEMPLATE.replace("{[TESTDATA_TESTS]}", simplejson.dumps(expected_data, separators=(',', ':')))
 137         actual_results = JsonResults.get_test_list(self._builder, input_results)
 138         self.assertEquals(actual_results, expected_results)
 139
 140     def test_merge_null_incremental_results(self):
 141         # Empty incremental results json.
 142         # Nothing to merge.
 143         self._test_merge(
 144             # Aggregated results
 145             {"builds": ["2", "1"],
 146              "tests": {"001.html": {
 147                            "results": [[200,"F"]],
 148                            "times": [[200,0]]}}},
 149             # Incremental results
 150             None,
 151             # Expect no merge happens.
 152             None)
 153
 154     def test_merge_empty_incremental_results(self):
 155         # No actual incremental test results (only prefix and suffix) to merge.
 156         # Nothing to merge.
 157         self._test_merge(
 158             # Aggregated results
 159             {"builds": ["2", "1"],
 160              "tests": {"001.html": {
 161                            "results": [[200,"F"]],
 162                            "times": [[200,0]]}}},
 163             # Incremental results
 164             {"builds": [],
 165              "tests": {}},
 166             # Expected no merge happens.
 167             None)
 168
 169     def test_merge_empty_aggregated_results(self):
 170         # No existing aggregated results.
 171         # Merged results == new incremental results.
 172         self._test_merge(
 173             # Aggregated results
 174             None,
 175             # Incremental results
 176
 177             {"builds": ["2", "1"],
 178              "tests": {"001.html": {
 179                            "results": [[200,"F"]],
 180                            "times": [[200,0]]}}},
 181             # Expected result
 182             {"builds": ["2", "1"],
 183              "tests": {"001.html": {
 184                            "results": [[200,"F"]],
 185                            "times": [[200,0]]}}})
 186
 187     def test_merge_incremental_single_test_single_run_same_result(self):
 188         # Incremental results has the latest build and same test results for
 189         # that run.
 190         # Insert the incremental results at the first place and sum number
 191         # of runs for "F" (200 + 1) to get merged results.
 192         self._test_merge(
 193             # Aggregated results
 194             {"builds": ["2", "1"],
 195              "tests": {"001.html": {
 196                            "results": [[200,"F"]],
 197                            "times": [[200,0]]}}},
 198             # Incremental results
 199             {"builds": ["3"],
 200              "tests": {"001.html": {
 201                            "results": [[1,"F"]],
 202                            "times": [[1,0]]}}},
 203             # Expected results
 204             {"builds": ["3", "2", "1"],
 205              "tests": {"001.html": {
 206                            "results": [[201,"F"]],
 207                            "times": [[201,0]]}}})
 208
 209     def test_merge_single_test_single_run_different_result(self):
 210         # Incremental results has the latest build but different test results
 211         # for that run.
 212         # Insert the incremental results at the first place.
 213         self._test_merge(
 214             # Aggregated results
 215             {"builds": ["2", "1"],
 216              "tests": {"001.html": {
 217                            "results": [[200,"F"]],
 218                            "times": [[200,0]]}}},
 219             # Incremental results
 220             {"builds": ["3"],
 221              "tests": {"001.html": {
 222                            "results": [[1, "I"]],
 223                            "times": [[1,1]]}}},
 224             # Expected results
 225             {"builds": ["3", "2", "1"],
 226              "tests": {"001.html": {
 227                            "results": [[1,"I"],[200,"F"]],
 228                            "times": [[1,1],[200,0]]}}})
 229
 230     def test_merge_single_test_single_run_result_changed(self):
 231         # Incremental results has the latest build but results which differ from
 232         # the latest result (but are the same as an older result).
 233         self._test_merge(
 234             # Aggregated results
 235             {"builds": ["2", "1"],
 236              "tests": {"001.html": {
 237                            "results": [[200,"F"],[10,"I"]],
 238                            "times": [[200,0],[10,1]]}}},
 239             # Incremental results
 240             {"builds": ["3"],
 241              "tests": {"001.html": {
 242                            "results": [[1,"I"]],
 243                            "times": [[1,1]]}}},
 244             # Expected results
 245             {"builds": ["3", "2", "1"],
 246              "tests": {"001.html": {
 247                            "results": [[1,"I"],[200,"F"],[10,"I"]],
 248                            "times": [[1,1],[200,0],[10,1]]}}})
 249
 250     def test_merge_multiple_tests_single_run(self):
 251         # All tests have incremental updates.
 252         self._test_merge(
 253             # Aggregated results
 254             {"builds": ["2", "1"],
 255              "tests": {"001.html": {
 256                            "results": [[200,"F"]],
 257                            "times": [[200,0]]},
 258                        "002.html": {
 259                            "results": [[100,"I"]],
 260                            "times": [[100,1]]}}},
 261             # Incremental results
 262             {"builds": ["3"],
 263              "tests": {"001.html": {
 264                            "results": [[1,"F"]],
 265                            "times": [[1,0]]},
 266                        "002.html": {
 267                            "results": [[1,"I"]],
 268                            "times": [[1,1]]}}},
 269             # Expected results
 270             {"builds": ["3", "2", "1"],
 271              "tests": {"001.html": {
 272                            "results": [[201,"F"]],
 273                            "times": [[201,0]]},
 274                        "002.html": {
 275                            "results": [[101,"I"]],
 276                            "times": [[101,1]]}}})
 277
 278     def test_merge_multiple_tests_single_run_one_no_result(self):
 279         self._test_merge(
 280             # Aggregated results
 281             {"builds": ["2", "1"],
 282              "tests": {"001.html": {
 283                            "results": [[200,"F"]],
 284                            "times": [[200,0]]},
 285                        "002.html": {
 286                            "results": [[100,"I"]],
 287                            "times": [[100,1]]}}},
 288             # Incremental results
 289             {"builds": ["3"],
 290              "tests": {"002.html": {
 291                            "results": [[1,"I"]],
 292                            "times": [[1,1]]}}},
 293             # Expected results
 294             {"builds": ["3", "2", "1"],
 295              "tests": {"001.html": {
 296                            "results": [[1,"N"],[200,"F"]],
 297                            "times": [[201,0]]},
 298                        "002.html": {
 299                            "results": [[101,"I"]],
 300                            "times": [[101,1]]}}})
 301
 302     def test_merge_single_test_multiple_runs(self):
 303         self._test_merge(
 304             # Aggregated results
 305             {"builds": ["2", "1"],
 306              "tests": {"001.html": {
 307                            "results": [[200,"F"]],
 308                            "times": [[200,0]]}}},
 309             # Incremental results
 310             {"builds": ["4", "3"],
 311              "tests": {"001.html": {
 312                            "results": [[2, "I"]],
 313                            "times": [[2,2]]}}},
 314             # Expected results
 315             {"builds": ["4", "3", "2", "1"],
 316              "tests": {"001.html": {
 317                            "results": [[2,"I"],[200,"F"]],
 318                            "times": [[2,2],[200,0]]}}})
 319
 320     def test_merge_multiple_tests_multiple_runs(self):
 321         self._test_merge(
 322             # Aggregated results
 323             {"builds": ["2", "1"],
 324              "tests": {"001.html": {
 325                            "results": [[200,"F"]],
 326                            "times": [[200,0]]},
 327                        "002.html": {
 328                            "results": [[10,"Z"]],
 329                            "times": [[10,0]]}}},
 330             # Incremental results
 331             {"builds": ["4", "3"],
 332              "tests": {"001.html": {
 333                            "results": [[2, "I"]],
 334                            "times": [[2,2]]},
 335                        "002.html": {
 336                            "results": [[1,"C"]],
 337                            "times": [[1,1]]}}},
 338             # Expected results
 339             {"builds": ["4", "3", "2", "1"],
 340              "tests": {"001.html": {
 341                            "results": [[2,"I"],[200,"F"]],
 342                            "times": [[2,2],[200,0]]},
 343                        "002.html": {
 344                            "results": [[1,"C"],[10,"Z"]],
 345                            "times": [[1,1],[10,0]]}}})
 346
 347     def test_merge_incremental_result_older_build(self):
 348         # Test the build in incremental results is older than the most recent
 349         # build in aggregated results.
 350         self._test_merge(
 351             # Aggregated results
 352             {"builds": ["3", "1"],
 353              "tests": {"001.html": {
 354                            "results": [[5,"F"]],
 355                            "times": [[5,0]]}}},
 356             # Incremental results
 357             {"builds": ["2"],
 358              "tests": {"001.html": {
 359                            "results": [[1, "F"]],
 360                            "times": [[1,0]]}}},
 361             # Expected no merge happens.
 362             {"builds": ["2", "3", "1"],
 363              "tests": {"001.html": {
 364                            "results": [[6,"F"]],
 365                            "times": [[6,0]]}}})
 366
 367     def test_merge_incremental_result_same_build(self):
 368         # Test the build in incremental results is same as the build in
 369         # aggregated results.
 370         self._test_merge(
 371             # Aggregated results
 372             {"builds": ["2", "1"],
 373              "tests": {"001.html": {
 374                            "results": [[5,"F"]],
 375                            "times": [[5,0]]}}},
 376             # Incremental results
 377             {"builds": ["3", "2"],
 378              "tests": {"001.html": {
 379                            "results": [[2, "F"]],
 380                            "times": [[2,0]]}}},
 381             # Expected no merge happens.
 382             {"builds": ["3", "2", "2", "1"],
 383              "tests": {"001.html": {
 384                            "results": [[7,"F"]],
 385                            "times": [[7,0]]}}})
 386
 387     def test_merge_remove_test_with_no_data(self):
 388         # Remove test where there is no data in all runs.
 389         self._test_merge(
 390             # Aggregated results
 391             {"builds": ["2", "1"],
 392              "tests": {"001.html": {
 393                            "results": [[200,"N"]],
 394                            "times": [[200,0]]},
 395                        "002.html": {
 396                            "results": [[10,"F"]],
 397                            "times": [[10,0]]}}},
 398             # Incremental results
 399             {"builds": ["3"],
 400              "tests": {"001.html": {
 401                            "results": [[1,"N"]],
 402                            "times": [[1,0]]},
 403                        "002.html": {
 404                            "results": [[1,"P"]],
 405                            "times": [[1,0]]}}},
 406             # Expected results
 407             {"builds": ["3", "2", "1"],
 408              "tests": {"002.html": {
 409                            "results": [[1,"P"],[10,"F"]],
 410                            "times": [[11,0]]}}})
 411
 412     def test_merge_remove_test_with_all_pass(self):
 413         # Remove test where all run pass and max running time < 1 seconds
 414         self._test_merge(
 415             # Aggregated results
 416             {"builds": ["2", "1"],
 417              "tests": {"001.html": {
 418                            "results": [[200,"P"]],
 419                            "times": [[200,0]]},
 420                        "002.html": {
 421                            "results": [[10,"F"]],
 422                            "times": [[10,0]]}}},
 423             # Incremental results
 424             {"builds": ["3"],
 425              "tests": {"001.html": {
 426                            "results": [[1,"P"]],
 427                            "times": [[1,0]]},
 428                        "002.html": {
 429                            "results": [[1,"P"]],
 430                            "times": [[1,0]]}}},
 431             # Expected results
 432             {"builds": ["3", "2", "1"],
 433              "tests": {"002.html": {
 434                            "results": [[1,"P"],[10,"F"]],
 435                            "times": [[11,0]]}}})
 436
 437     def test_merge_keep_test_with_all_pass_but_slow_time(self):
 438         # Do not remove test where all run pass but max running time >= 1 seconds
 439         self._test_merge(
 440             # Aggregated results
 441             {"builds": ["2", "1"],
 442              "tests": {"001.html": {
 443                            "results": [[200,"P"]],
 444                            "times": [[200,0]]},
 445                        "002.html": {
 446                            "results": [[10,"F"]],
 447                            "times": [[10,0]]}}},
 448             # Incremental results
 449             {"builds": ["3"],
 450              "tests": {"001.html": {
 451                            "results": [[1,"P"]],
 452                            "times": [[1,1]]},
 453                        "002.html": {
 454                            "results": [[1,"P"]],
 455                            "times": [[1,0]]}}},
 456             # Expected results
 457             {"builds": ["3", "2", "1"],
 458              "tests": {"001.html": {
 459                            "results": [[201,"P"]],
 460                            "times": [[1,1],[200,0]]},
 461                        "002.html": {
 462                            "results": [[1,"P"],[10,"F"]],
 463                            "times": [[11,0]]}}})
 464
 465     def test_merge_prune_extra_results(self):
 466         # Remove items from test results and times that exceed the max number
 467         # of builds to track.
 468         max_builds = jsonresults.JSON_RESULTS_MAX_BUILDS
 469         self._test_merge(
 470             # Aggregated results
 471             {"builds": ["2", "1"],
 472              "tests": {"001.html": {
 473                            "results": [[max_builds,"F"],[1,"I"]],
 474                            "times": [[max_builds,0],[1,1]]}}},
 475             # Incremental results
 476             {"builds": ["3"],
 477              "tests": {"001.html": {
 478                            "results": [[1,"T"]],
 479                            "times": [[1,1]]}}},
 480             # Expected results
 481             {"builds": ["3", "2", "1"],
 482              "tests": {"001.html": {
 483                            "results": [[1,"T"],[max_builds,"F"]],
 484                            "times": [[1,1],[max_builds,0]]}}})
 485
 486     def test_merge_prune_extra_results_small(self):
 487         # Remove items from test results and times that exceed the max number
 488         # of builds to track, using smaller threshold.
 489         max_builds = jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL
 490         self._test_merge(
 491             # Aggregated results
 492             {"builds": ["2", "1"],
 493              "tests": {"001.html": {
 494                            "results": [[max_builds,"F"],[1,"I"]],
 495                            "times": [[max_builds,0],[1,1]]}}},
 496             # Incremental results
 497             {"builds": ["3"],
 498              "tests": {"001.html": {
 499                            "results": [[1,"T"]],
 500                            "times": [[1,1]]}}},
 501             # Expected results
 502             {"builds": ["3", "2", "1"],
 503              "tests": {"001.html": {
 504                            "results": [[1,"T"],[max_builds,"F"]],
 505                            "times": [[1,1],[max_builds,0]]}}},
 506             int(max_builds))
 507
 508     def test_merge_prune_extra_results_with_new_result_of_same_type(self):
 509         # Test that merging in a new result of the same type as the last result
 510         # causes old results to fall off.
 511         max_builds = jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL
 512         self._test_merge(
 513             # Aggregated results
 514             {"builds": ["2", "1"],
 515              "tests": {"001.html": {
 516                            "results": [[max_builds,"F"],[1,"N"]],
 517                            "times": [[max_builds,0],[1,1]]}}},
 518             # Incremental results
 519             {"builds": ["3"],
 520              "tests": {"001.html": {
 521                            "results": [[1,"F"]],
 522                            "times": [[1,0]]}}},
 523             # Expected results
 524             {"builds": ["3", "2", "1"],
 525              "tests": {"001.html": {
 526                            "results": [[max_builds,"F"]],
 527                            "times": [[max_builds,0]]}}},
 528             int(max_builds))
 529
 530     def test_merge_build_directory_hierarchy_old_version(self):
 531         self._test_merge(
 532             # Aggregated results
 533             {"builds": ["2", "1"],
 534              "tests": {"bar/003.html": {
 535                            "results": [[25,"F"]],
 536                            "times": [[25,0]]},
 537                        "foo/001.html": {
 538                            "results": [[50,"F"]],
 539                            "times": [[50,0]]},
 540                        "foo/002.html": {
 541                            "results": [[100,"I"]],
 542                            "times": [[100,0]]}},
 543              "version": 3},
 544             # Incremental results
 545             {"builds": ["3"],
 546              "tests": {"baz": {
 547                            "004.html": {
 548                                "results": [[1,"I"]],
 549                                "times": [[1,0]]}},
 550                        "foo": {
 551                            "001.html": {
 552                                "results": [[1,"F"]],
 553                                "times": [[1,0]]},
 554                            "002.html": {
 555                                "results": [[1,"I"]],
 556                                "times": [[1,0]]}}},
 557              "version": 4},
 558             # Expected results
 559             {"builds": ["3", "2", "1"],
 560              "tests": {"bar": {
 561                            "003.html": {
 562                                "results": [[1,"N"],[25,"F"]],
 563                                "times": [[26,0]]}},
 564                        "baz": {
 565                            "004.html": {
 566                                "results": [[1,"I"]],
 567                                "times": [[1,0]]}},
 568                        "foo": {
 569                            "001.html": {
 570                                "results": [[51,"F"]],
 571                                "times": [[51,0]]},
 572                            "002.html": {
 573                                "results": [[101,"I"]],
 574                                "times": [[101,0]]}}},
 575              "version": 4})
 576
 577     # FIXME: Some data got corrupted and has results and times at the directory level.
 578     # Once we've purged this from all the data, we should throw an error on this case.
 579     def test_merge_directory_hierarchy_extra_results_and_times(self):
 580         self._test_merge(
 581             # Aggregated results
 582             {"builds": ["2", "1"],
 583              "tests": {"baz": {
 584                             "003.html": {
 585                                 "results": [[25,"F"]],
 586                                 "times": [[25,0]]}},
 587                         "results": [[25,"F"]],
 588                         "times": [[25,0]]}},
 589              # Incremental results
 590              {"builds": ["3"],
 591              "tests": {"baz": {
 592                             "003.html": {
 593                                 "results": [[1,"F"]],
 594                                 "times": [[1,0]]}}}},
 595              # Expected results
 596              {"builds": ["3", "2", "1"],
 597              "tests": {"baz": {
 598                             "003.html": {
 599                                 "results": [[26,"F"]],
 600                                 "times": [[26,0]]}}},
 601               "version": 4})
 602
 603     def test_merge_build_directory_hierarchy(self):
 604         self._test_merge(
 605             # Aggregated results
 606             {"builds": ["2", "1"],
 607              "tests": {"bar": {"baz": {
 608                            "003.html": {
 609                                 "results": [[25,"F"]],
 610                                 "times": [[25,0]]}}},
 611                        "foo": {
 612                            "001.html": {
 613                                 "results": [[50,"F"]],
 614                                 "times": [[50,0]]},
 615                            "002.html": {
 616                                 "results": [[100,"I"]],
 617                                 "times": [[100,0]]}}},
 618               "version": 4},
 619             # Incremental results
 620             {"builds": ["3"],
 621              "tests": {"baz": {
 622                            "004.html": {
 623                                "results": [[1,"I"]],
 624                                "times": [[1,0]]}},
 625                        "foo": {
 626                            "001.html": {
 627                                "results": [[1,"F"]],
 628                                "times": [[1,0]]},
 629                            "002.html": {
 630                                "results": [[1,"I"]],
 631                                "times": [[1,0]]}}},
 632              "version": 4},
 633             # Expected results
 634             {"builds": ["3", "2", "1"],
 635              "tests": {"bar": {"baz": {
 636                            "003.html": {
 637                                "results": [[1,"N"],[25,"F"]],
 638                                "times": [[26,0]]}}},
 639                        "baz": {
 640                            "004.html": {
 641                                "results": [[1,"I"]],
 642                                "times": [[1,0]]}},
 643                        "foo": {
 644                            "001.html": {
 645                                "results": [[51,"F"]],
 646                                "times": [[51,0]]},
 647                            "002.html": {
 648                                "results": [[101,"I"]],
 649                                "times": [[101,0]]}}},
 650              "version": 4})
 651
 652     # FIXME(aboxhall): Add some tests for xhtml/svg test results.
 653
 654     def test_get_test_name_list(self):
 655         # Get test name list only. Don't include non-test-list data and
 656         # of test result details.
 657         # FIXME: This also tests a temporary bug in the data where directory-level
 658         # results have a results and times values. Once that bug is fixed,
 659         # remove this test-case and assert we don't ever hit it.
 660         self._test_get_test_list(
 661             # Input results
 662             {"builds": ["3", "2", "1"],
 663              "tests": {"foo": {
 664                            "001.html": {
 665                                "results": [[200,"P"]],
 666                                "times": [[200,0]]},
 667                            "results": [[1,"N"]],
 668                            "times": [[1,0]]},
 669                        "002.html": {
 670                            "results": [[10,"F"]],
 671                            "times": [[10,0]]}}},
 672             # Expected results
 673             {"foo": {"001.html":{}}, "002.html":{}})
 674
 675     def test_remove_gtest_modifiers(self):
 676         self._test_merge(
 677             # Aggregated results
 678             {"builds": ["2", "1"],
 679              "tests": {"foo.bar": {
 680                            "results": [[50,"F"]],
 681                            "times": [[50,0]]},
 682                        "foo.bar2": {
 683                            "results": [[100,"I"]],
 684                            "times": [[100,0]]},
 685                        "foo.FAILS_bar3": {
 686                            "results": [[100,"I"]],
 687                            "times": [[100,0]]},
 688                        },
 689              "version": 3},
 690             # Incremental results
 691             {"builds": ["3"],
 692              "tests": {"foo.DISABLED_bar": {
 693                            "results": [[1,"F"]],
 694                            "times": [[1,0]]},
 695                        "foo.FLAKY_bar2": {
 696                            "results": [[1,"N"]],
 697                            "times": [[1,0]]},
 698                        "foo.bar2": {
 699                            "results": [[1,"I"]],
 700                            "times": [[1,0]]},
 701                        "foo.bar3": {
 702                            "results": [[1,"N"]],
 703                            "times": [[1,0]]},
 704                        "foo.FAILS_bar3": {
 705                            "results": [[1,"I"]],
 706                            "times": [[1,0]]},
 707                        "foo.MAYBE_bar4": {
 708                            "results": [[1,"I"]],
 709                            "times": [[1,0]]}},
 710              "version": 4},
 711             # Expected results
 712             {"builds": ["3", "2", "1"],
 713              "tests": {"foo.FAILS_bar3": {
 714                            "results": [[1,"N"],[100,"I"]],
 715                            "times": [[101,0]]},
 716                        "foo.bar": {
 717                            "results": [[51,"F"]],
 718                            "times": [[51,0]]},
 719                        "foo.bar2": {
 720                            "results": [[101,"I"]],
 721                            "times": [[101,0]]},
 722                        "foo.bar3": {
 723                            "results": [[1,"I"]],
 724                            "times": [[1,0]]},
 725                        "foo.bar4": {
 726                            "results": [[1,"I"]],
 727                            "times": [[1,0]]}},
 728              "version": 4})
 729
 730 if __name__ == '__main__':
 731     unittest.main()