src/third_party/WebKit/Tools/TestResultServer/model/jsonresults.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #     * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #     * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #     * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 import collections
  30 import json
  31 import logging
  32 import re
  33 import sys
  34 import traceback
  35
  36 from testfile import TestFile
  37
  38 JSON_RESULTS_FILE = "results.json"
  39 JSON_RESULTS_FILE_SMALL = "results-small.json"
  40 JSON_RESULTS_PREFIX = "ADD_RESULTS("
  41 JSON_RESULTS_SUFFIX = ");"
  42
  43 JSON_RESULTS_MIN_TIME = 3
  44 JSON_RESULTS_HIERARCHICAL_VERSION = 4
  45 JSON_RESULTS_MAX_BUILDS = 500
  46 JSON_RESULTS_MAX_BUILDS_SMALL = 100
  47
  48 ACTUAL_KEY = "actual"
  49 BUG_KEY = "bugs"
  50 BUILD_NUMBERS_KEY = "buildNumbers"
  51 BUILDER_NAME_KEY = "builder_name"
  52 EXPECTED_KEY = "expected"
  53 FAILURE_MAP_KEY = "failure_map"
  54 FAILURES_BY_TYPE_KEY = "num_failures_by_type"
  55 FIXABLE_COUNTS_KEY = "fixableCounts"
  56 RESULTS_KEY = "results"
  57 TESTS_KEY = "tests"
  58 TIME_KEY = "time"
  59 TIMES_KEY = "times"
  60 VERSIONS_KEY = "version"
  61
  62 AUDIO = "A"
  63 CRASH = "C"
  64 FAIL = "Q"
  65 # This is only output by gtests.
  66 FLAKY = "L"
  67 IMAGE = "I"
  68 IMAGE_PLUS_TEXT = "Z"
  69 MISSING = "O"
  70 NO_DATA = "N"
  71 NOTRUN = "Y"
  72 PASS = "P"
  73 SKIP = "X"
  74 TEXT = "F"
  75 TIMEOUT = "T"
  76 LEAK = "K"
  77
  78 AUDIO_STRING = "AUDIO"
  79 CRASH_STRING = "CRASH"
  80 IMAGE_PLUS_TEXT_STRING = "IMAGE+TEXT"
  81 IMAGE_STRING = "IMAGE"
  82 FAIL_STRING = "FAIL"
  83 FLAKY_STRING = "FLAKY"
  84 MISSING_STRING = "MISSING"
  85 NO_DATA_STRING = "NO DATA"
  86 NOTRUN_STRING = "NOTRUN"
  87 PASS_STRING = "PASS"
  88 SKIP_STRING = "SKIP"
  89 TEXT_STRING = "TEXT"
  90 TIMEOUT_STRING = "TIMEOUT"
  91 LEAK_STRING = "LEAK"
  92
  93 FAILURE_TO_CHAR = {
  94     AUDIO_STRING: AUDIO,
  95     CRASH_STRING: CRASH,
  96     IMAGE_PLUS_TEXT_STRING: IMAGE_PLUS_TEXT,
  97     IMAGE_STRING: IMAGE,
  98     FLAKY_STRING: FLAKY,
  99     FAIL_STRING: FAIL,
 100     MISSING_STRING: MISSING,
 101     NO_DATA_STRING: NO_DATA,
 102     NOTRUN_STRING: NOTRUN,
 103     PASS_STRING: PASS,
 104     SKIP_STRING: SKIP,
 105     TEXT_STRING: TEXT,
 106     TIMEOUT_STRING: TIMEOUT,
 107     LEAK_STRING: LEAK,
 108 }
 109
 110 # FIXME: Use dict comprehensions once we update the server to python 2.7.
 111 CHAR_TO_FAILURE = dict((value, key) for key, value in FAILURE_TO_CHAR.items())
 112
 113 def _is_directory(subtree):
 114     return RESULTS_KEY not in subtree or not isinstance(subtree[RESULTS_KEY], collections.Sequence)
 115
 116
 117 class JsonResults(object):
 118     @staticmethod
 119     def is_aggregate_file(name):
 120         return name in (JSON_RESULTS_FILE, JSON_RESULTS_FILE_SMALL)
 121
 122     @classmethod
 123     def _strip_prefix_suffix(cls, data):
 124         if data.startswith(JSON_RESULTS_PREFIX) and data.endswith(JSON_RESULTS_SUFFIX):
 125             return data[len(JSON_RESULTS_PREFIX):len(data) - len(JSON_RESULTS_SUFFIX)]
 126         return data
 127
 128     @classmethod
 129     def _generate_file_data(cls, jsonObject, sort_keys=False):
 130         return json.dumps(jsonObject, separators=(',', ':'), sort_keys=sort_keys)
 131
 132     @classmethod
 133     def _load_json(cls, file_data):
 134         json_results_str = cls._strip_prefix_suffix(file_data)
 135         if not json_results_str:
 136             logging.warning("No json results data.")
 137             return None
 138
 139         try:
 140             return json.loads(json_results_str)
 141         except:
 142             logging.debug(json_results_str)
 143             logging.error("Failed to load json results: %s", traceback.print_exception(*sys.exc_info()))
 144             return None
 145
 146     @classmethod
 147     def _merge_json(cls, aggregated_json, incremental_json, num_runs):
 148         # We have to delete expected entries because the incremental json may not have any
 149         # entry for every test in the aggregated json. But, the incremental json will have
 150         # all the correct expected entries for that run.
 151         cls._delete_expected_entries(aggregated_json[TESTS_KEY])
 152         cls._merge_non_test_data(aggregated_json, incremental_json, num_runs)
 153         incremental_tests = incremental_json[TESTS_KEY]
 154         if incremental_tests:
 155             aggregated_tests = aggregated_json[TESTS_KEY]
 156             cls._merge_tests(aggregated_tests, incremental_tests, num_runs)
 157
 158     @classmethod
 159     def _delete_expected_entries(cls, aggregated_json):
 160         for key in aggregated_json:
 161             item = aggregated_json[key]
 162             if _is_directory(item):
 163                 cls._delete_expected_entries(item)
 164             else:
 165                 if EXPECTED_KEY in item:
 166                     del item[EXPECTED_KEY]
 167                 if BUG_KEY in item:
 168                     del item[BUG_KEY]
 169
 170     @classmethod
 171     def _merge_non_test_data(cls, aggregated_json, incremental_json, num_runs):
 172         incremental_builds = incremental_json[BUILD_NUMBERS_KEY]
 173         aggregated_builds = aggregated_json[BUILD_NUMBERS_KEY]
 174         aggregated_build_number = int(aggregated_builds[0])
 175
 176         # FIXME: It's no longer possible to have multiple runs worth of data in the incremental_json,
 177         # So we can get rid of this for-loop and the associated index.
 178         for index in reversed(range(len(incremental_builds))):
 179             build_number = int(incremental_builds[index])
 180             logging.debug("Merging build %s, incremental json index: %d.", build_number, index)
 181
 182             # Merge this build into aggreagated results.
 183             cls._merge_one_build(aggregated_json, incremental_json, index, num_runs)
 184
 185     @classmethod
 186     def _merge_one_build(cls, aggregated_json, incremental_json, incremental_index, num_runs):
 187         for key in incremental_json.keys():
 188             # Merge json results except "tests" properties (results, times etc).
 189             # "tests" properties will be handled separately.
 190             if key == TESTS_KEY or key == FAILURE_MAP_KEY:
 191                 continue
 192
 193             if key in aggregated_json:
 194                 if key == FAILURES_BY_TYPE_KEY:
 195                     cls._merge_one_build(aggregated_json[key], incremental_json[key], incremental_index, num_runs=num_runs)
 196                 else:
 197                     aggregated_json[key].insert(0, incremental_json[key][incremental_index])
 198                     aggregated_json[key] = aggregated_json[key][:num_runs]
 199             else:
 200                 aggregated_json[key] = incremental_json[key]
 201
 202     @classmethod
 203     def _merge_tests(cls, aggregated_json, incremental_json, num_runs):
 204         # FIXME: Some data got corrupted and has results/times at the directory level.
 205         # Once the data is fixe, this should assert that the directory level does not have
 206         # results or times and just return "RESULTS_KEY not in subtree".
 207         if RESULTS_KEY in aggregated_json:
 208             del aggregated_json[RESULTS_KEY]
 209         if TIMES_KEY in aggregated_json:
 210             del aggregated_json[TIMES_KEY]
 211
 212         all_tests = set(aggregated_json.iterkeys())
 213         if incremental_json:
 214             all_tests |= set(incremental_json.iterkeys())
 215
 216         for test_name in all_tests:
 217             if test_name not in aggregated_json:
 218                 aggregated_json[test_name] = incremental_json[test_name]
 219                 continue
 220
 221             incremental_sub_result = incremental_json[test_name] if incremental_json and test_name in incremental_json else None
 222             if _is_directory(aggregated_json[test_name]):
 223                 cls._merge_tests(aggregated_json[test_name], incremental_sub_result, num_runs)
 224                 continue
 225
 226             aggregated_test = aggregated_json[test_name]
 227
 228             if incremental_sub_result:
 229                 results = incremental_sub_result[RESULTS_KEY]
 230                 times = incremental_sub_result[TIMES_KEY]
 231                 if EXPECTED_KEY in incremental_sub_result and incremental_sub_result[EXPECTED_KEY] != PASS_STRING:
 232                     aggregated_test[EXPECTED_KEY] = incremental_sub_result[EXPECTED_KEY]
 233                 if BUG_KEY in incremental_sub_result:
 234                     aggregated_test[BUG_KEY] = incremental_sub_result[BUG_KEY]
 235             else:
 236                 results = [[1, NO_DATA]]
 237                 times = [[1, 0]]
 238
 239             cls._insert_item_run_length_encoded(results, aggregated_test[RESULTS_KEY], num_runs)
 240             cls._insert_item_run_length_encoded(times, aggregated_test[TIMES_KEY], num_runs)
 241
 242     @classmethod
 243     def _insert_item_run_length_encoded(cls, incremental_item, aggregated_item, num_runs):
 244         for item in incremental_item:
 245             if len(aggregated_item) and item[1] == aggregated_item[0][1]:
 246                 aggregated_item[0][0] = min(aggregated_item[0][0] + item[0], num_runs)
 247             else:
 248                 aggregated_item.insert(0, item)
 249
 250     @classmethod
 251     def _normalize_results(cls, aggregated_json, num_runs, run_time_pruning_threshold):
 252         names_to_delete = []
 253         for test_name in aggregated_json:
 254             if _is_directory(aggregated_json[test_name]):
 255                 cls._normalize_results(aggregated_json[test_name], num_runs, run_time_pruning_threshold)
 256                 # If normalizing deletes all the children of this directory, also delete the directory.
 257                 if not aggregated_json[test_name]:
 258                     names_to_delete.append(test_name)
 259             else:
 260                 leaf = aggregated_json[test_name]
 261                 leaf[RESULTS_KEY] = cls._remove_items_over_max_number_of_builds(leaf[RESULTS_KEY], num_runs)
 262                 leaf[TIMES_KEY] = cls._remove_items_over_max_number_of_builds(leaf[TIMES_KEY], num_runs)
 263                 if cls._should_delete_leaf(leaf, run_time_pruning_threshold):
 264                     names_to_delete.append(test_name)
 265
 266         for test_name in names_to_delete:
 267             del aggregated_json[test_name]
 268
 269     @classmethod
 270     def _should_delete_leaf(cls, leaf, run_time_pruning_threshold):
 271         if leaf.get(EXPECTED_KEY, PASS_STRING) != PASS_STRING:
 272             return False
 273
 274         if BUG_KEY in leaf:
 275             return False
 276
 277         deletable_types = set((PASS, NO_DATA, NOTRUN))
 278         for result in leaf[RESULTS_KEY]:
 279             if result[1] not in deletable_types:
 280                 return False
 281
 282         for time in leaf[TIMES_KEY]:
 283             if time[1] >= run_time_pruning_threshold:
 284                 return False
 285
 286         return True
 287
 288     @classmethod
 289     def _remove_items_over_max_number_of_builds(cls, encoded_list, num_runs):
 290         num_builds = 0
 291         index = 0
 292         for result in encoded_list:
 293             num_builds = num_builds + result[0]
 294             index = index + 1
 295             if num_builds >= num_runs:
 296                 return encoded_list[:index]
 297
 298         return encoded_list
 299
 300     @classmethod
 301     def _convert_gtest_json_to_aggregate_results_format(cls, json):
 302         # FIXME: Change gtests over to uploading the full results format like layout-tests
 303         # so we don't have to do this normalizing.
 304         # http://crbug.com/247192.
 305
 306         if FAILURES_BY_TYPE_KEY in json:
 307             # This is already in the right format.
 308             return
 309
 310         failures_by_type = {}
 311         for fixableCount in json[FIXABLE_COUNTS_KEY]:
 312             for failure_type, count in fixableCount.items():
 313                 failure_string = CHAR_TO_FAILURE[failure_type]
 314                 if failure_string not in failures_by_type:
 315                     failures_by_type[failure_string] = []
 316                 failures_by_type[failure_string].append(count)
 317         json[FAILURES_BY_TYPE_KEY] = failures_by_type
 318
 319     @classmethod
 320     def _check_json(cls, builder, json):
 321         version = json[VERSIONS_KEY]
 322         if version > JSON_RESULTS_HIERARCHICAL_VERSION:
 323             return "Results JSON version '%s' is not supported." % version
 324
 325         if not builder in json:
 326             return "Builder '%s' is not in json results." % builder
 327
 328         results_for_builder = json[builder]
 329         if not BUILD_NUMBERS_KEY in results_for_builder:
 330             return "Missing build number in json results."
 331
 332         cls._convert_gtest_json_to_aggregate_results_format(json[builder])
 333
 334         # FIXME: Remove this once all the bots have cycled with this code.
 335         # The failure map was moved from the top-level to being below the builder
 336         # like everything else.
 337         if FAILURE_MAP_KEY in json:
 338             del json[FAILURE_MAP_KEY]
 339
 340         # FIXME: Remove this code once the gtests switch over to uploading the full_results.json format.
 341         # Once the bots have cycled with this code, we can move this loop into _convert_gtest_json_to_aggregate_results_format.
 342         KEYS_TO_DELETE = ["fixableCount", "fixableCounts", "allFixableCount"]
 343         for key in KEYS_TO_DELETE:
 344             if key in json[builder]:
 345                 del json[builder][key]
 346
 347         return ""
 348
 349     @classmethod
 350     def _populate_tests_from_full_results(cls, full_results, new_results):
 351         if EXPECTED_KEY in full_results:
 352             expected = full_results[EXPECTED_KEY]
 353             if expected != PASS_STRING and expected != NOTRUN_STRING:
 354                 new_results[EXPECTED_KEY] = expected
 355             time = int(round(full_results[TIME_KEY])) if TIME_KEY in full_results else 0
 356             new_results[TIMES_KEY] = [[1, time]]
 357
 358             actual_failures = full_results[ACTUAL_KEY]
 359             # Treat unexpected skips like NOTRUNs to avoid exploding the results JSON files
 360             # when a bot exits early (e.g. due to too many crashes/timeouts).
 361             if expected != SKIP_STRING and actual_failures == SKIP_STRING:
 362                 expected = first_actual_failure = NOTRUN_STRING
 363             elif expected == NOTRUN_STRING:
 364                 first_actual_failure = expected
 365             else:
 366                 # FIXME: Include the retry result as well and find a nice way to display it in the flakiness dashboard.
 367                 first_actual_failure = actual_failures.split(' ')[0]
 368             new_results[RESULTS_KEY] = [[1, FAILURE_TO_CHAR[first_actual_failure]]]
 369
 370             if BUG_KEY in full_results:
 371                 new_results[BUG_KEY] = full_results[BUG_KEY]
 372             return
 373
 374         for key in full_results:
 375             new_results[key] = {}
 376             cls._populate_tests_from_full_results(full_results[key], new_results[key])
 377
 378     @classmethod
 379     def _convert_full_results_format_to_aggregate(cls, full_results_format):
 380         num_total_tests = 0
 381         num_failing_tests = 0
 382         failures_by_type = full_results_format[FAILURES_BY_TYPE_KEY]
 383
 384         tests = {}
 385         cls._populate_tests_from_full_results(full_results_format[TESTS_KEY], tests)
 386
 387         aggregate_results_format = {
 388             VERSIONS_KEY: JSON_RESULTS_HIERARCHICAL_VERSION,
 389             full_results_format[BUILDER_NAME_KEY]: {
 390                 # FIXME: Use dict comprehensions once we update the server to python 2.7.
 391                 FAILURES_BY_TYPE_KEY: dict((key, [value]) for key, value in failures_by_type.items()),
 392                 TESTS_KEY: tests,
 393                 # FIXME: Have all the consumers of this switch over to the full_results_format keys
 394                 # so we don't have to do this silly conversion. Or switch the full_results_format keys
 395                 # to be camel-case.
 396                 BUILD_NUMBERS_KEY: [full_results_format['build_number']],
 397                 'chromeRevision': [full_results_format['chromium_revision']],
 398                 'blinkRevision': [full_results_format['blink_revision']],
 399                 'secondsSinceEpoch': [full_results_format['seconds_since_epoch']],
 400             }
 401         }
 402         return aggregate_results_format
 403
 404     @classmethod
 405     def _get_incremental_json(cls, builder, results_json, is_full_results_format):
 406         if not results_json:
 407             return "No incremental JSON data to merge.", 403
 408
 409         if is_full_results_format:
 410             logging.info("Converting full results format to aggregate.")
 411             results_json = cls._convert_full_results_format_to_aggregate(results_json)
 412
 413         logging.info("Checking incremental json.")
 414         check_json_error_string = cls._check_json(builder, results_json)
 415         if check_json_error_string:
 416             return check_json_error_string, 403
 417         return results_json, 200
 418
 419     @classmethod
 420     def _get_aggregated_json(cls, builder, aggregated_string):
 421         logging.info("Loading existing aggregated json.")
 422         aggregated_json = cls._load_json(aggregated_string)
 423         if not aggregated_json:
 424             return None, 200
 425
 426         logging.info("Checking existing aggregated json.")
 427         check_json_error_string = cls._check_json(builder, aggregated_json)
 428         if check_json_error_string:
 429             return check_json_error_string, 500
 430
 431         return aggregated_json, 200
 432
 433     @classmethod
 434     def merge(cls, builder, aggregated_string, incremental_json, num_runs, sort_keys=False):
 435         aggregated_json, status_code = cls._get_aggregated_json(builder, aggregated_string)
 436         if not aggregated_json:
 437             aggregated_json = incremental_json
 438         elif status_code != 200:
 439             return aggregated_json, status_code
 440         else:
 441             if aggregated_json[builder][BUILD_NUMBERS_KEY][0] == incremental_json[builder][BUILD_NUMBERS_KEY][0]:
 442                 status_string = "Incremental JSON's build number %s is the latest build number in the aggregated JSON." % str(aggregated_json[builder][BUILD_NUMBERS_KEY][0])
 443                 return status_string, 409
 444
 445             logging.info("Merging json results.")
 446             try:
 447                 cls._merge_json(aggregated_json[builder], incremental_json[builder], num_runs)
 448             except:
 449                 return "Failed to merge json results: %s", traceback.print_exception(*sys.exc_info()), 500
 450
 451         aggregated_json[VERSIONS_KEY] = JSON_RESULTS_HIERARCHICAL_VERSION
 452         aggregated_json[builder][FAILURE_MAP_KEY] = CHAR_TO_FAILURE
 453
 454         is_debug_builder = re.search(r"(Debug|Dbg)", builder, re.I)
 455         run_time_pruning_threshold = 3 * JSON_RESULTS_MIN_TIME if is_debug_builder else JSON_RESULTS_MIN_TIME
 456         cls._normalize_results(aggregated_json[builder][TESTS_KEY], num_runs, run_time_pruning_threshold)
 457         return cls._generate_file_data(aggregated_json, sort_keys), 200
 458
 459     @classmethod
 460     def _get_aggregate_file(cls, master, builder, test_type, filename, deprecated_master):
 461         files = TestFile.get_files(master, builder, test_type, None, filename)
 462         if files:
 463             return files[0]
 464
 465         if deprecated_master:
 466             files = TestFile.get_files(deprecated_master, builder, test_type, None, filename)
 467             if files:
 468                 deprecated_file = files[0]
 469                 # Change the master so it gets saved out with the new master name.
 470                 deprecated_file.master = master
 471                 return deprecated_file
 472
 473         file = TestFile()
 474         file.master = master
 475         file.builder = builder
 476         file.test_type = test_type
 477         file.build_number = None
 478         file.name = filename
 479         file.data = ""
 480         return file
 481
 482     @classmethod
 483     def update(cls, master, builder, test_type, results_json, deprecated_master, is_full_results_format):
 484         logging.info("Updating %s and %s." % (JSON_RESULTS_FILE_SMALL, JSON_RESULTS_FILE))
 485         small_file = cls._get_aggregate_file(master, builder, test_type, JSON_RESULTS_FILE_SMALL, deprecated_master)
 486         large_file = cls._get_aggregate_file(master, builder, test_type, JSON_RESULTS_FILE, deprecated_master)
 487         return cls.update_files(builder, results_json, small_file, large_file, is_full_results_format)
 488
 489     @classmethod
 490     def update_files(cls, builder, results_json, small_file, large_file, is_full_results_format):
 491         incremental_json, status_code = cls._get_incremental_json(builder, results_json, is_full_results_format)
 492         if status_code != 200:
 493             return incremental_json, status_code
 494
 495         status_string, status_code = cls.update_file(builder, small_file, incremental_json, JSON_RESULTS_MAX_BUILDS_SMALL)
 496         if status_code != 200:
 497             return status_string, status_code
 498
 499         return cls.update_file(builder, large_file, incremental_json, JSON_RESULTS_MAX_BUILDS)
 500
 501     @classmethod
 502     def update_file(cls, builder, file, incremental_json, num_runs):
 503         new_results, status_code = cls.merge(builder, file.data, incremental_json, num_runs)
 504         if status_code != 200:
 505             return new_results, status_code
 506         return TestFile.save_file(file, new_results)
 507
 508     @classmethod
 509     def _delete_results_and_times(cls, tests):
 510         for key in tests.keys():
 511             if key in (RESULTS_KEY, TIMES_KEY):
 512                 del tests[key]
 513             else:
 514                 cls._delete_results_and_times(tests[key])
 515
 516     @classmethod
 517     def get_test_list(cls, builder, json_file_data):
 518         logging.debug("Loading test results json...")
 519         json = cls._load_json(json_file_data)
 520         if not json:
 521             return None
 522
 523         logging.debug("Checking test results json...")
 524
 525         check_json_error_string = cls._check_json(builder, json)
 526         if check_json_error_string:
 527             return None
 528
 529         test_list_json = {}
 530         tests = json[builder][TESTS_KEY]
 531         cls._delete_results_and_times(tests)
 532         test_list_json[builder] = {TESTS_KEY: tests}
 533         return cls._generate_file_data(test_list_json)