src/third_party/skia/gm/rebaseline_server/compare_configs.py

   1 #!/usr/bin/python
   2
   3 """
   4 Copyright 2014 Google Inc.
   5
   6 Use of this source code is governed by a BSD-style license that can be
   7 found in the LICENSE file.
   8
   9 Compare GM results for two configs, across all builders.
  10 """
  11
  12 # System-level imports
  13 import argparse
  14 import logging
  15 import time
  16
  17 # Must fix up PYTHONPATH before importing from within Skia
  18 import rs_fixpypath  # pylint: disable=W0611
  19
  20 # Imports from within Skia
  21 from py.utils import url_utils
  22 import gm_json
  23 import imagediffdb
  24 import imagepair
  25 import imagepairset
  26 import results
  27
  28
  29 class ConfigComparisons(results.BaseComparisons):
  30   """Loads results from two different configurations into an ImagePairSet.
  31
  32   Loads actual and expected results from all builders, except for those skipped
  33   by _ignore_builder().
  34   """
  35
  36   def __init__(self, configs, actuals_root=results.DEFAULT_ACTUALS_DIR,
  37                generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
  38                diff_base_url=None, builder_regex_list=None):
  39     """
  40     Args:
  41       configs: (string, string) tuple; pair of configs to compare
  42       actuals_root: root directory containing all actual-results.json files
  43       generated_images_root: directory within which to create all pixel diffs;
  44           if this directory does not yet exist, it will be created
  45       diff_base_url: base URL within which the client should look for diff
  46           images; if not specified, defaults to a "file:///" URL representation
  47           of generated_images_root
  48       builder_regex_list: List of regular expressions specifying which builders
  49           we will process. If None, process all builders.
  50     """
  51     super(ConfigComparisons, self).__init__()
  52     time_start = int(time.time())
  53     if builder_regex_list != None:
  54       self.set_match_builders_pattern_list(builder_regex_list)
  55     self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
  56     self._diff_base_url = (
  57         diff_base_url or
  58         url_utils.create_filepath_url(generated_images_root))
  59     self._actuals_root = actuals_root
  60     self._load_config_pairs(configs)
  61     self._timestamp = int(time.time())
  62     logging.info('Results complete; took %d seconds.' %
  63                  (self._timestamp - time_start))
  64
  65   def _load_config_pairs(self, configs):
  66     """Loads the results of all tests, across all builders (based on the
  67     files within self._actuals_root), compares them across two configs,
  68     and stores the summary in self._results.
  69
  70     Args:
  71       configs: tuple of strings; pair of configs to compare
  72     """
  73     logging.info('Reading actual-results JSON files from %s...' %
  74                  self._actuals_root)
  75     actual_builder_dicts = self._read_builder_dicts_from_root(
  76         self._actuals_root)
  77     configA, configB = configs
  78     logging.info('Comparing configs %s and %s...' % (configA, configB))
  79
  80     all_image_pairs = imagepairset.ImagePairSet(
  81         descriptions=configs,
  82         diff_base_url=self._diff_base_url)
  83     failing_image_pairs = imagepairset.ImagePairSet(
  84         descriptions=configs,
  85         diff_base_url=self._diff_base_url)
  86
  87     all_image_pairs.ensure_extra_column_values_in_summary(
  88         column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
  89             results.KEY__RESULT_TYPE__FAILED,
  90             results.KEY__RESULT_TYPE__NOCOMPARISON,
  91             results.KEY__RESULT_TYPE__SUCCEEDED,
  92         ])
  93     failing_image_pairs.ensure_extra_column_values_in_summary(
  94         column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
  95             results.KEY__RESULT_TYPE__FAILED,
  96             results.KEY__RESULT_TYPE__NOCOMPARISON,
  97         ])
  98
  99     builders = sorted(actual_builder_dicts.keys())
 100     num_builders = len(builders)
 101     builder_num = 0
 102     for builder in builders:
 103       builder_num += 1
 104       logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' %
 105                    (builder_num, num_builders, builder))
 106       actual_results_for_this_builder = (
 107           actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
 108       for result_type in sorted(actual_results_for_this_builder.keys()):
 109         results_of_this_type = actual_results_for_this_builder[result_type]
 110         if not results_of_this_type:
 111           continue
 112
 113         tests_found = set()
 114         for image_name in sorted(results_of_this_type.keys()):
 115           (test, _) = results.IMAGE_FILENAME_RE.match(image_name).groups()
 116           tests_found.add(test)
 117
 118         for test in tests_found:
 119           # Get image_relative_url (or None) for each of configA, configB
 120           image_name_A = results.IMAGE_FILENAME_FORMATTER % (test, configA)
 121           configA_image_relative_url = ConfigComparisons._create_relative_url(
 122               hashtype_and_digest=results_of_this_type.get(image_name_A),
 123               test_name=test)
 124           image_name_B = results.IMAGE_FILENAME_FORMATTER % (test, configB)
 125           configB_image_relative_url = ConfigComparisons._create_relative_url(
 126               hashtype_and_digest=results_of_this_type.get(image_name_B),
 127               test_name=test)
 128
 129           # If we have images for at least one of these two configs,
 130           # add them to our list.
 131           if configA_image_relative_url or configB_image_relative_url:
 132             if configA_image_relative_url == configB_image_relative_url:
 133               result_type = results.KEY__RESULT_TYPE__SUCCEEDED
 134             elif not configA_image_relative_url:
 135               result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
 136             elif not configB_image_relative_url:
 137               result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
 138             else:
 139               result_type = results.KEY__RESULT_TYPE__FAILED
 140
 141             extra_columns_dict = {
 142                 results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type,
 143                 results.KEY__EXTRACOLUMNS__BUILDER: builder,
 144                 results.KEY__EXTRACOLUMNS__TEST: test,
 145                 # TODO(epoger): Right now, the client UI crashes if it receives
 146                 # results that do not include a 'config' column.
 147                 # Until we fix that, keep the client happy.
 148                 results.KEY__EXTRACOLUMNS__CONFIG: 'TODO',
 149             }
 150
 151             try:
 152               image_pair = imagepair.ImagePair(
 153                   image_diff_db=self._image_diff_db,
 154                   imageA_base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
 155                   imageB_base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
 156                   imageA_relative_url=configA_image_relative_url,
 157                   imageB_relative_url=configB_image_relative_url,
 158                   extra_columns=extra_columns_dict)
 159               all_image_pairs.add_image_pair(image_pair)
 160               if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
 161                 failing_image_pairs.add_image_pair(image_pair)
 162             except (KeyError, TypeError):
 163               logging.exception(
 164                   'got exception while creating ImagePair for test '
 165                   '"%s", builder "%s"' % (test, builder))
 166
 167     # pylint: disable=W0201
 168     self._results = {
 169       results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
 170       results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
 171     }
 172
 173
 174 def main():
 175   logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
 176                       datefmt='%m/%d/%Y %H:%M:%S',
 177                       level=logging.INFO)
 178   parser = argparse.ArgumentParser()
 179   parser.add_argument(
 180       '--actuals', default=results.DEFAULT_ACTUALS_DIR,
 181       help='Directory containing all actual-result JSON files; defaults to '
 182       '\'%(default)s\' .')
 183   parser.add_argument(
 184       'config', nargs=2,
 185       help='Two configurations to compare (8888, gpu, etc.).')
 186   parser.add_argument(
 187       '--outfile', required=True,
 188       help='File to write result summary into, in JSON format.')
 189   parser.add_argument(
 190       '--results', default=results.KEY__HEADER__RESULTS_FAILURES,
 191       help='Which result types to include. Defaults to \'%(default)s\'; '
 192       'must be one of ' +
 193       str([results.KEY__HEADER__RESULTS_FAILURES,
 194            results.KEY__HEADER__RESULTS_ALL]))
 195   parser.add_argument(
 196       '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT,
 197       help='Directory within which to download images and generate diffs; '
 198       'defaults to \'%(default)s\' .')
 199   args = parser.parse_args()
 200   results_obj = ConfigComparisons(configs=args.config,
 201                                   actuals_root=args.actuals,
 202                                   generated_images_root=args.workdir)
 203   gm_json.WriteToFile(
 204       results_obj.get_packaged_results_of_type(results_type=args.results),
 205       args.outfile)
 206
 207
 208 if __name__ == '__main__':
 209   main()