src/third_party/skia/gm/rebaseline_server/compare_to_expectations.py

   1 #!/usr/bin/python
   2
   3 """
   4 Copyright 2013 Google Inc.
   5
   6 Use of this source code is governed by a BSD-style license that can be
   7 found in the LICENSE file.
   8
   9 Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
  10 """
  11
  12 # System-level imports
  13 import argparse
  14 import fnmatch
  15 import json
  16 import logging
  17 import os
  18 import re
  19 import sys
  20 import time
  21
  22 # Imports from within Skia
  23 import fix_pythonpath  # must do this first
  24 from pyutils import url_utils
  25 import gm_json
  26 import imagediffdb
  27 import imagepair
  28 import imagepairset
  29 import results
  30
  31 EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [
  32     results.KEY__EXPECTATIONS__BUGS,
  33     results.KEY__EXPECTATIONS__IGNOREFAILURE,
  34     results.KEY__EXPECTATIONS__REVIEWED,
  35 ]
  36 TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
  37 DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm')
  38 DEFAULT_IGNORE_FAILURES_FILE = 'ignored-tests.txt'
  39
  40 IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image')
  41
  42
  43 class ExpectationComparisons(results.BaseComparisons):
  44   """Loads actual and expected GM results into an ImagePairSet.
  45
  46   Loads actual and expected results from all builders, except for those skipped
  47   by _ignore_builder().
  48
  49   Once this object has been constructed, the results (in self._results[])
  50   are immutable.  If you want to update the results based on updated JSON
  51   file contents, you will need to create a new ExpectationComparisons object."""
  52
  53   def __init__(self, actuals_root=results.DEFAULT_ACTUALS_DIR,
  54                expected_root=DEFAULT_EXPECTATIONS_DIR,
  55                ignore_failures_file=DEFAULT_IGNORE_FAILURES_FILE,
  56                generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
  57                diff_base_url=None, builder_regex_list=None):
  58     """
  59     Args:
  60       actuals_root: root directory containing all actual-results.json files
  61       expected_root: root directory containing all expected-results.json files
  62       ignore_failures_file: if a file with this name is found within
  63           expected_root, ignore failures for any tests listed in the file
  64       generated_images_root: directory within which to create all pixel diffs;
  65           if this directory does not yet exist, it will be created
  66       diff_base_url: base URL within which the client should look for diff
  67           images; if not specified, defaults to a "file:///" URL representation
  68           of generated_images_root
  69       builder_regex_list: List of regular expressions specifying which builders
  70           we will process. If None, process all builders.
  71     """
  72     time_start = int(time.time())
  73     if builder_regex_list != None:
  74       self.set_match_builders_pattern_list(builder_regex_list)
  75     self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
  76     self._diff_base_url = (
  77         diff_base_url or
  78         url_utils.create_filepath_url(generated_images_root))
  79     self._actuals_root = actuals_root
  80     self._expected_root = expected_root
  81     self._ignore_failures_on_these_tests = []
  82     if ignore_failures_file:
  83       self._ignore_failures_on_these_tests = (
  84           ExpectationComparisons._read_noncomment_lines(
  85               os.path.join(expected_root, ignore_failures_file)))
  86     self._load_actual_and_expected()
  87     self._timestamp = int(time.time())
  88     logging.info('Results complete; took %d seconds.' %
  89                  (self._timestamp - time_start))
  90
  91   def edit_expectations(self, modifications):
  92     """Edit the expectations stored within this object and write them back
  93     to disk.
  94
  95     Note that this will NOT update the results stored in self._results[] ;
  96     in order to see those updates, you must instantiate a new
  97     ExpectationComparisons object based on the (now updated) files on disk.
  98
  99     Args:
 100       modifications: a list of dictionaries, one for each expectation to update:
 101
 102          [
 103            {
 104              imagepair.KEY__IMAGEPAIRS__EXPECTATIONS: {
 105                results.KEY__EXPECTATIONS__BUGS: [123, 456],
 106                results.KEY__EXPECTATIONS__IGNOREFAILURE: false,
 107                results.KEY__EXPECTATIONS__REVIEWED: true,
 108              },
 109              imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS: {
 110                results.KEY__EXTRACOLUMNS__BUILDER: 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug',
 111                results.KEY__EXTRACOLUMNS__CONFIG: '8888',
 112                results.KEY__EXTRACOLUMNS__TEST: 'bigmatrix',
 113              },
 114              results.KEY__IMAGEPAIRS__IMAGE_B_URL: 'bitmap-64bitMD5/bigmatrix/10894408024079689926.png',
 115            },
 116            ...
 117          ]
 118
 119     """
 120     expected_builder_dicts = self._read_builder_dicts_from_root(
 121         self._expected_root)
 122     for mod in modifications:
 123       image_name = results.IMAGE_FILENAME_FORMATTER % (
 124           mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS]
 125              [results.KEY__EXTRACOLUMNS__TEST],
 126           mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS]
 127              [results.KEY__EXTRACOLUMNS__CONFIG])
 128       _, hash_type, hash_digest = gm_json.SplitGmRelativeUrl(
 129           mod[imagepair.KEY__IMAGEPAIRS__IMAGE_B_URL])
 130       allowed_digests = [[hash_type, int(hash_digest)]]
 131       new_expectations = {
 132           gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: allowed_digests,
 133       }
 134       for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM:
 135         value = mod[imagepair.KEY__IMAGEPAIRS__EXPECTATIONS].get(field)
 136         if value is not None:
 137           new_expectations[field] = value
 138       builder_dict = expected_builder_dicts[
 139           mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS]
 140              [results.KEY__EXTRACOLUMNS__BUILDER]]
 141       builder_expectations = builder_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS)
 142       if not builder_expectations:
 143         builder_expectations = {}
 144         builder_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = builder_expectations
 145       builder_expectations[image_name] = new_expectations
 146     ExpectationComparisons._write_dicts_to_root(
 147         expected_builder_dicts, self._expected_root)
 148
 149   @staticmethod
 150   def _write_dicts_to_root(meta_dict, root, pattern='*.json'):
 151     """Write all per-builder dictionaries within meta_dict to files under
 152     the root path.
 153
 154     Security note: this will only write to files that already exist within
 155     the root path (as found by os.walk() within root), so we don't need to
 156     worry about malformed content writing to disk outside of root.
 157     However, the data written to those files is not double-checked, so it
 158     could contain poisonous data.
 159
 160     Args:
 161       meta_dict: a builder-keyed meta-dictionary containing all the JSON
 162                  dictionaries we want to write out
 163       root: path to root of directory tree within which to write files
 164       pattern: which files to write within root (fnmatch-style pattern)
 165
 166     Raises:
 167       IOError if root does not refer to an existing directory
 168       KeyError if the set of per-builder dictionaries written out was
 169                different than expected
 170     """
 171     if not os.path.isdir(root):
 172       raise IOError('no directory found at path %s' % root)
 173     actual_builders_written = []
 174     for dirpath, dirnames, filenames in os.walk(root):
 175       for matching_filename in fnmatch.filter(filenames, pattern):
 176         builder = os.path.basename(dirpath)
 177         per_builder_dict = meta_dict.get(builder)
 178         if per_builder_dict is not None:
 179           fullpath = os.path.join(dirpath, matching_filename)
 180           gm_json.WriteToFile(per_builder_dict, fullpath)
 181           actual_builders_written.append(builder)
 182
 183     # Check: did we write out the set of per-builder dictionaries we
 184     # expected to?
 185     expected_builders_written = sorted(meta_dict.keys())
 186     actual_builders_written.sort()
 187     if expected_builders_written != actual_builders_written:
 188       raise KeyError(
 189           'expected to write dicts for builders %s, but actually wrote them '
 190           'for builders %s' % (
 191               expected_builders_written, actual_builders_written))
 192
 193   def _load_actual_and_expected(self):
 194     """Loads the results of all tests, across all builders (based on the
 195     files within self._actuals_root and self._expected_root),
 196     and stores them in self._results.
 197     """
 198     logging.info('Reading actual-results JSON files from %s...' %
 199                  self._actuals_root)
 200     actual_builder_dicts = self._read_builder_dicts_from_root(
 201         self._actuals_root)
 202     logging.info('Reading expected-results JSON files from %s...' %
 203                  self._expected_root)
 204     expected_builder_dicts = self._read_builder_dicts_from_root(
 205         self._expected_root)
 206
 207     all_image_pairs = imagepairset.ImagePairSet(
 208         descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
 209         diff_base_url=self._diff_base_url)
 210     failing_image_pairs = imagepairset.ImagePairSet(
 211         descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
 212         diff_base_url=self._diff_base_url)
 213
 214     all_image_pairs.ensure_extra_column_values_in_summary(
 215         column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
 216             results.KEY__RESULT_TYPE__FAILED,
 217             results.KEY__RESULT_TYPE__FAILUREIGNORED,
 218             results.KEY__RESULT_TYPE__NOCOMPARISON,
 219             results.KEY__RESULT_TYPE__SUCCEEDED,
 220         ])
 221     failing_image_pairs.ensure_extra_column_values_in_summary(
 222         column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
 223             results.KEY__RESULT_TYPE__FAILED,
 224             results.KEY__RESULT_TYPE__FAILUREIGNORED,
 225             results.KEY__RESULT_TYPE__NOCOMPARISON,
 226         ])
 227
 228     # Only consider builders we have both expected and actual results for.
 229     # Fixes http://skbug.com/2486 ('rebaseline_server shows actual results
 230     # (but not expectations) for Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug
 231     # builder')
 232     actual_builder_set = set(actual_builder_dicts.keys())
 233     expected_builder_set = set(expected_builder_dicts.keys())
 234     builders = sorted(actual_builder_set.intersection(expected_builder_set))
 235
 236     num_builders = len(builders)
 237     builder_num = 0
 238     for builder in builders:
 239       builder_num += 1
 240       logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' %
 241                    (builder_num, num_builders, builder))
 242       actual_results_for_this_builder = (
 243           actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
 244       for result_type in sorted(actual_results_for_this_builder.keys()):
 245         results_of_this_type = actual_results_for_this_builder[result_type]
 246         if not results_of_this_type:
 247           continue
 248         for image_name in sorted(results_of_this_type.keys()):
 249           (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups()
 250           actual_image_relative_url = (
 251               ExpectationComparisons._create_relative_url(
 252                   hashtype_and_digest=results_of_this_type[image_name],
 253                   test_name=test))
 254
 255           # Default empty expectations; overwrite these if we find any real ones
 256           expectations_per_test = None
 257           expected_image_relative_url = None
 258           expectations_dict = None
 259           try:
 260             expectations_per_test = (
 261                 expected_builder_dicts
 262                 [builder][gm_json.JSONKEY_EXPECTEDRESULTS][image_name])
 263             # TODO(epoger): assumes a single allowed digest per test, which is
 264             # fine; see https://code.google.com/p/skia/issues/detail?id=1787
 265             expected_image_hashtype_and_digest = (
 266                 expectations_per_test
 267                 [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0])
 268             expected_image_relative_url = (
 269                 ExpectationComparisons._create_relative_url(
 270                     hashtype_and_digest=expected_image_hashtype_and_digest,
 271                     test_name=test))
 272             expectations_dict = {}
 273             for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM:
 274               expectations_dict[field] = expectations_per_test.get(field)
 275           except (KeyError, TypeError):
 276             # There are several cases in which we would expect to find
 277             # no expectations for a given test:
 278             #
 279             # 1. result_type == NOCOMPARISON
 280             #   There are no expectations for this test yet!
 281             #
 282             # 2. alternate rendering mode failures (e.g. serialized)
 283             #   In cases like
 284             #   https://code.google.com/p/skia/issues/detail?id=1684
 285             #   ('tileimagefilter GM test failing in serialized render mode'),
 286             #   the gm-actuals will list a failure for the alternate
 287             #   rendering mode even though we don't have explicit expectations
 288             #   for the test (the implicit expectation is that it must
 289             #   render the same in all rendering modes).
 290             #
 291             # Don't log type 1, because it is common.
 292             # Log other types, because they are rare and we should know about
 293             # them, but don't throw an exception, because we need to keep our
 294             # tools working in the meanwhile!
 295             if result_type != results.KEY__RESULT_TYPE__NOCOMPARISON:
 296               logging.warning('No expectations found for test: %s' % {
 297                   results.KEY__EXTRACOLUMNS__BUILDER: builder,
 298                   results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type,
 299                   'image_name': image_name,
 300                   })
 301
 302           # If this test was recently rebaselined, it will remain in
 303           # the 'failed' set of actuals until all the bots have
 304           # cycled (although the expectations have indeed been set
 305           # from the most recent actuals).  Treat these as successes
 306           # instead of failures.
 307           #
 308           # TODO(epoger): Do we need to do something similar in
 309           # other cases, such as when we have recently marked a test
 310           # as ignoreFailure but it still shows up in the 'failed'
 311           # category?  Maybe we should not rely on the result_type
 312           # categories recorded within the gm_actuals AT ALL, and
 313           # instead evaluate the result_type ourselves based on what
 314           # we see in expectations vs actual checksum?
 315           if expected_image_relative_url == actual_image_relative_url:
 316             updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED
 317           elif ((result_type == results.KEY__RESULT_TYPE__FAILED) and
 318                 (test in self._ignore_failures_on_these_tests)):
 319             updated_result_type = results.KEY__RESULT_TYPE__FAILUREIGNORED
 320           else:
 321             updated_result_type = result_type
 322           extra_columns_dict = {
 323               results.KEY__EXTRACOLUMNS__RESULT_TYPE: updated_result_type,
 324               results.KEY__EXTRACOLUMNS__BUILDER: builder,
 325               results.KEY__EXTRACOLUMNS__TEST: test,
 326               results.KEY__EXTRACOLUMNS__CONFIG: config,
 327           }
 328           try:
 329             image_pair = imagepair.ImagePair(
 330                 image_diff_db=self._image_diff_db,
 331                 base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
 332                 imageA_relative_url=expected_image_relative_url,
 333                 imageB_relative_url=actual_image_relative_url,
 334                 expectations=expectations_dict,
 335                 extra_columns=extra_columns_dict)
 336             all_image_pairs.add_image_pair(image_pair)
 337             if updated_result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
 338               failing_image_pairs.add_image_pair(image_pair)
 339           except Exception:
 340             logging.exception('got exception while creating new ImagePair')
 341
 342     self._results = {
 343       results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
 344       results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
 345     }
 346
 347
 348 def main():
 349   logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
 350                       datefmt='%m/%d/%Y %H:%M:%S',
 351                       level=logging.INFO)
 352   parser = argparse.ArgumentParser()
 353   parser.add_argument(
 354       '--actuals', default=results.DEFAULT_ACTUALS_DIR,
 355       help='Directory containing all actual-result JSON files; defaults to '
 356       '\'%(default)s\' .')
 357   parser.add_argument(
 358       '--expectations', default=DEFAULT_EXPECTATIONS_DIR,
 359       help='Directory containing all expected-result JSON files; defaults to '
 360       '\'%(default)s\' .')
 361   parser.add_argument(
 362       '--ignore-failures-file', default=DEFAULT_IGNORE_FAILURES_FILE,
 363       help='If a file with this name is found within the EXPECTATIONS dir, '
 364       'ignore failures for any tests listed in the file; defaults to '
 365       '\'%(default)s\' .')
 366   parser.add_argument(
 367       '--outfile', required=True,
 368       help='File to write result summary into, in JSON format.')
 369   parser.add_argument(
 370       '--results', default=results.KEY__HEADER__RESULTS_FAILURES,
 371       help='Which result types to include. Defaults to \'%(default)s\'; '
 372       'must be one of ' +
 373       str([results.KEY__HEADER__RESULTS_FAILURES,
 374            results.KEY__HEADER__RESULTS_ALL]))
 375   parser.add_argument(
 376       '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT,
 377       help='Directory within which to download images and generate diffs; '
 378       'defaults to \'%(default)s\' .')
 379   args = parser.parse_args()
 380   results_obj = ExpectationComparisons(
 381       actuals_root=args.actuals,
 382       expected_root=args.expectations,
 383       ignore_failures_file=args.ignore_failures_file,
 384       generated_images_root=args.workdir)
 385   gm_json.WriteToFile(
 386       results_obj.get_packaged_results_of_type(results_type=args.results),
 387       args.outfile)
 388
 389
 390 if __name__ == '__main__':
 391   main()