src/third_party/skia/tools/tests/render_pictures_test.py

   1 #!/usr/bin/python
   2
   3 """
   4 Copyright 2014 Google Inc.
   5
   6 Use of this source code is governed by a BSD-style license that can be
   7 found in the LICENSE file.
   8
   9 Test the render_pictures binary.
  10 """
  11
  12 # System-level imports
  13 import copy
  14 import json
  15 import os
  16 import shutil
  17 import tempfile
  18
  19 # Must fix up PYTHONPATH before importing from within Skia
  20 import fix_pythonpath  # pylint: disable=W0611
  21
  22 # Imports from within Skia
  23 import base_unittest
  24 import find_run_binary
  25
  26 # Maximum length of text diffs to show when tests fail
  27 MAX_DIFF_LENGTH = 30000
  28
  29 EXPECTED_HEADER_CONTENTS = {
  30     "type" : "ChecksummedImages",
  31     "revision" : 1,
  32 }
  33
  34 # Manually verified: 640x400 red rectangle with black border
  35 # Standard expectations will be set up in such a way that this image fails
  36 # the comparison.
  37 RED_WHOLEIMAGE = {
  38     "checksumAlgorithm" : "bitmap-64bitMD5",
  39     "checksumValue" : 2853310525600416231,
  40     "comparisonResult" : "failed",
  41     "filepath" : "red_skp.png",
  42 }
  43
  44 # Manually verified: 640x400 green rectangle with black border
  45 # Standard expectations will be set up in such a way that this image passes
  46 # the comparison.
  47 GREEN_WHOLEIMAGE = {
  48     "checksumAlgorithm" : "bitmap-64bitMD5",
  49     "checksumValue" : 11143979097452425335,
  50     "comparisonResult" : "succeeded",
  51     "filepath" : "green_skp.png",
  52 }
  53
  54 # Manually verified these 6 images, all 256x256 tiles,
  55 # consistent with a tiled version of the 640x400 red rect
  56 # with black borders.
  57 # Standard expectations will be set up in such a way that these images fail
  58 # the comparison.
  59 RED_TILES = [{
  60     "checksumAlgorithm" : "bitmap-64bitMD5",
  61     "checksumValue" : 5815827069051002745,
  62     "comparisonResult" : "failed",
  63     "filepath" : "red_skp-tile0.png",
  64 },{
  65     "checksumAlgorithm" : "bitmap-64bitMD5",
  66     "checksumValue" : 9323613075234140270,
  67     "comparisonResult" : "failed",
  68     "filepath" : "red_skp-tile1.png",
  69 }, {
  70     "checksumAlgorithm" : "bitmap-64bitMD5",
  71     "checksumValue" : 15939355025996362179,
  72     "comparisonResult" : "failed",
  73     "filepath" : "red_skp-tile2.png",
  74 }, {
  75     "checksumAlgorithm" : "bitmap-64bitMD5",
  76     "checksumValue" : 649771916797529222,
  77     "comparisonResult" : "failed",
  78     "filepath" : "red_skp-tile3.png",
  79 }, {
  80     "checksumAlgorithm" : "bitmap-64bitMD5",
  81     "checksumValue" : 8132820002266077288,
  82     "comparisonResult" : "failed",
  83     "filepath" : "red_skp-tile4.png",
  84 }, {
  85     "checksumAlgorithm" : "bitmap-64bitMD5",
  86     "checksumValue" : 2406160701181324581,
  87     "comparisonResult" : "failed",
  88     "filepath" : "red_skp-tile5.png",
  89 }]
  90
  91 # Manually verified these 6 images, all 256x256 tiles,
  92 # consistent with a tiled version of the 640x400 green rect
  93 # with black borders.
  94 # Standard expectations will be set up in such a way that these images pass
  95 # the comparison.
  96 GREEN_TILES = [{
  97     "checksumAlgorithm" : "bitmap-64bitMD5",
  98     "checksumValue" : 12587324416545178013,
  99     "comparisonResult" : "succeeded",
 100     "filepath" : "green_skp-tile0.png",
 101 }, {
 102     "checksumAlgorithm" : "bitmap-64bitMD5",
 103     "checksumValue" : 7624374914829746293,
 104     "comparisonResult" : "succeeded",
 105     "filepath" : "green_skp-tile1.png",
 106 }, {
 107     "checksumAlgorithm" : "bitmap-64bitMD5",
 108     "checksumValue" : 11866144860997809880,
 109     "comparisonResult" : "succeeded",
 110     "filepath" : "green_skp-tile2.png",
 111 }, {
 112     "checksumAlgorithm" : "bitmap-64bitMD5",
 113     "checksumValue" : 3893392565127823822,
 114     "comparisonResult" : "succeeded",
 115     "filepath" : "green_skp-tile3.png",
 116 }, {
 117     "checksumAlgorithm" : "bitmap-64bitMD5",
 118     "checksumValue" : 2083084978343901738,
 119     "comparisonResult" : "succeeded",
 120     "filepath" : "green_skp-tile4.png",
 121 }, {
 122     "checksumAlgorithm" : "bitmap-64bitMD5",
 123     "checksumValue" : 89620927366502076,
 124     "comparisonResult" : "succeeded",
 125     "filepath" : "green_skp-tile5.png",
 126 }]
 127
 128
 129 def modified_dict(input_dict, modification_dict):
 130   """Returns a dict, with some modifications applied to it.
 131
 132   Args:
 133     input_dict: a dictionary (which will be copied, not modified in place)
 134     modification_dict: a set of key/value pairs to overwrite in the dict
 135   """
 136   output_dict = input_dict.copy()
 137   output_dict.update(modification_dict)
 138   return output_dict
 139
 140
 141 def modified_list_of_dicts(input_list, modification_dict):
 142   """Returns a list of dicts, with some modifications applied to each dict.
 143
 144   Args:
 145     input_list: a list of dictionaries; these dicts will be copied, not
 146         modified in place
 147     modification_dict: a set of key/value pairs to overwrite in each dict
 148         within input_list
 149   """
 150   output_list = []
 151   for input_dict in input_list:
 152     output_dict = modified_dict(input_dict, modification_dict)
 153     output_list.append(output_dict)
 154   return output_list
 155
 156
 157 class RenderPicturesTest(base_unittest.TestCase):
 158
 159   def setUp(self):
 160     self.maxDiff = MAX_DIFF_LENGTH
 161     self._expectations_dir = tempfile.mkdtemp()
 162     self._input_skp_dir = tempfile.mkdtemp()
 163     # All output of render_pictures binary will go into this directory.
 164     self._output_dir = tempfile.mkdtemp()
 165
 166   def tearDown(self):
 167     shutil.rmtree(self._expectations_dir)
 168     shutil.rmtree(self._input_skp_dir)
 169     shutil.rmtree(self._output_dir)
 170
 171   def test_tiled_whole_image(self):
 172     """Run render_pictures with tiles and --writeWholeImage flag.
 173
 174     TODO(epoger): This test generates undesired results!  The JSON summary
 175     includes both whole-image and tiled-images (as it should), but only
 176     whole-images are written out to disk.  See http://skbug.com/2463
 177     Once I fix that, I should add a similar test that exercises mismatchPath.
 178
 179     TODO(epoger): I noticed that when this is run without --writePath being
 180     specified, this test writes red_skp.png and green_skp.png to the current
 181     directory.  We should fix that... if --writePath is not specified, this
 182     probably shouldn't write out red_skp.png and green_skp.png at all!
 183     See http://skbug.com/2464
 184     """
 185     output_json_path = os.path.join(self._output_dir, 'actuals.json')
 186     write_path_dir = self.create_empty_dir(
 187         path=os.path.join(self._output_dir, 'writePath'))
 188     self._generate_skps()
 189     expectations_path = self._create_expectations()
 190     self._run_render_pictures([
 191         '-r', self._input_skp_dir,
 192         '--bbh', 'grid', '256', '256',
 193         '--mode', 'tile', '256', '256',
 194         '--readJsonSummaryPath', expectations_path,
 195         '--writeJsonSummaryPath', output_json_path,
 196         '--writePath', write_path_dir,
 197         '--writeWholeImage'])
 198     expected_summary_dict = {
 199         "header" : EXPECTED_HEADER_CONTENTS,
 200         "image-base-gs-url" : None,
 201         "descriptions" : None,
 202         "actual-results" : {
 203             "red.skp": {
 204                 "tiled-images": RED_TILES,
 205                 "whole-image": RED_WHOLEIMAGE,
 206             },
 207             "green.skp": {
 208                 "tiled-images": GREEN_TILES,
 209                 "whole-image": GREEN_WHOLEIMAGE,
 210             }
 211         }
 212     }
 213     self._assert_json_contents(output_json_path, expected_summary_dict)
 214     self._assert_directory_contents(
 215         write_path_dir, ['red_skp.png', 'green_skp.png'])
 216
 217   def test_ignore_some_failures(self):
 218     """test_tiled_whole_image, but ignoring some failed tests.
 219     """
 220     output_json_path = os.path.join(self._output_dir, 'actuals.json')
 221     write_path_dir = self.create_empty_dir(
 222         path=os.path.join(self._output_dir, 'writePath'))
 223     self._generate_skps()
 224     expectations_path = self._create_expectations(ignore_some_failures=True)
 225     self._run_render_pictures([
 226         '-r', self._input_skp_dir,
 227         '--bbh', 'grid', '256', '256',
 228         '--mode', 'tile', '256', '256',
 229         '--readJsonSummaryPath', expectations_path,
 230         '--writeJsonSummaryPath', output_json_path,
 231         '--writePath', write_path_dir,
 232         '--writeWholeImage'])
 233     modified_red_tiles = copy.deepcopy(RED_TILES)
 234     modified_red_tiles[5]['comparisonResult'] = 'failure-ignored'
 235     expected_summary_dict = {
 236         "header" : EXPECTED_HEADER_CONTENTS,
 237         "image-base-gs-url" : None,
 238         "descriptions" : None,
 239         "actual-results" : {
 240             "red.skp": {
 241                 "tiled-images": modified_red_tiles,
 242                 "whole-image": modified_dict(
 243                     RED_WHOLEIMAGE, {"comparisonResult" : "failure-ignored"}),
 244             },
 245             "green.skp": {
 246                 "tiled-images": GREEN_TILES,
 247                 "whole-image": GREEN_WHOLEIMAGE,
 248             }
 249         }
 250     }
 251     self._assert_json_contents(output_json_path, expected_summary_dict)
 252     self._assert_directory_contents(
 253         write_path_dir, ['red_skp.png', 'green_skp.png'])
 254
 255   def test_missing_tile_and_whole_image(self):
 256     """test_tiled_whole_image, but missing expectations for some images.
 257     """
 258     output_json_path = os.path.join(self._output_dir, 'actuals.json')
 259     write_path_dir = self.create_empty_dir(
 260         path=os.path.join(self._output_dir, 'writePath'))
 261     self._generate_skps()
 262     expectations_path = self._create_expectations(missing_some_images=True)
 263     self._run_render_pictures([
 264         '-r', self._input_skp_dir,
 265         '--bbh', 'grid', '256', '256',
 266         '--mode', 'tile', '256', '256',
 267         '--readJsonSummaryPath', expectations_path,
 268         '--writeJsonSummaryPath', output_json_path,
 269         '--writePath', write_path_dir,
 270         '--writeWholeImage'])
 271     modified_red_tiles = copy.deepcopy(RED_TILES)
 272     modified_red_tiles[5]['comparisonResult'] = 'no-comparison'
 273     expected_summary_dict = {
 274         "header" : EXPECTED_HEADER_CONTENTS,
 275         "image-base-gs-url" : None,
 276         "descriptions" : None,
 277         "actual-results" : {
 278             "red.skp": {
 279                 "tiled-images": modified_red_tiles,
 280                 "whole-image": modified_dict(
 281                     RED_WHOLEIMAGE, {"comparisonResult" : "no-comparison"}),
 282             },
 283             "green.skp": {
 284                 "tiled-images": GREEN_TILES,
 285                 "whole-image": GREEN_WHOLEIMAGE,
 286             }
 287         }
 288     }
 289     self._assert_json_contents(output_json_path, expected_summary_dict)
 290
 291   def _test_untiled(self, expectations_path=None, expected_summary_dict=None,
 292                     additional_args=None):
 293     """Base for multiple tests without tiles.
 294
 295     Args:
 296       expectations_path: path we should pass using --readJsonSummaryPath, or
 297           None if we should create the default expectations file
 298       expected_summary_dict: dict we should compare against the output actual
 299           results summary, or None if we should use a default comparison dict
 300       additional_args: array of command-line args to add when we run
 301           render_pictures
 302     """
 303     output_json_path = os.path.join(self._output_dir, 'actuals.json')
 304     write_path_dir = self.create_empty_dir(
 305         path=os.path.join(self._output_dir, 'writePath'))
 306     self._generate_skps()
 307     if expectations_path == None:
 308       expectations_path = self._create_expectations()
 309     args = [
 310         '-r', self._input_skp_dir,
 311         '--readJsonSummaryPath', expectations_path,
 312         '--writePath', write_path_dir,
 313         '--writeJsonSummaryPath', output_json_path,
 314     ]
 315     if additional_args:
 316       args.extend(additional_args)
 317     self._run_render_pictures(args)
 318     if expected_summary_dict == None:
 319       expected_summary_dict = {
 320           "header" : EXPECTED_HEADER_CONTENTS,
 321           "image-base-gs-url" : None,
 322           "descriptions" : None,
 323           "actual-results" : {
 324               "red.skp": {
 325                   "whole-image": RED_WHOLEIMAGE,
 326               },
 327               "green.skp": {
 328                   "whole-image": GREEN_WHOLEIMAGE,
 329               }
 330           }
 331       }
 332     self._assert_json_contents(output_json_path, expected_summary_dict)
 333     self._assert_directory_contents(
 334         write_path_dir, ['red_skp.png', 'green_skp.png'])
 335
 336   def test_untiled(self):
 337     """Basic test without tiles."""
 338     self._test_untiled()
 339
 340   def test_untiled_empty_expectations_file(self):
 341     """Same as test_untiled, but with an empty expectations file."""
 342     expectations_path = os.path.join(self._expectations_dir, 'empty')
 343     with open(expectations_path, 'w'):
 344       pass
 345     expected_summary_dict = {
 346         "header" : EXPECTED_HEADER_CONTENTS,
 347         "image-base-gs-url" : None,
 348         "descriptions" : None,
 349         "actual-results" : {
 350             "red.skp": {
 351                 "whole-image": modified_dict(
 352                     RED_WHOLEIMAGE, {"comparisonResult" : "no-comparison"}),
 353             },
 354             "green.skp": {
 355                 "whole-image": modified_dict(
 356                     GREEN_WHOLEIMAGE, {"comparisonResult" : "no-comparison"}),
 357             }
 358         }
 359     }
 360     self._test_untiled(expectations_path=expectations_path,
 361                        expected_summary_dict=expected_summary_dict)
 362
 363   def test_untiled_writeChecksumBasedFilenames(self):
 364     """Same as test_untiled, but with --writeChecksumBasedFilenames."""
 365     output_json_path = os.path.join(self._output_dir, 'actuals.json')
 366     write_path_dir = self.create_empty_dir(
 367         path=os.path.join(self._output_dir, 'writePath'))
 368     self._generate_skps()
 369     self._run_render_pictures([
 370         '-r', self._input_skp_dir,
 371         '--descriptions', 'builder=builderName', 'renderMode=renderModeName',
 372         '--writeChecksumBasedFilenames',
 373         '--writePath', write_path_dir,
 374         '--writeJsonSummaryPath', output_json_path,
 375     ])
 376     expected_summary_dict = {
 377         "header" : EXPECTED_HEADER_CONTENTS,
 378         "image-base-gs-url" : None,
 379         "descriptions" : {
 380             "builder": "builderName",
 381             "renderMode": "renderModeName",
 382         },
 383         "actual-results" : {
 384             "red.skp": {
 385                 # Manually verified: 640x400 red rectangle with black border
 386                 "whole-image": {
 387                     "checksumAlgorithm" : "bitmap-64bitMD5",
 388                     "checksumValue" : 2853310525600416231,
 389                     "comparisonResult" : "no-comparison",
 390                     "filepath" :
 391                         "red_skp/bitmap-64bitMD5_2853310525600416231.png",
 392                 },
 393             },
 394             "green.skp": {
 395                 # Manually verified: 640x400 green rectangle with black border
 396                 "whole-image": {
 397                     "checksumAlgorithm" : "bitmap-64bitMD5",
 398                     "checksumValue" : 11143979097452425335,
 399                     "comparisonResult" : "no-comparison",
 400                     "filepath" :
 401                         "green_skp/bitmap-64bitMD5_11143979097452425335.png",
 402                 },
 403             }
 404         }
 405     }
 406     self._assert_json_contents(output_json_path, expected_summary_dict)
 407     self._assert_directory_contents(write_path_dir, ['red_skp', 'green_skp'])
 408     self._assert_directory_contents(
 409         os.path.join(write_path_dir, 'red_skp'),
 410         ['bitmap-64bitMD5_2853310525600416231.png'])
 411     self._assert_directory_contents(
 412         os.path.join(write_path_dir, 'green_skp'),
 413         ['bitmap-64bitMD5_11143979097452425335.png'])
 414
 415   def test_untiled_validate(self):
 416     """Same as test_untiled, but with --validate."""
 417     self._test_untiled(additional_args=['--validate'])
 418
 419   def test_untiled_without_writePath(self):
 420     """Same as test_untiled, but without --writePath."""
 421     output_json_path = os.path.join(self._output_dir, 'actuals.json')
 422     self._generate_skps()
 423     expectations_path = self._create_expectations()
 424     self._run_render_pictures([
 425         '-r', self._input_skp_dir,
 426         '--readJsonSummaryPath', expectations_path,
 427         '--writeJsonSummaryPath', output_json_path])
 428     expected_summary_dict = {
 429         "header" : EXPECTED_HEADER_CONTENTS,
 430         "image-base-gs-url" : None,
 431         "descriptions" : None,
 432         "actual-results" : {
 433             "red.skp": {
 434                 "whole-image": RED_WHOLEIMAGE,
 435             },
 436             "green.skp": {
 437                 "whole-image": GREEN_WHOLEIMAGE,
 438             }
 439         }
 440     }
 441     self._assert_json_contents(output_json_path, expected_summary_dict)
 442
 443   def test_tiled(self):
 444     """Generate individual tiles."""
 445     output_json_path = os.path.join(self._output_dir, 'actuals.json')
 446     write_path_dir = self.create_empty_dir(
 447         path=os.path.join(self._output_dir, 'writePath'))
 448     self._generate_skps()
 449     expectations_path = self._create_expectations()
 450     self._run_render_pictures([
 451         '-r', self._input_skp_dir,
 452         '--bbh', 'grid', '256', '256',
 453         '--mode', 'tile', '256', '256',
 454         '--readJsonSummaryPath', expectations_path,
 455         '--writePath', write_path_dir,
 456         '--writeJsonSummaryPath', output_json_path])
 457     expected_summary_dict = {
 458         "header" : EXPECTED_HEADER_CONTENTS,
 459         "image-base-gs-url" : None,
 460         "descriptions" : None,
 461         "actual-results" : {
 462             "red.skp": {
 463                 "tiled-images": RED_TILES,
 464             },
 465             "green.skp": {
 466                 "tiled-images": GREEN_TILES,
 467             }
 468         }
 469     }
 470     self._assert_json_contents(output_json_path, expected_summary_dict)
 471     self._assert_directory_contents(
 472         write_path_dir,
 473         ['red_skp-tile0.png', 'red_skp-tile1.png', 'red_skp-tile2.png',
 474          'red_skp-tile3.png', 'red_skp-tile4.png', 'red_skp-tile5.png',
 475          'green_skp-tile0.png', 'green_skp-tile1.png', 'green_skp-tile2.png',
 476          'green_skp-tile3.png', 'green_skp-tile4.png', 'green_skp-tile5.png',
 477         ])
 478
 479   def test_tiled_mismatches(self):
 480     """Same as test_tiled, but only write out mismatching images."""
 481     output_json_path = os.path.join(self._output_dir, 'actuals.json')
 482     mismatch_path_dir = self.create_empty_dir(
 483         path=os.path.join(self._output_dir, 'mismatchPath'))
 484     self._generate_skps()
 485     expectations_path = self._create_expectations()
 486     self._run_render_pictures([
 487         '-r', self._input_skp_dir,
 488         '--bbh', 'grid', '256', '256',
 489         '--mode', 'tile', '256', '256',
 490         '--readJsonSummaryPath', expectations_path,
 491         '--mismatchPath', mismatch_path_dir,
 492         '--writeJsonSummaryPath', output_json_path])
 493     expected_summary_dict = {
 494         "header" : EXPECTED_HEADER_CONTENTS,
 495         "image-base-gs-url" : None,
 496         "descriptions" : None,
 497         "actual-results" : {
 498             "red.skp": {
 499                 "tiled-images": RED_TILES,
 500             },
 501             "green.skp": {
 502                 "tiled-images": GREEN_TILES,
 503             }
 504         }
 505     }
 506     self._assert_json_contents(output_json_path, expected_summary_dict)
 507     self._assert_directory_contents(
 508         mismatch_path_dir,
 509         ['red_skp-tile0.png', 'red_skp-tile1.png', 'red_skp-tile2.png',
 510          'red_skp-tile3.png', 'red_skp-tile4.png', 'red_skp-tile5.png',
 511         ])
 512
 513   def test_tiled_writeChecksumBasedFilenames(self):
 514     """Same as test_tiled, but with --writeChecksumBasedFilenames."""
 515     output_json_path = os.path.join(self._output_dir, 'actuals.json')
 516     write_path_dir = self.create_empty_dir(
 517         path=os.path.join(self._output_dir, 'writePath'))
 518     self._generate_skps()
 519     self._run_render_pictures(['-r', self._input_skp_dir,
 520                                '--bbh', 'grid', '256', '256',
 521                                '--mode', 'tile', '256', '256',
 522                                '--writeChecksumBasedFilenames',
 523                                '--writePath', write_path_dir,
 524                                '--writeJsonSummaryPath', output_json_path])
 525     expected_summary_dict = {
 526         "header" : EXPECTED_HEADER_CONTENTS,
 527         "image-base-gs-url" : None,
 528         "descriptions" : None,
 529         "actual-results" : {
 530             "red.skp": {
 531                 # Manually verified these 6 images, all 256x256 tiles,
 532                 # consistent with a tiled version of the 640x400 red rect
 533                 # with black borders.
 534                 "tiled-images": [{
 535                     "checksumAlgorithm" : "bitmap-64bitMD5",
 536                     "checksumValue" : 5815827069051002745,
 537                     "comparisonResult" : "no-comparison",
 538                     "filepath" :
 539                         "red_skp/bitmap-64bitMD5_5815827069051002745.png",
 540                 }, {
 541                     "checksumAlgorithm" : "bitmap-64bitMD5",
 542                     "checksumValue" : 9323613075234140270,
 543                     "comparisonResult" : "no-comparison",
 544                     "filepath" :
 545                         "red_skp/bitmap-64bitMD5_9323613075234140270.png",
 546                 }, {
 547                     "checksumAlgorithm" : "bitmap-64bitMD5",
 548                     "checksumValue" : 15939355025996362179,
 549                     "comparisonResult" : "no-comparison",
 550                     "filepath" :
 551                         "red_skp/bitmap-64bitMD5_15939355025996362179.png",
 552                 }, {
 553                     "checksumAlgorithm" : "bitmap-64bitMD5",
 554                     "checksumValue" : 649771916797529222,
 555                     "comparisonResult" : "no-comparison",
 556                     "filepath" :
 557                         "red_skp/bitmap-64bitMD5_649771916797529222.png",
 558                 }, {
 559                     "checksumAlgorithm" : "bitmap-64bitMD5",
 560                     "checksumValue" : 8132820002266077288,
 561                     "comparisonResult" : "no-comparison",
 562                     "filepath" :
 563                         "red_skp/bitmap-64bitMD5_8132820002266077288.png",
 564                 }, {
 565                     "checksumAlgorithm" : "bitmap-64bitMD5",
 566                     "checksumValue" : 2406160701181324581,
 567                     "comparisonResult" : "no-comparison",
 568                     "filepath" :
 569                         "red_skp/bitmap-64bitMD5_2406160701181324581.png",
 570                 }],
 571             },
 572             "green.skp": {
 573                 # Manually verified these 6 images, all 256x256 tiles,
 574                 # consistent with a tiled version of the 640x400 green rect
 575                 # with black borders.
 576                 "tiled-images": [{
 577                     "checksumAlgorithm" : "bitmap-64bitMD5",
 578                     "checksumValue" : 12587324416545178013,
 579                     "comparisonResult" : "no-comparison",
 580                     "filepath" :
 581                         "green_skp/bitmap-64bitMD5_12587324416545178013.png",
 582                 }, {
 583                     "checksumAlgorithm" : "bitmap-64bitMD5",
 584                     "checksumValue" : 7624374914829746293,
 585                     "comparisonResult" : "no-comparison",
 586                     "filepath" :
 587                         "green_skp/bitmap-64bitMD5_7624374914829746293.png",
 588                 }, {
 589                     "checksumAlgorithm" : "bitmap-64bitMD5",
 590                     "checksumValue" : 11866144860997809880,
 591                     "comparisonResult" : "no-comparison",
 592                     "filepath" :
 593                         "green_skp/bitmap-64bitMD5_11866144860997809880.png",
 594                 }, {
 595                     "checksumAlgorithm" : "bitmap-64bitMD5",
 596                     "checksumValue" : 3893392565127823822,
 597                     "comparisonResult" : "no-comparison",
 598                     "filepath" :
 599                         "green_skp/bitmap-64bitMD5_3893392565127823822.png",
 600                 }, {
 601                     "checksumAlgorithm" : "bitmap-64bitMD5",
 602                     "checksumValue" : 2083084978343901738,
 603                     "comparisonResult" : "no-comparison",
 604                     "filepath" :
 605                         "green_skp/bitmap-64bitMD5_2083084978343901738.png",
 606                 }, {
 607                     "checksumAlgorithm" : "bitmap-64bitMD5",
 608                     "checksumValue" : 89620927366502076,
 609                     "comparisonResult" : "no-comparison",
 610                     "filepath" :
 611                         "green_skp/bitmap-64bitMD5_89620927366502076.png",
 612                 }],
 613             }
 614         }
 615     }
 616     self._assert_json_contents(output_json_path, expected_summary_dict)
 617     self._assert_directory_contents(write_path_dir, ['red_skp', 'green_skp'])
 618     self._assert_directory_contents(
 619         os.path.join(write_path_dir, 'red_skp'),
 620         ['bitmap-64bitMD5_5815827069051002745.png',
 621          'bitmap-64bitMD5_9323613075234140270.png',
 622          'bitmap-64bitMD5_15939355025996362179.png',
 623          'bitmap-64bitMD5_649771916797529222.png',
 624          'bitmap-64bitMD5_8132820002266077288.png',
 625          'bitmap-64bitMD5_2406160701181324581.png'])
 626     self._assert_directory_contents(
 627         os.path.join(write_path_dir, 'green_skp'),
 628         ['bitmap-64bitMD5_12587324416545178013.png',
 629          'bitmap-64bitMD5_7624374914829746293.png',
 630          'bitmap-64bitMD5_11866144860997809880.png',
 631          'bitmap-64bitMD5_3893392565127823822.png',
 632          'bitmap-64bitMD5_2083084978343901738.png',
 633          'bitmap-64bitMD5_89620927366502076.png'])
 634
 635   def _run_render_pictures(self, args):
 636     binary = find_run_binary.find_path_to_program('render_pictures')
 637     return find_run_binary.run_command(
 638         [binary, '--config', '8888'] + args)
 639
 640   def _create_expectations(self, missing_some_images=False,
 641                            ignore_some_failures=False,
 642                            rel_path='expectations.json'):
 643     """Creates expectations JSON file within self._expectations_dir .
 644
 645     Args:
 646       missing_some_images: (bool) whether to remove expectations for a subset
 647           of the images
 648       ignore_some_failures: (bool) whether to ignore some failing tests
 649       rel_path: (string) relative path within self._expectations_dir to write
 650           the expectations into
 651
 652     Returns: full path to the expectations file created.
 653     """
 654     expectations_dict = {
 655         "header" : EXPECTED_HEADER_CONTENTS,
 656         "descriptions" : None,
 657         "expected-results" : {
 658             # red.skp: these should fail the comparison
 659             "red.skp": {
 660                 "tiled-images": modified_list_of_dicts(
 661                     RED_TILES, {'checksumValue': 11111}),
 662                 "whole-image": modified_dict(
 663                     RED_WHOLEIMAGE, {'checksumValue': 22222}),
 664             },
 665             # green.skp: these should pass the comparison
 666             "green.skp": {
 667                 "tiled-images": GREEN_TILES,
 668                 "whole-image": GREEN_WHOLEIMAGE,
 669             }
 670         }
 671     }
 672     if missing_some_images:
 673       red_subdict = expectations_dict['expected-results']['red.skp']
 674       del red_subdict['whole-image']
 675       del red_subdict['tiled-images'][-1]
 676     elif ignore_some_failures:
 677       red_subdict = expectations_dict['expected-results']['red.skp']
 678       red_subdict['whole-image']['ignoreFailure'] = True
 679       red_subdict['tiled-images'][-1]['ignoreFailure'] = True
 680     path = os.path.join(self._expectations_dir, rel_path)
 681     with open(path, 'w') as fh:
 682       json.dump(expectations_dict, fh)
 683     return path
 684
 685   def _generate_skps(self):
 686     """Runs the skpmaker binary to generate files in self._input_skp_dir."""
 687     self._run_skpmaker(
 688         output_path=os.path.join(self._input_skp_dir, 'red.skp'), red=255)
 689     self._run_skpmaker(
 690         output_path=os.path.join(self._input_skp_dir, 'green.skp'), green=255)
 691
 692   def _run_skpmaker(self, output_path, red=0, green=0, blue=0,
 693                     width=640, height=400):
 694     """Runs the skpmaker binary to generate SKP with known characteristics.
 695
 696     Args:
 697       output_path: Filepath to write the SKP into.
 698       red: Value of red color channel in image, 0-255.
 699       green: Value of green color channel in image, 0-255.
 700       blue: Value of blue color channel in image, 0-255.
 701       width: Width of canvas to create.
 702       height: Height of canvas to create.
 703     """
 704     binary = find_run_binary.find_path_to_program('skpmaker')
 705     return find_run_binary.run_command([
 706         binary,
 707         '--red', str(red),
 708         '--green', str(green),
 709         '--blue', str(blue),
 710         '--width', str(width),
 711         '--height', str(height),
 712         '--writePath', str(output_path),
 713     ])
 714
 715   def _assert_directory_contents(self, dir_path, expected_filenames):
 716     """Asserts that files found in a dir are identical to expected_filenames.
 717
 718     Args:
 719       dir_path: Path to a directory on local disk.
 720       expected_filenames: Set containing the expected filenames within the dir.
 721
 722     Raises:
 723       AssertionError: contents of the directory are not identical to
 724                       expected_filenames.
 725     """
 726     self.assertEqual(set(os.listdir(dir_path)), set(expected_filenames))
 727
 728   def _assert_json_contents(self, json_path, expected_dict):
 729     """Asserts that contents of a JSON file are identical to expected_dict.
 730
 731     Args:
 732       json_path: Path to a JSON file.
 733       expected_dict: Dictionary indicating the expected contents of the JSON
 734                      file.
 735
 736     Raises:
 737       AssertionError: contents of the JSON file are not identical to
 738                       expected_dict.
 739     """
 740     prettyprinted_expected_dict = json.dumps(expected_dict, sort_keys=True,
 741                                              indent=2)
 742     with open(json_path, 'r') as fh:
 743       prettyprinted_json_dict = json.dumps(json.load(fh), sort_keys=True,
 744                                            indent=2)
 745     self.assertMultiLineEqual(prettyprinted_expected_dict,
 746                               prettyprinted_json_dict)
 747
 748
 749 def main():
 750   base_unittest.main(RenderPicturesTest)
 751
 752
 753 if __name__ == '__main__':
 754   main()