From: commit-bot@chromium.org Date: Wed, 7 May 2014 19:29:03 +0000 (+0000) Subject: make compare_rendered_pictures_test.py run end-to-end test X-Git-Tag: accepted/tizen/5.0/unified/20181102.025319~7839 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=a60d0370b11b2905059be9f6d84b09da7ee2b4fb;p=platform%2Fupstream%2FlibSkiaSharp.git make compare_rendered_pictures_test.py run end-to-end test 1. generate test SKPs using skpmaker 2. render those SKPs using render_pictures 3. compare results using compare_rendered_pictures.py BUG=skia:1942,skia:2230 NOTRY=True R=borenet@google.com Author: epoger@google.com Review URL: https://codereview.chromium.org/274623002 git-svn-id: http://skia.googlecode.com/svn/trunk@14624 2bbb7eff-a529-9590-31e7-b0007b416f81 --- diff --git a/gm/rebaseline_server/base_unittest.py b/gm/rebaseline_server/base_unittest.py index eec0c8d..0699db6 100755 --- a/gm/rebaseline_server/base_unittest.py +++ b/gm/rebaseline_server/base_unittest.py @@ -17,6 +17,7 @@ import tempfile import unittest PARENT_DIR = os.path.dirname(os.path.realpath(__file__)) +TRUNK_DIR = os.path.dirname(os.path.dirname(PARENT_DIR)) TESTDATA_DIR = os.path.join(PARENT_DIR, 'testdata') OUTPUT_DIR_ACTUAL = os.path.join(TESTDATA_DIR, 'outputs', 'actual') OUTPUT_DIR_EXPECTED = os.path.join(TESTDATA_DIR, 'outputs', 'expected') @@ -59,6 +60,30 @@ class TestCase(unittest.TestCase): """Tell unittest framework to not print docstrings for test cases.""" return None + def find_path_to_program(self, program): + """Returns path to an existing program binary. + + Args: + program: Basename of the program to find (e.g., 'render_pictures'). + + Returns: + Absolute path to the program binary, as a string. + + Raises: + Exception: unable to find the program binary. + """ + possible_paths = [os.path.join(TRUNK_DIR, 'out', 'Release', program), + os.path.join(TRUNK_DIR, 'out', 'Debug', program), + os.path.join(TRUNK_DIR, 'out', 'Release', + program + '.exe'), + os.path.join(TRUNK_DIR, 'out', 'Debug', + program + '.exe')] + for try_path in possible_paths: + if os.path.isfile(try_path): + return try_path + raise Exception('cannot find %s in paths %s; maybe you need to ' + 'build %s?' % (program, possible_paths, program)) + def create_empty_dir(path): """Create an empty directory at the given path.""" diff --git a/gm/rebaseline_server/compare_rendered_pictures_test.py b/gm/rebaseline_server/compare_rendered_pictures_test.py index c1eebb7..e6d2574 100755 --- a/gm/rebaseline_server/compare_rendered_pictures_test.py +++ b/gm/rebaseline_server/compare_rendered_pictures_test.py @@ -19,6 +19,7 @@ within self._output_dir_expected, which wouldn't be good... """ import os +import subprocess import sys # Imports from within Skia @@ -31,22 +32,80 @@ import gm_json # must import results first, so that gm_json will be in sys.path class CompareRenderedPicturesTest(base_unittest.TestCase): def test_endToEnd(self): - """Compare results of two render_pictures runs.""" - # TODO(epoger): Specify image_base_url pointing at the directory on local - # disk containing our test images, so that we can actually compute pixel - # diffs. For now, this test attempts to download images from - # DEFAULT_IMAGE_BASE_URL, and there aren't any there yet. + """Generate two sets of SKPs, run render_pictures over both, and compare + the results.""" + self._generate_skps_and_run_render_pictures( + subdir='before_patch', skpdict={ + 'changed.skp': 200, + 'unchanged.skp': 100, + 'only-in-before.skp': 128, + }) + self._generate_skps_and_run_render_pictures( + subdir='after_patch', skpdict={ + 'changed.skp': 201, + 'unchanged.skp': 100, + 'only-in-after.skp': 128, + }) + results_obj = compare_rendered_pictures.RenderedPicturesComparisons( - actuals_root=os.path.join(self._input_dir, 'render_pictures_output'), + actuals_root=self._temp_dir, subdirs=('before_patch', 'after_patch'), generated_images_root=self._temp_dir, diff_base_url='/static/generated-images') results_obj.get_timestamp = mock_get_timestamp + gm_json.WriteToFile( results_obj.get_packaged_results_of_type( results.KEY__HEADER__RESULTS_ALL), os.path.join(self._output_dir_actual, 'compare_rendered_pictures.json')) + def _generate_skps_and_run_render_pictures(self, subdir, skpdict): + """Generate SKPs and run render_pictures on them. + + Args: + subdir: subdirectory (within self._temp_dir) to write all files into + skpdict: {skpname: redvalue} dictionary describing the SKP files to render + """ + out_path = os.path.join(self._temp_dir, subdir) + os.makedirs(out_path) + for skpname, redvalue in skpdict.iteritems(): + self._run_skpmaker( + output_path=os.path.join(out_path, skpname), red=redvalue) + + # TODO(epoger): Add --mode tile 256 256 --writeWholeImage to the unittest, + # and fix its result! (imageURLs within whole-image entries are wrong when + # I tried adding that) + binary = self.find_path_to_program('render_pictures') + return subprocess.check_output([ + binary, + '--clone', '1', + '--config', '8888', + '-r', out_path, + '--writeChecksumBasedFilenames', + '--writeJsonSummaryPath', os.path.join(out_path, 'summary.json'), + '--writePath', out_path]) + + def _run_skpmaker(self, output_path, red=0, green=0, blue=0, + width=640, height=400): + """Runs the skpmaker binary to generate SKP with known characteristics. + + Args: + output_path: Filepath to write the SKP into. + red: Value of red color channel in image, 0-255. + green: Value of green color channel in image, 0-255. + blue: Value of blue color channel in image, 0-255. + width: Width of canvas to create. + height: Height of canvas to create. + """ + binary = self.find_path_to_program('skpmaker') + return subprocess.check_output([ + binary, + '--red', str(red), + '--green', str(green), + '--blue', str(blue), + '--width', str(width), + '--height', str(height), + '--writePath', str(output_path)]) def mock_get_timestamp(): """Mock version of BaseComparisons.get_timestamp() for testing.""" diff --git a/gm/rebaseline_server/testdata/inputs/render_pictures_output/.gitattributes b/gm/rebaseline_server/testdata/inputs/render_pictures_output/.gitattributes deleted file mode 100644 index 947f120..0000000 --- a/gm/rebaseline_server/testdata/inputs/render_pictures_output/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -*.png binary diff --git a/gm/rebaseline_server/testdata/inputs/render_pictures_output/after_patch/builder1/bitmap-64bitMD5_11092453015575919668.png b/gm/rebaseline_server/testdata/inputs/render_pictures_output/after_patch/builder1/bitmap-64bitMD5_11092453015575919668.png deleted file mode 100644 index 349c961..0000000 Binary files a/gm/rebaseline_server/testdata/inputs/render_pictures_output/after_patch/builder1/bitmap-64bitMD5_11092453015575919668.png and /dev/null differ diff --git a/gm/rebaseline_server/testdata/inputs/render_pictures_output/after_patch/builder1/bitmap-64bitMD5_2520753504544298264.png b/gm/rebaseline_server/testdata/inputs/render_pictures_output/after_patch/builder1/bitmap-64bitMD5_2520753504544298264.png deleted file mode 100644 index 321aca3..0000000 Binary files a/gm/rebaseline_server/testdata/inputs/render_pictures_output/after_patch/builder1/bitmap-64bitMD5_2520753504544298264.png and /dev/null differ diff --git a/gm/rebaseline_server/testdata/inputs/render_pictures_output/after_patch/builder1/output.json b/gm/rebaseline_server/testdata/inputs/render_pictures_output/after_patch/builder1/output.json deleted file mode 100644 index 9bd6607..0000000 --- a/gm/rebaseline_server/testdata/inputs/render_pictures_output/after_patch/builder1/output.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "header" : { - "type" : "ChecksummedImages", - "revision" : 1 - }, - "actual-results" : { - "changed.skp" : { - "whole-image" : { - "checksumAlgorithm" : "bitmap-64bitMD5", - "checksumValue" : 2520753504544298264, - "comparisonResult" : "no-comparison", - "filepath" : "bitmap-64bitMD5_2520753504544298264.png" - } - }, - "unchanged.skp" : { - "whole-image" : { - "checksumAlgorithm" : "bitmap-64bitMD5", - "checksumValue" : 11092453015575919668, - "comparisonResult" : "no-comparison", - "filepath" : "bitmap-64bitMD5_11092453015575919668.png" - } - } - } -} diff --git a/gm/rebaseline_server/testdata/inputs/render_pictures_output/before_patch/builder1/bitmap-64bitMD5_11092453015575919668.png b/gm/rebaseline_server/testdata/inputs/render_pictures_output/before_patch/builder1/bitmap-64bitMD5_11092453015575919668.png deleted file mode 100644 index 349c961..0000000 Binary files a/gm/rebaseline_server/testdata/inputs/render_pictures_output/before_patch/builder1/bitmap-64bitMD5_11092453015575919668.png and /dev/null differ diff --git a/gm/rebaseline_server/testdata/inputs/render_pictures_output/before_patch/builder1/bitmap-64bitMD5_8891695120562235492.png b/gm/rebaseline_server/testdata/inputs/render_pictures_output/before_patch/builder1/bitmap-64bitMD5_8891695120562235492.png deleted file mode 100644 index 67168c9..0000000 Binary files a/gm/rebaseline_server/testdata/inputs/render_pictures_output/before_patch/builder1/bitmap-64bitMD5_8891695120562235492.png and /dev/null differ diff --git a/gm/rebaseline_server/testdata/inputs/render_pictures_output/before_patch/builder1/output.json b/gm/rebaseline_server/testdata/inputs/render_pictures_output/before_patch/builder1/output.json deleted file mode 100644 index 8bc81c8..0000000 --- a/gm/rebaseline_server/testdata/inputs/render_pictures_output/before_patch/builder1/output.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "header" : { - "type" : "ChecksummedImages", - "revision" : 1 - }, - "actual-results" : { - "changed.skp" : { - "whole-image" : { - "checksumAlgorithm" : "bitmap-64bitMD5", - "checksumValue" : 8891695120562235492, - "comparisonResult" : "no-comparison", - "filepath" : "bitmap-64bitMD5_8891695120562235492.png" - } - }, - "only-in-before.skp" : { - "whole-image" : { - "checksumAlgorithm" : "bitmap-64bitMD5", - "checksumValue" : 8891695120562235492, - "comparisonResult" : "no-comparison", - "filepath" : "bitmap-64bitMD5_8891695120562235492.png" - } - }, - "unchanged.skp" : { - "whole-image" : { - "checksumAlgorithm" : "bitmap-64bitMD5", - "checksumValue" : 11092453015575919668, - "comparisonResult" : "no-comparison", - "filepath" : "bitmap-64bitMD5_11092453015575919668.png" - } - } - } -} diff --git a/gm/rebaseline_server/testdata/outputs/expected/compare_rendered_pictures_test.CompareRenderedPicturesTest.test_endToEnd/compare_rendered_pictures.json b/gm/rebaseline_server/testdata/outputs/expected/compare_rendered_pictures_test.CompareRenderedPicturesTest.test_endToEnd/compare_rendered_pictures.json index 7a0c912..fbe48c0 100644 --- a/gm/rebaseline_server/testdata/outputs/expected/compare_rendered_pictures_test.CompareRenderedPicturesTest.test_endToEnd/compare_rendered_pictures.json +++ b/gm/rebaseline_server/testdata/outputs/expected/compare_rendered_pictures_test.CompareRenderedPicturesTest.test_endToEnd/compare_rendered_pictures.json @@ -5,7 +5,7 @@ "isFilterable": true, "isSortable": true, "valuesAndCounts": { - "TODO": 3 + "TODO": 4 } }, "config": { @@ -13,7 +13,7 @@ "isFilterable": true, "isSortable": true, "valuesAndCounts": { - "whole-image": 3 + "whole-image": 4 } }, "resultType": { @@ -22,7 +22,7 @@ "isSortable": true, "valuesAndCounts": { "failed": 1, - "no-comparison": 1, + "no-comparison": 2, "succeeded": 1 } }, @@ -32,13 +32,14 @@ "isSortable": true, "valuesAndCounts": { "changed.skp": 1, + "only-in-after.skp": 1, "only-in-before.skp": 1, "unchanged.skp": 1 } } }, "header": { - "dataHash": "182723807383859624", + "dataHash": "-595743736412687673", "isEditable": false, "isExported": true, "schemaVersion": 2, @@ -54,8 +55,19 @@ "resultType": "failed", "test": "changed.skp" }, - "imageAUrl": "bitmap-64bitMD5_8891695120562235492.png", - "imageBUrl": "bitmap-64bitMD5_2520753504544298264.png", + "imageAUrl": "changed_skp/bitmap-64bitMD5_3101044995537104462.png", + "imageBUrl": "changed_skp/bitmap-64bitMD5_13623922271964399662.png", + "isDifferent": true + }, + { + "extraColumns": { + "builder": "TODO", + "config": "whole-image", + "resultType": "no-comparison", + "test": "only-in-after.skp" + }, + "imageAUrl": null, + "imageBUrl": "only-in-after_skp/bitmap-64bitMD5_2320185040577047131.png", "isDifferent": true }, { @@ -65,7 +77,7 @@ "resultType": "no-comparison", "test": "only-in-before.skp" }, - "imageAUrl": "bitmap-64bitMD5_8891695120562235492.png", + "imageAUrl": "only-in-before_skp/bitmap-64bitMD5_2320185040577047131.png", "imageBUrl": null, "isDifferent": true }, @@ -76,8 +88,8 @@ "resultType": "succeeded", "test": "unchanged.skp" }, - "imageAUrl": "bitmap-64bitMD5_11092453015575919668.png", - "imageBUrl": "bitmap-64bitMD5_11092453015575919668.png", + "imageAUrl": "unchanged_skp/bitmap-64bitMD5_3322248763049618493.png", + "imageBUrl": "unchanged_skp/bitmap-64bitMD5_3322248763049618493.png", "isDifferent": false } ],