From 7909f47b423dacaff8623e8be247586108c3be66 Mon Sep 17 00:00:00 2001 From: epoger Date: Wed, 6 Aug 2014 08:37:53 -0700 Subject: [PATCH] rebaseline_server live queries: allow comparison against summary files within the Skia repo This is a step towards providing SKP comparisons against expectations, not just other actual results... we still need to allow the user to select JSONKEY_EXPECTEDRESULTS vs JSONKEY_ACTUALRESULTS within the summary files found. BUG=skia:1942 NOTREECHECKS=true NOTRY=true R=rmistry@google.com Author: epoger@google.com Review URL: https://codereview.chromium.org/442203002 --- gm/rebaseline_server/compare_rendered_pictures.py | 18 ++- .../compare_rendered_pictures_test.py | 18 +++ .../testdata/inputs/skp-actuals/setA/summary.json | 32 +++++ .../testdata/inputs/skp-actuals/setB/summary.json | 32 +++++ .../compare_rendered_pictures.json | 151 +++++++++++++++++++++ 5 files changed, 248 insertions(+), 3 deletions(-) create mode 100644 gm/rebaseline_server/testdata/inputs/skp-actuals/setA/summary.json create mode 100644 gm/rebaseline_server/testdata/inputs/skp-actuals/setB/summary.json create mode 100644 gm/rebaseline_server/testdata/outputs/expected/compare_rendered_pictures_test.CompareRenderedPicturesTest.test_repo_url/compare_rendered_pictures.json diff --git a/gm/rebaseline_server/compare_rendered_pictures.py b/gm/rebaseline_server/compare_rendered_pictures.py index 907ea63..ed36c7c 100755 --- a/gm/rebaseline_server/compare_rendered_pictures.py +++ b/gm/rebaseline_server/compare_rendered_pictures.py @@ -55,6 +55,12 @@ ORDERED_COLUMN_IDS = [ COLUMN__TILENUM, ] +# A special "repo:" URL type that we use to refer to Skia repo contents. +# (Useful for comparing against expectations files we store in our repo.) +REPO_URL_PREFIX = 'repo:' +REPO_BASEPATH = os.path.abspath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir)) + class RenderedPicturesComparisons(results.BaseComparisons): """Loads results from multiple render_pictures runs into an ImagePairSet. @@ -68,9 +74,11 @@ class RenderedPicturesComparisons(results.BaseComparisons): """ Args: setA_dirs: list of root directories to copy all JSON summaries from, - and to use as setA within the comparisons + and to use as setA within the comparisons. These directories may be + gs:// URLs, special "repo:" URLs, or local filepaths. setB_dirs: list of root directories to copy all JSON summaries from, - and to use as setB within the comparisons + and to use as setB within the comparisons. These directories may be + gs:// URLs, special "repo:" URLs, or local filepaths. image_diff_db: ImageDiffDB instance image_base_gs_url: "gs://" URL pointing at the Google Storage bucket/dir under which all render_pictures result images can @@ -325,7 +333,8 @@ class RenderedPicturesComparisons(results.BaseComparisons): """Copy all contents of source_dir into dest_dir, recursing into subdirs. Args: - source_dir: path to source dir (GS URL or local filepath) + source_dir: path to source dir (GS URL, local filepath, or a special + "repo:" URL type that points at a file within our Skia checkout) dest_dir: path to destination dir (local filepath) The copy operates as a "merge with overwrite": any files in source_dir will @@ -336,5 +345,8 @@ class RenderedPicturesComparisons(results.BaseComparisons): (bucket, path) = gs_utils.GSUtils.split_gs_url(source_dir) self._gs.download_dir_contents(source_bucket=bucket, source_dir=path, dest_dir=dest_dir) + elif source_dir.lower().startswith(REPO_URL_PREFIX): + repo_dir = os.path.join(REPO_BASEPATH, source_dir[len(REPO_URL_PREFIX):]) + shutil.copytree(repo_dir, dest_dir) else: shutil.copytree(source_dir, dest_dir) diff --git a/gm/rebaseline_server/compare_rendered_pictures_test.py b/gm/rebaseline_server/compare_rendered_pictures_test.py index 5ddaf10..c6e0e79 100755 --- a/gm/rebaseline_server/compare_rendered_pictures_test.py +++ b/gm/rebaseline_server/compare_rendered_pictures_test.py @@ -68,6 +68,24 @@ class CompareRenderedPicturesTest(base_unittest.TestCase): results.KEY__HEADER__RESULTS_ALL), os.path.join(self.output_dir_actual, 'compare_rendered_pictures.json')) + def test_repo_url(self): + """Use repo: URL to specify summary files.""" + results_obj = compare_rendered_pictures.RenderedPicturesComparisons( + setA_dirs=[ + 'repo:gm/rebaseline_server/testdata/inputs/skp-actuals/setA'], + setB_dirs=[ + 'repo:gm/rebaseline_server/testdata/inputs/skp-actuals/setB'], + image_diff_db=imagediffdb.ImageDiffDB(self.temp_dir), + image_base_gs_url='gs://fakebucket/fake/path', + diff_base_url='/static/generated-images', + setA_label='setA', setB_label='setB') + results_obj.get_timestamp = mock_get_timestamp + + gm_json.WriteToFile( + results_obj.get_packaged_results_of_type( + results.KEY__HEADER__RESULTS_ALL), + os.path.join(self.output_dir_actual, 'compare_rendered_pictures.json')) + def _generate_skps_and_run_render_pictures(self, subdir, skpdict): """Generate SKPs and run render_pictures on them. diff --git a/gm/rebaseline_server/testdata/inputs/skp-actuals/setA/summary.json b/gm/rebaseline_server/testdata/inputs/skp-actuals/setA/summary.json new file mode 100644 index 0000000..d740107 --- /dev/null +++ b/gm/rebaseline_server/testdata/inputs/skp-actuals/setA/summary.json @@ -0,0 +1,32 @@ +{ + "actual-results" : { + "changed.skp" : { + "whole-image" : { + "checksumAlgorithm" : "bitmap-64bitMD5", + "checksumValue" : 3101044995537104462, + "comparisonResult" : "no-comparison", + "filepath" : "changed_skp/bitmap-64bitMD5_3101044995537104462.png" + } + }, + "only-in-before.skp" : { + "whole-image" : { + "checksumAlgorithm" : "bitmap-64bitMD5", + "checksumValue" : 2320185040577047131, + "comparisonResult" : "no-comparison", + "filepath" : "only-in-before_skp/bitmap-64bitMD5_2320185040577047131.png" + } + }, + "unchanged.skp" : { + "whole-image" : { + "checksumAlgorithm" : "bitmap-64bitMD5", + "checksumValue" : 3322248763049618493, + "comparisonResult" : "no-comparison", + "filepath" : "unchanged_skp/bitmap-64bitMD5_3322248763049618493.png" + } + } + }, + "header" : { + "revision" : 1, + "type" : "ChecksummedImages" + } +} diff --git a/gm/rebaseline_server/testdata/inputs/skp-actuals/setB/summary.json b/gm/rebaseline_server/testdata/inputs/skp-actuals/setB/summary.json new file mode 100644 index 0000000..67e8409 --- /dev/null +++ b/gm/rebaseline_server/testdata/inputs/skp-actuals/setB/summary.json @@ -0,0 +1,32 @@ +{ + "actual-results" : { + "changed.skp" : { + "whole-image" : { + "checksumAlgorithm" : "bitmap-64bitMD5", + "checksumValue" : 13623922271964399662, + "comparisonResult" : "no-comparison", + "filepath" : "changed_skp/bitmap-64bitMD5_13623922271964399662.png" + } + }, + "only-in-after.skp" : { + "whole-image" : { + "checksumAlgorithm" : "bitmap-64bitMD5", + "checksumValue" : 2320185040577047131, + "comparisonResult" : "no-comparison", + "filepath" : "only-in-after_skp/bitmap-64bitMD5_2320185040577047131.png" + } + }, + "unchanged.skp" : { + "whole-image" : { + "checksumAlgorithm" : "bitmap-64bitMD5", + "checksumValue" : 3322248763049618493, + "comparisonResult" : "no-comparison", + "filepath" : "unchanged_skp/bitmap-64bitMD5_3322248763049618493.png" + } + } + }, + "header" : { + "revision" : 1, + "type" : "ChecksummedImages" + } +} diff --git a/gm/rebaseline_server/testdata/outputs/expected/compare_rendered_pictures_test.CompareRenderedPicturesTest.test_repo_url/compare_rendered_pictures.json b/gm/rebaseline_server/testdata/outputs/expected/compare_rendered_pictures_test.CompareRenderedPicturesTest.test_repo_url/compare_rendered_pictures.json new file mode 100644 index 0000000..4a817d3 --- /dev/null +++ b/gm/rebaseline_server/testdata/outputs/expected/compare_rendered_pictures_test.CompareRenderedPicturesTest.test_repo_url/compare_rendered_pictures.json @@ -0,0 +1,151 @@ +{ + "extraColumnHeaders": { + "resultType": { + "headerText": "resultType", + "isFilterable": true, + "isSortable": true, + "useFreeformFilter": false, + "valuesAndCounts": [ + [ + "failed", + 1 + ], + [ + "no-comparison", + 2 + ], + [ + "succeeded", + 1 + ] + ] + }, + "sourceSkpFile": { + "headerText": "sourceSkpFile", + "isFilterable": true, + "isSortable": true, + "useFreeformFilter": false, + "valuesAndCounts": [ + [ + "changed.skp", + 1 + ], + [ + "only-in-after.skp", + 1 + ], + [ + "only-in-before.skp", + 1 + ], + [ + "unchanged.skp", + 1 + ] + ] + }, + "tiledOrWhole": { + "headerText": "tiledOrWhole", + "isFilterable": true, + "isSortable": true, + "useFreeformFilter": false, + "valuesAndCounts": [ + [ + "whole", + 4 + ] + ] + }, + "tilenum": { + "headerText": "tilenum", + "isFilterable": true, + "isSortable": true, + "useFreeformFilter": true, + "valuesAndCounts": [ + [ + "N/A", + 4 + ] + ] + } + }, + "extraColumnOrder": [ + "resultType", + "sourceSkpFile", + "tiledOrWhole", + "tilenum" + ], + "header": { + "dataHash": "-4754972663365911725", + "isEditable": false, + "isExported": true, + "schemaVersion": 4, + "timeNextUpdateAvailable": null, + "timeUpdated": 12345678, + "type": "all" + }, + "imagePairs": [ + { + "extraColumns": { + "resultType": "failed", + "sourceSkpFile": "changed.skp", + "tiledOrWhole": "whole", + "tilenum": "N/A" + }, + "imageAUrl": "changed_skp/bitmap-64bitMD5_3101044995537104462.png", + "imageBUrl": "changed_skp/bitmap-64bitMD5_13623922271964399662.png", + "isDifferent": true + }, + { + "extraColumns": { + "resultType": "no-comparison", + "sourceSkpFile": "only-in-after.skp", + "tiledOrWhole": "whole", + "tilenum": "N/A" + }, + "imageAUrl": null, + "imageBUrl": "only-in-after_skp/bitmap-64bitMD5_2320185040577047131.png", + "isDifferent": true + }, + { + "extraColumns": { + "resultType": "no-comparison", + "sourceSkpFile": "only-in-before.skp", + "tiledOrWhole": "whole", + "tilenum": "N/A" + }, + "imageAUrl": "only-in-before_skp/bitmap-64bitMD5_2320185040577047131.png", + "imageBUrl": null, + "isDifferent": true + }, + { + "extraColumns": { + "resultType": "succeeded", + "sourceSkpFile": "unchanged.skp", + "tiledOrWhole": "whole", + "tilenum": "N/A" + }, + "imageAUrl": "unchanged_skp/bitmap-64bitMD5_3322248763049618493.png", + "imageBUrl": "unchanged_skp/bitmap-64bitMD5_3322248763049618493.png", + "isDifferent": false + } + ], + "imageSets": { + "diffs": { + "baseUrl": "/static/generated-images/diffs", + "description": "color difference per channel" + }, + "imageA": { + "baseUrl": "http://storage.cloud.google.com/fakebucket/fake/path", + "description": "setA" + }, + "imageB": { + "baseUrl": "http://storage.cloud.google.com/fakebucket/fake/path", + "description": "setB" + }, + "whiteDiffs": { + "baseUrl": "/static/generated-images/whitediffs", + "description": "differing pixels in white" + } + } +} \ No newline at end of file -- 2.7.4