From: commit-bot@chromium.org Date: Wed, 14 May 2014 18:04:46 +0000 (+0000) Subject: rebaseline_server: apply ignored-tests.txt within rebaseline_server, not just on... X-Git-Tag: accepted/tizen/5.0/unified/20181102.025319~7723 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=4cef1be82193a9d527f7b1f873197c443d0dde2e;p=platform%2Fupstream%2FlibSkiaSharp.git rebaseline_server: apply ignored-tests.txt within rebaseline_server, not just on the bots (SkipBuildbotRuns) BUG=skia:2514 NOTREECHECKS=True NOTRY=True R=rmistry@google.com Author: epoger@google.com Review URL: https://codereview.chromium.org/287623006 git-svn-id: http://skia.googlecode.com/svn/trunk@14740 2bbb7eff-a529-9590-31e7-b0007b416f81 --- diff --git a/gm/rebaseline_server/compare_to_expectations.py b/gm/rebaseline_server/compare_to_expectations.py index 0f3c352..e9677bd 100755 --- a/gm/rebaseline_server/compare_to_expectations.py +++ b/gm/rebaseline_server/compare_to_expectations.py @@ -48,6 +48,7 @@ EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [ results.KEY__EXPECTATIONS__REVIEWED, ] DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm') +DEFAULT_IGNORE_FAILURES_FILE = 'ignored-tests.txt' IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image') @@ -64,12 +65,15 @@ class ExpectationComparisons(results.BaseComparisons): def __init__(self, actuals_root=results.DEFAULT_ACTUALS_DIR, expected_root=DEFAULT_EXPECTATIONS_DIR, + ignore_failures_file=DEFAULT_IGNORE_FAILURES_FILE, generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT, diff_base_url=None, builder_regex_list=None): """ Args: actuals_root: root directory containing all actual-results.json files expected_root: root directory containing all expected-results.json files + ignore_failures_file: if a file with this name is found within + expected_root, ignore failures for any tests listed in the file generated_images_root: directory within which to create all pixel diffs; if this directory does not yet exist, it will be created diff_base_url: base URL within which the client should look for diff @@ -87,6 +91,11 @@ class ExpectationComparisons(results.BaseComparisons): download_actuals.create_filepath_url(generated_images_root)) self._actuals_root = actuals_root self._expected_root = expected_root + self._ignore_failures_on_these_tests = [] + if ignore_failures_file: + self._ignore_failures_on_these_tests = ( + ExpectationComparisons._read_noncomment_lines( + os.path.join(expected_root, ignore_failures_file))) self._load_actual_and_expected() self._timestamp = int(time.time()) logging.info('Results complete; took %d seconds.' % @@ -316,10 +325,11 @@ class ExpectationComparisons(results.BaseComparisons): # categories recorded within the gm_actuals AT ALL, and # instead evaluate the result_type ourselves based on what # we see in expectations vs actual checksum? - # See related http://skbug.com/2514 ('rebaseline_server: apply - # ignored-tests.txt within rebaseline_server, not just on the bots') if expected_image_relative_url == actual_image_relative_url: updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED + elif ((result_type == results.KEY__RESULT_TYPE__FAILED) and + (test in self._ignore_failures_on_these_tests)): + updated_result_type = results.KEY__RESULT_TYPE__FAILUREIGNORED else: updated_result_type = result_type extra_columns_dict = { @@ -362,6 +372,11 @@ def main(): help='Directory containing all expected-result JSON files; defaults to ' '\'%(default)s\' .') parser.add_argument( + '--ignore-failures-file', default=DEFAULT_IGNORE_FAILURES_FILE, + help='If a file with this name is found within the EXPECTATIONS dir, ' + 'ignore failures for any tests listed in the file; defaults to ' + '\'%(default)s\' .') + parser.add_argument( '--outfile', required=True, help='File to write result summary into, in JSON format.') parser.add_argument( @@ -375,9 +390,11 @@ def main(): help='Directory within which to download images and generate diffs; ' 'defaults to \'%(default)s\' .') args = parser.parse_args() - results_obj = ExpectationComparisons(actuals_root=args.actuals, - expected_root=args.expectations, - generated_images_root=args.workdir) + results_obj = ExpectationComparisons( + actuals_root=args.actuals, + expected_root=args.expectations, + ignore_failures_file=args.ignore_failures_file, + generated_images_root=args.workdir) gm_json.WriteToFile( results_obj.get_packaged_results_of_type(results_type=args.results), args.outfile) diff --git a/gm/rebaseline_server/results.py b/gm/rebaseline_server/results.py index 45ab133..70b2342 100755 --- a/gm/rebaseline_server/results.py +++ b/gm/rebaseline_server/results.py @@ -246,6 +246,22 @@ class BaseComparisons(object): return meta_dict @staticmethod + def _read_noncomment_lines(path): + """Return a list of all noncomment lines within a file. + + (A "noncomment" line is one that does not start with a '#'.) + + Args: + path: path to file + """ + lines = [] + with open(path, 'r') as fh: + for line in fh: + if not line.startswith('#'): + lines.append(line.strip()) + return lines + + @staticmethod def _create_relative_url(hashtype_and_digest, test_name): """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL. diff --git a/gm/rebaseline_server/testdata/inputs/gm-actuals/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/actual-results.json b/gm/rebaseline_server/testdata/inputs/gm-actuals/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/actual-results.json index 0a1de74..354f439 100644 --- a/gm/rebaseline_server/testdata/inputs/gm-actuals/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/actual-results.json +++ b/gm/rebaseline_server/testdata/inputs/gm-actuals/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/actual-results.json @@ -1,6 +1,9 @@ { "actual-results" : { - "failed" : null, + "failed" : { + "blanket-ignored_565.png" : [ "bitmap-64bitMD5", 22222222 ], + "blanket-ignored_8888.png" : [ "bitmap-64bitMD5", 22222222 ] + }, "failure-ignored" : { "displacement_565.png" : [ "bitmap-64bitMD5", 4569468668668628191 ], "displacement_8888.png" : [ "bitmap-64bitMD5", 11401048196735046263 ], @@ -54,6 +57,18 @@ ], "ignore-failure" : false }, + "blanket-ignored_565.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 111111111 ] + ], + "ignore-failure" : false + }, + "blanket-ignored_8888.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 111111111 ] + ], + "ignore-failure" : false + }, "displacement_565.png" : { "allowed-digests" : [ [ "bitmap-64bitMD5", 16249664097236120848 ] diff --git a/gm/rebaseline_server/testdata/inputs/gm-expectations/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/expected-results.json b/gm/rebaseline_server/testdata/inputs/gm-expectations/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/expected-results.json index 1b3fdc6..f3f49bb 100644 --- a/gm/rebaseline_server/testdata/inputs/gm-expectations/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/expected-results.json +++ b/gm/rebaseline_server/testdata/inputs/gm-expectations/Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug/expected-results.json @@ -66,6 +66,24 @@ ], "reviewed-by-human": false }, + "blanket-ignored_565.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 111111111 + ] + ], + "ignore-failure": false + }, + "blanket-ignored_8888.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 111111111 + ] + ], + "ignore-failure": false + }, "displacement_565.png": { "allowed-digests": [ [ diff --git a/gm/rebaseline_server/testdata/inputs/gm-expectations/ignored-tests.txt b/gm/rebaseline_server/testdata/inputs/gm-expectations/ignored-tests.txt new file mode 100644 index 0000000..b14eb96 --- /dev/null +++ b/gm/rebaseline_server/testdata/inputs/gm-expectations/ignored-tests.txt @@ -0,0 +1,10 @@ +# Failures of any GM tests listed in this file will be ignored [1], as +# if they had been marked "ignore-failure": true in the per-builder +# expected-results.json files. +# + +blanket-ignored + +# "texdata" tests should NOT be ignored, since the next lines is commented out: +# texdata + diff --git a/gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json b/gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json index 6d63113..d98ac41 100644 --- a/gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json +++ b/gm/rebaseline_server/testdata/outputs/expected/compare_configs_test.CompareConfigsTest.test_gm/gm.json @@ -15,7 +15,7 @@ ], [ "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug", - 3 + 4 ] ] }, @@ -26,7 +26,7 @@ "valuesAndCounts": [ [ "TODO", - 13 + 14 ] ] }, @@ -41,7 +41,7 @@ ], [ "no-comparison", - 7 + 8 ], [ "succeeded", @@ -71,6 +71,10 @@ 1 ], [ + "blanket-ignored", + 1 + ], + [ "displacement", 2 ], @@ -90,7 +94,7 @@ } }, "header": { - "dataHash": "-7043844904261310530", + "dataHash": "-3232866296817265056", "isEditable": false, "isExported": true, "schemaVersion": 3, @@ -232,6 +236,17 @@ "isDifferent": false }, { + "extraColumns": { + "builder": "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug", + "config": "TODO", + "resultType": "no-comparison", + "test": "blanket-ignored" + }, + "imageAUrl": "bitmap-64bitMD5/blanket-ignored/22222222.png", + "imageBUrl": null, + "isDifferent": true + }, + { "differenceData": { "maxDiffPerChannel": [ 136, diff --git a/gm/rebaseline_server/testdata/outputs/expected/compare_to_expectations_test.CompareToExpectationsTest.test_gm/gm.json b/gm/rebaseline_server/testdata/outputs/expected/compare_to_expectations_test.CompareToExpectationsTest.test_gm/gm.json index 4f4eef7..9ad771e 100644 --- a/gm/rebaseline_server/testdata/outputs/expected/compare_to_expectations_test.CompareToExpectationsTest.test_gm/gm.json +++ b/gm/rebaseline_server/testdata/outputs/expected/compare_to_expectations_test.CompareToExpectationsTest.test_gm/gm.json @@ -11,7 +11,7 @@ ], [ "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug", - 15 + 17 ] ] }, @@ -22,11 +22,11 @@ "valuesAndCounts": [ [ "565", - 9 + 10 ], [ "8888", - 9 + 10 ], [ "gpu", @@ -53,7 +53,7 @@ ], [ "failure-ignored", - 4 + 6 ], [ "no-comparison", @@ -87,6 +87,10 @@ 2 ], [ + "blanket-ignored", + 2 + ], + [ "displacement", 5 ], @@ -106,7 +110,7 @@ } }, "header": { - "dataHash": "2775016045957284034", + "dataHash": "-7988419419043310401", "isEditable": false, "isExported": true, "schemaVersion": 3, @@ -379,6 +383,38 @@ "extraColumns": { "builder": "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug", "config": "565", + "resultType": "failure-ignored", + "test": "blanket-ignored" + }, + "imageAUrl": "bitmap-64bitMD5/blanket-ignored/111111111.png", + "imageBUrl": "bitmap-64bitMD5/blanket-ignored/22222222.png", + "isDifferent": true + }, + { + "expectations": { + "bugs": null, + "ignore-failure": false, + "reviewed-by-human": null + }, + "extraColumns": { + "builder": "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug", + "config": "8888", + "resultType": "failure-ignored", + "test": "blanket-ignored" + }, + "imageAUrl": "bitmap-64bitMD5/blanket-ignored/111111111.png", + "imageBUrl": "bitmap-64bitMD5/blanket-ignored/22222222.png", + "isDifferent": true + }, + { + "expectations": { + "bugs": null, + "ignore-failure": false, + "reviewed-by-human": null + }, + "extraColumns": { + "builder": "Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug", + "config": "565", "resultType": "succeeded", "test": "displacement" },