From 3a882dda0787e6957cfb37f6b5a68bcd31ce5cf7 Mon Sep 17 00:00:00 2001 From: "epoger@google.com" Date: Mon, 7 Oct 2013 18:55:09 +0000 Subject: [PATCH] GM: Allow ignored-tests.txt to list configs as well as tests (and combinations too) BUG=skia:1657 This will allow us to ignore poppler failures until bug 1657 is resolved (but this CL doesn't actually make that change to ignored-tests.txt yet) R=scroggo@google.com Review URL: https://codereview.chromium.org/26294004 git-svn-id: http://skia.googlecode.com/svn/trunk@11639 2bbb7eff-a529-9590-31e7-b0007b416f81 --- expectations/gm/ignored-tests.txt | 12 ++++++--- gm/gm.cpp | 1 - gm/gm.h | 6 ----- gm/gmmain.cpp | 31 +++++++++++++++++++--- .../ignoring-one-test/output-expected/command_line | 2 +- .../output-expected/json-summary.txt | 7 ++--- gm/tests/run.sh | 2 +- 7 files changed, 42 insertions(+), 19 deletions(-) diff --git a/expectations/gm/ignored-tests.txt b/expectations/gm/ignored-tests.txt index a26591c..f902ec1 100644 --- a/expectations/gm/ignored-tests.txt +++ b/expectations/gm/ignored-tests.txt @@ -1,5 +1,5 @@ -# Failures of any GM tests listed in this file will be ignored, as if they -# had been marked "ignore-failure": true in the per-builder +# Failures of any GM tests/configs listed in this file will be ignored, as if +# they had been marked "ignore-failure": true in the per-builder # expected-results.json files. # # The idea is that, if you modify a GM test in such a way that you know it will @@ -17,6 +17,12 @@ # ('GM expectations: create tool to suppress failures until new baselines are # checked in') -# EXAMPLE: (remove the first '#' on each line) +# EXAMPLES: (remove the first '#' on each line) +# ## Added by edisonn as part of https://codereview.chromium.org/23851037/ #gradients +# +## Added by epoger as part of MADE-UP BUG +## https://code.google.com/p/skia/issues/detail?id=123456 : ignoring failures on +## gpu config of gradtext GM test +#gradtext_gpu diff --git a/gm/gm.cpp b/gm/gm.cpp index 966e59b..9da1a17 100644 --- a/gm/gm.cpp +++ b/gm/gm.cpp @@ -14,7 +14,6 @@ GM::GM() { fBGColor = SK_ColorWHITE; fCanvasIsDeferred = false; fHaveCalledOnceBeforeDraw = false; - fIgnoreFailures = false; fStarterMatrix.reset(); } GM::~GM() {} diff --git a/gm/gm.h b/gm/gm.h index bf1a411..a0ad3cf 100644 --- a/gm/gm.h +++ b/gm/gm.h @@ -94,11 +94,6 @@ namespace skiagm { return gResourcePath; } - bool isIgnoringFailures() const { return fIgnoreFailures; } - void setIgnoreFailures(bool val) { - fIgnoreFailures = val; - } - bool isCanvasDeferred() const { return fCanvasIsDeferred; } void setCanvasIsDeferred(bool isDeferred) { fCanvasIsDeferred = isDeferred; @@ -129,7 +124,6 @@ namespace skiagm { SkColor fBGColor; bool fCanvasIsDeferred; // work-around problem in srcmode.cpp bool fHaveCalledOnceBeforeDraw; - bool fIgnoreFailures; // whether to file any failures as failure-ignored SkMatrix fStarterMatrix; }; diff --git a/gm/gmmain.cpp b/gm/gmmain.cpp index a06138a..99cd2a4 100644 --- a/gm/gmmain.cpp +++ b/gm/gmmain.cpp @@ -329,6 +329,18 @@ public: } /** + * Returns true if failures on this test should be ignored. + */ + bool ShouldIgnoreTest(const SkString &name) const { + for (int i = 0; i < fIgnorableTestSubstrings.count(); i++) { + if (name.contains(fIgnorableTestSubstrings[i].c_str())) { + return true; + } + } + return false; + } + + /** * Records the results of this test in fTestsRun and fFailedTests. * * We even record successes, and errors that we regard as @@ -905,7 +917,7 @@ public: * See comments above complete_bitmap() for more detail. */ Expectations expectations = expectationsSource->get(nameWithExtension.c_str()); - if (gm->isIgnoringFailures()) { + if (this->ShouldIgnoreTest(shortNamePlusConfig)) { expectations.setIgnoreFailure(true); } errors.add(compare_to_expectations(expectations, *actualBitmapAndDigest, @@ -1212,6 +1224,7 @@ public: bool fUseFileHierarchy, fWriteChecksumBasedFilenames; ErrorCombination fIgnorableErrorTypes; + SkTArray fIgnorableTestSubstrings; const char* fMismatchPath; const char* fMissingExpectationsPath; @@ -1979,6 +1992,18 @@ static bool parse_flags_ignore_error_types(ErrorCombination* outErrorTypes) { return true; } +/** + * Replace contents of ignoreTestSubstrings with a list of testname/config substrings, indicating + * which tests' failures should be ignored. + */ +static bool parse_flags_ignore_tests(SkTArray &ignoreTestSubstrings) { + ignoreTestSubstrings.reset(); + for (int i = 0; i < FLAGS_ignoreTests.count(); i++) { + ignoreTestSubstrings.push_back(SkString(FLAGS_ignoreTests[i])); + } + return true; +} + static bool parse_flags_modulo(int* moduloRemainder, int* moduloDivisor) { if (FLAGS_modulo.count() == 2) { *moduloRemainder = atoi(FLAGS_modulo[0]); @@ -2110,6 +2135,7 @@ int tool_main(int argc, char** argv) { if (!parse_flags_modulo(&moduloRemainder, &moduloDivisor) || !parse_flags_ignore_error_types(&gmmain.fIgnorableErrorTypes) || + !parse_flags_ignore_tests(gmmain.fIgnorableTestSubstrings) || #if SK_SUPPORT_GPU !parse_flags_gpu_cache(&gGpuCacheSizeBytes, &gGpuCacheSizeCount) || #endif @@ -2191,9 +2217,6 @@ int tool_main(int argc, char** argv) { if (SkCommandLineFlags::ShouldSkip(FLAGS_match, shortName)) { continue; } - if (FLAGS_ignoreTests.contains(shortName)) { - gm->setIgnoreFailures(true); - } gmsRun++; SkISize size = gm->getISize(); diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/command_line b/gm/tests/outputs/ignoring-one-test/output-expected/command_line index 49025ad..5f5f66e 100644 --- a/gm/tests/outputs/ignoring-one-test/output-expected/command_line +++ b/gm/tests/outputs/ignoring-one-test/output-expected/command_line @@ -1 +1 @@ -out/Debug/gm --verbose --hierarchy --match selftest1 --ignoreTests selftest1 --config 8888 565 -r gm/tests/inputs/json/different-pixels.json --writeJsonSummaryPath gm/tests/outputs/ignoring-one-test/output-actual/json-summary.txt --writePath gm/tests/outputs/ignoring-one-test/output-actual/writePath --mismatchPath gm/tests/outputs/ignoring-one-test/output-actual/mismatchPath --missingExpectationsPath gm/tests/outputs/ignoring-one-test/output-actual/missingExpectationsPath +out/Debug/gm --verbose --hierarchy --match selftest1 --ignoreTests 8888/selfte --config 8888 565 -r gm/tests/inputs/json/different-pixels.json --writeJsonSummaryPath gm/tests/outputs/ignoring-one-test/output-actual/json-summary.txt --writePath gm/tests/outputs/ignoring-one-test/output-actual/writePath --mismatchPath gm/tests/outputs/ignoring-one-test/output-actual/mismatchPath --missingExpectationsPath gm/tests/outputs/ignoring-one-test/output-actual/missingExpectationsPath diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/json-summary.txt b/gm/tests/outputs/ignoring-one-test/output-expected/json-summary.txt index 5f8aa4e..a639f7f 100644 --- a/gm/tests/outputs/ignoring-one-test/output-expected/json-summary.txt +++ b/gm/tests/outputs/ignoring-one-test/output-expected/json-summary.txt @@ -1,8 +1,9 @@ { "actual-results" : { - "failed" : null, + "failed" : { + "565/selftest1.png" : [ "bitmap-64bitMD5", 12927999507540085554 ] + }, "failure-ignored" : { - "565/selftest1.png" : [ "bitmap-64bitMD5", 12927999507540085554 ], "8888/selftest1.png" : [ "bitmap-64bitMD5", 1209453360120438698 ] }, "no-comparison" : null, @@ -13,7 +14,7 @@ "allowed-digests" : [ [ "bitmap-64bitMD5", 8863920166200910451 ] ], - "ignore-failure" : true + "ignore-failure" : false }, "8888/selftest1.png" : { "allowed-digests" : [ diff --git a/gm/tests/run.sh b/gm/tests/run.sh index 7a7bc13..5c133b4 100755 --- a/gm/tests/run.sh +++ b/gm/tests/run.sh @@ -202,7 +202,7 @@ gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/d gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json" # Exercise --ignoreTests flag. -gm_test "--verbose --hierarchy --match selftest1 --ignoreTests selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/ignoring-one-test" +gm_test "--verbose --hierarchy --match selftest1 --ignoreTests 8888/selfte $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/ignoring-one-test" # Compare different pixels, but with a SUBSET of the expectations marked as # ignore-failure. -- 2.7.4