GM: Allow ignored-tests.txt to list configs as well as tests (and combinations too)

BUG=skia:1657

This will allow us to ignore poppler failures until bug 1657 is resolved
(but this CL doesn't actually make that change to ignored-tests.txt yet)

R=scroggo@google.com

Review URL: https://codereview.chromium.org/26294004

git-svn-id: http://skia.googlecode.com/svn/trunk@11639 2bbb7eff-a529-9590-31e7-b0007b416f81

diff --git a/expectations/gm/ignored-tests.txt b/expectations/gm/ignored-tests.txt
index a26591c..f902ec1 100644 (file)
--- a/expectations/gm/ignored-tests.txt
+++ b/expectations/gm/ignored-tests.txt
@@ -1,5 +1,5 @@
-# Failures of any GM tests listed in this file will be ignored, as if they
-# had been marked "ignore-failure": true in the per-builder
+# Failures of any GM tests/configs listed in this file will be ignored, as if
+# they had been marked "ignore-failure": true in the per-builder
 # expected-results.json files.
 #
 # The idea is that, if you modify a GM test in such a way that you know it will
@@ -17,6 +17,12 @@
 # ('GM expectations: create tool to suppress failures until new baselines are
 # checked in')
 
-# EXAMPLE: (remove the first '#' on each line)
+# EXAMPLES: (remove the first '#' on each line)
+#
 ## Added by edisonn as part of https://codereview.chromium.org/23851037/
 #gradients
+#
+## Added by epoger as part of MADE-UP BUG
+## https://code.google.com/p/skia/issues/detail?id=123456 : ignoring failures on
+## gpu config of gradtext GM test
+#gradtext_gpu

diff --git a/gm/gm.cpp b/gm/gm.cpp
index 966e59b..9da1a17 100644 (file)
--- a/gm/gm.cpp
+++ b/gm/gm.cpp
@@ -14,7 +14,6 @@ GM::GM() {
     fBGColor = SK_ColorWHITE;
     fCanvasIsDeferred = false;
     fHaveCalledOnceBeforeDraw = false;
-    fIgnoreFailures = false;
     fStarterMatrix.reset();
 }
 GM::~GM() {}

diff --git a/gm/gm.h b/gm/gm.h
index bf1a411..a0ad3cf 100644 (file)
--- a/gm/gm.h
+++ b/gm/gm.h
@@ -94,11 +94,6 @@ namespace skiagm {
             return gResourcePath;
         }
 
-        bool isIgnoringFailures() const { return fIgnoreFailures; }
-        void setIgnoreFailures(bool val) {
-            fIgnoreFailures = val;
-        }
-
         bool isCanvasDeferred() const { return fCanvasIsDeferred; }
         void setCanvasIsDeferred(bool isDeferred) {
             fCanvasIsDeferred = isDeferred;
@@ -129,7 +124,6 @@ namespace skiagm {
         SkColor  fBGColor;
         bool     fCanvasIsDeferred; // work-around problem in srcmode.cpp
         bool     fHaveCalledOnceBeforeDraw;
-        bool     fIgnoreFailures; // whether to file any failures as failure-ignored
         SkMatrix fStarterMatrix;
     };
 

diff --git a/gm/gmmain.cpp b/gm/gmmain.cpp
index a06138a..99cd2a4 100644 (file)
--- a/gm/gmmain.cpp
+++ b/gm/gmmain.cpp
@@ -329,6 +329,18 @@ public:
     }
 
     /**
+     * Returns true if failures on this test should be ignored.
+     */
+    bool ShouldIgnoreTest(const SkString &name) const {
+        for (int i = 0; i < fIgnorableTestSubstrings.count(); i++) {
+            if (name.contains(fIgnorableTestSubstrings[i].c_str())) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
      * Records the results of this test in fTestsRun and fFailedTests.
      *
      * We even record successes, and errors that we regard as
@@ -905,7 +917,7 @@ public:
                  * See comments above complete_bitmap() for more detail.
                  */
                 Expectations expectations = expectationsSource->get(nameWithExtension.c_str());
-                if (gm->isIgnoringFailures()) {
+                if (this->ShouldIgnoreTest(shortNamePlusConfig)) {
                     expectations.setIgnoreFailure(true);
                 }
                 errors.add(compare_to_expectations(expectations, *actualBitmapAndDigest,
@@ -1212,6 +1224,7 @@ public:
 
     bool fUseFileHierarchy, fWriteChecksumBasedFilenames;
     ErrorCombination fIgnorableErrorTypes;
+    SkTArray<SkString> fIgnorableTestSubstrings;
 
     const char* fMismatchPath;
     const char* fMissingExpectationsPath;
@@ -1979,6 +1992,18 @@ static bool parse_flags_ignore_error_types(ErrorCombination* outErrorTypes) {
     return true;
 }
 
+/**
+ * Replace contents of ignoreTestSubstrings with a list of testname/config substrings, indicating
+ * which tests' failures should be ignored.
+ */
+static bool parse_flags_ignore_tests(SkTArray<SkString> &ignoreTestSubstrings) {
+    ignoreTestSubstrings.reset();
+    for (int i = 0; i < FLAGS_ignoreTests.count(); i++) {
+        ignoreTestSubstrings.push_back(SkString(FLAGS_ignoreTests[i]));
+    }
+    return true;
+}
+
 static bool parse_flags_modulo(int* moduloRemainder, int* moduloDivisor) {
     if (FLAGS_modulo.count() == 2) {
         *moduloRemainder = atoi(FLAGS_modulo[0]);
@@ -2110,6 +2135,7 @@ int tool_main(int argc, char** argv) {
 
     if (!parse_flags_modulo(&moduloRemainder, &moduloDivisor) ||
         !parse_flags_ignore_error_types(&gmmain.fIgnorableErrorTypes) ||
+        !parse_flags_ignore_tests(gmmain.fIgnorableTestSubstrings) ||
 #if SK_SUPPORT_GPU
         !parse_flags_gpu_cache(&gGpuCacheSizeBytes, &gGpuCacheSizeCount) ||
 #endif
@@ -2191,9 +2217,6 @@ int tool_main(int argc, char** argv) {
         if (SkCommandLineFlags::ShouldSkip(FLAGS_match, shortName)) {
             continue;
         }
-        if (FLAGS_ignoreTests.contains(shortName)) {
-            gm->setIgnoreFailures(true);
-        }
 
         gmsRun++;
         SkISize size = gm->getISize();

diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/command_line b/gm/tests/outputs/ignoring-one-test/output-expected/command_line
index 49025ad..5f5f66e 100644 (file)
--- a/gm/tests/outputs/ignoring-one-test/output-expected/command_line
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --verbose --hierarchy --match selftest1 --ignoreTests selftest1 --config 8888 565 -r gm/tests/inputs/json/different-pixels.json --writeJsonSummaryPath gm/tests/outputs/ignoring-one-test/output-actual/json-summary.txt --writePath gm/tests/outputs/ignoring-one-test/output-actual/writePath --mismatchPath gm/tests/outputs/ignoring-one-test/output-actual/mismatchPath --missingExpectationsPath gm/tests/outputs/ignoring-one-test/output-actual/missingExpectationsPath
+out/Debug/gm --verbose --hierarchy --match selftest1 --ignoreTests 8888/selfte --config 8888 565 -r gm/tests/inputs/json/different-pixels.json --writeJsonSummaryPath gm/tests/outputs/ignoring-one-test/output-actual/json-summary.txt --writePath gm/tests/outputs/ignoring-one-test/output-actual/writePath --mismatchPath gm/tests/outputs/ignoring-one-test/output-actual/mismatchPath --missingExpectationsPath gm/tests/outputs/ignoring-one-test/output-actual/missingExpectationsPath

diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/json-summary.txt b/gm/tests/outputs/ignoring-one-test/output-expected/json-summary.txt
index 5f8aa4e..a639f7f 100644 (file)
--- a/gm/tests/outputs/ignoring-one-test/output-expected/json-summary.txt
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/json-summary.txt
@@ -1,8 +1,9 @@
 {
    "actual-results" : {
-      "failed" : null,
+      "failed" : {
+         "565/selftest1.png" : [ "bitmap-64bitMD5", 12927999507540085554 ]
+      },
       "failure-ignored" : {
-         "565/selftest1.png" : [ "bitmap-64bitMD5", 12927999507540085554 ],
          "8888/selftest1.png" : [ "bitmap-64bitMD5", 1209453360120438698 ]
       },
       "no-comparison" : null,
@@ -13,7 +14,7 @@
          "allowed-digests" : [
             [ "bitmap-64bitMD5", 8863920166200910451 ]
          ],
-         "ignore-failure" : true
+         "ignore-failure" : false
       },
       "8888/selftest1.png" : {
          "allowed-digests" : [

diff --git a/gm/tests/run.sh b/gm/tests/run.sh
index 7a7bc13..5c133b4 100755 (executable)
--- a/gm/tests/run.sh
+++ b/gm/tests/run.sh
@@ -202,7 +202,7 @@ gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/d
 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json"
 
 # Exercise --ignoreTests flag.
-gm_test "--verbose --hierarchy --match selftest1 --ignoreTests selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/ignoring-one-test"
+gm_test "--verbose --hierarchy --match selftest1 --ignoreTests 8888/selfte $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/ignoring-one-test"
 
 # Compare different pixels, but with a SUBSET of the expectations marked as
 # ignore-failure.