-# Failures of any GM tests/configs listed in this file will be ignored [1], as
+# Failures of any GM tests listed in this file will be ignored [1], as
# if they had been marked "ignore-failure": true in the per-builder
# expected-results.json files.
#
# If there are any lingering failures that need to be suppressed (ignored), you
# MUST move those suppressions into the per-builder expected-results.json files.
#
+# The test name must match exactly (no partial string matches).
+# See http://skbug.com/2022
+#
# Any lines starting with '#' are comments and will be ignored.
# Other than that, each line must either be whitespace or list just one test.
#
# EXAMPLES: (remove the first '#' on each line)
#
-## Added by edisonn as part of https://codereview.chromium.org/23851037/
-#gradients
-#
## Added by epoger as part of MADE-UP BUG
## https://code.google.com/p/skia/issues/detail?id=123456 : ignoring failures on
-## gpu config of gradtext GM test
-#gradtext_gpu
+## gradtext GM test
+#gradtext
# Added for skbug.com/1998 by bsalomon. Will require rebaselining when changes are complete.
bleed
/**
* Returns true if failures on this test should be ignored.
*/
- bool ShouldIgnoreTest(const SkString &name) const {
- for (int i = 0; i < fIgnorableTestSubstrings.count(); i++) {
- if (name.contains(fIgnorableTestSubstrings[i].c_str())) {
+ bool ShouldIgnoreTest(const char *name) const {
+ for (int i = 0; i < fIgnorableTestNames.count(); i++) {
+ if (fIgnorableTestNames[i].equals(name)) {
return true;
}
}
ErrorCombination compare_test_results_to_stored_expectations(
GM* gm, const ConfigData& gRec, const char* configName,
const BitmapAndDigest* actualBitmapAndDigest) {
-
- SkString shortNamePlusConfig = make_shortname_plus_config(gm->shortName(), configName);
-
ErrorCombination errors;
if (NULL == actualBitmapAndDigest) {
errors.add(ErrorCombination(kIntentionallySkipped_ErrorType));
} else {
ExpectationsSource *expectationsSource = this->fExpectationsSource.get();
- SkString nameWithExtension(shortNamePlusConfig);
+ SkString nameWithExtension = make_shortname_plus_config(gm->shortName(), configName);
nameWithExtension.append(".");
nameWithExtension.append(kPNG_FileExtension);
* See comments above complete_bitmap() for more detail.
*/
Expectations expectations = expectationsSource->get(nameWithExtension.c_str());
- if (this->ShouldIgnoreTest(shortNamePlusConfig)) {
+ if (this->ShouldIgnoreTest(gm->shortName())) {
expectations.setIgnoreFailure(true);
}
errors.add(compare_to_expectations(expectations, *actualBitmapAndDigest,
bool fUseFileHierarchy, fWriteChecksumBasedFilenames;
ErrorCombination fIgnorableErrorTypes;
- SkTArray<SkString> fIgnorableTestSubstrings;
+ SkTArray<SkString> fIgnorableTestNames;
const char* fMismatchPath;
const char* fMissingExpectationsPath;
}
/**
- * Replace contents of ignoreTestSubstrings with a list of testname/config substrings, indicating
+ * Replace contents of ignoreTestNames with a list of test names, indicating
* which tests' failures should be ignored.
*/
-static bool parse_flags_ignore_tests(SkTArray<SkString> &ignoreTestSubstrings) {
- ignoreTestSubstrings.reset();
+static bool parse_flags_ignore_tests(SkTArray<SkString> &ignoreTestNames) {
+ ignoreTestNames.reset();
// Parse --ignoreFailuresFile
for (int i = 0; i < FLAGS_ignoreFailuresFile.count(); i++) {
if (thisLine.isEmpty() || thisLine.startsWith('#')) {
// skip this line
} else {
- ignoreTestSubstrings.push_back(thisLine);
+ ignoreTestNames.push_back(thisLine);
}
}
}
if (!parse_flags_modulo(&moduloRemainder, &moduloDivisor) ||
!parse_flags_ignore_error_types(&gmmain.fIgnorableErrorTypes) ||
- !parse_flags_ignore_tests(gmmain.fIgnorableTestSubstrings) ||
+ !parse_flags_ignore_tests(gmmain.fIgnorableTestNames) ||
#if SK_SUPPORT_GPU
!parse_flags_gpu_cache(&gGpuCacheSizeBytes, &gGpuCacheSizeCount) ||
#endif
-out/Debug/gm --verbose --hierarchy --match selftest1 --ignoreFailuresFile gm/tests/inputs/ignored-tests.txt --config 8888 565 -r gm/tests/inputs/images/empty-dir --writeJsonSummaryPath gm/tests/outputs/compared-against-empty-dir/output-actual/json-summary.txt --writePath gm/tests/outputs/compared-against-empty-dir/output-actual/writePath --mismatchPath gm/tests/outputs/compared-against-empty-dir/output-actual/mismatchPath --missingExpectationsPath gm/tests/outputs/compared-against-empty-dir/output-actual/missingExpectationsPath
+out/Debug/gm --verbose --hierarchy --match selftest --ignoreFailuresFile gm/tests/inputs/ignored-tests.txt --config 8888 565 -r gm/tests/inputs/images/empty-dir --writeJsonSummaryPath gm/tests/outputs/compared-against-empty-dir/output-actual/json-summary.txt --writePath gm/tests/outputs/compared-against-empty-dir/output-actual/writePath --mismatchPath gm/tests/outputs/compared-against-empty-dir/output-actual/mismatchPath --missingExpectationsPath gm/tests/outputs/compared-against-empty-dir/output-actual/missingExpectationsPath
"failure-ignored" : null,
"no-comparison" : {
"565/selftest1.png" : [ "bitmap-64bitMD5", 12927999507540085554 ],
- "8888/selftest1.png" : [ "bitmap-64bitMD5", 1209453360120438698 ]
+ "565/selftest2.png" : [ "bitmap-64bitMD5", 8863920166200910451 ],
+ "8888/selftest1.png" : [ "bitmap-64bitMD5", 1209453360120438698 ],
+ "8888/selftest2.png" : [ "bitmap-64bitMD5", 13451349865803053525 ]
},
"succeeded" : null
},
"expected-results" : {
"565/selftest1.png" : {
"allowed-digests" : null,
+ "ignore-failure" : true
+ },
+ "565/selftest2.png" : {
+ "allowed-digests" : null,
"ignore-failure" : false
},
"8888/selftest1.png" : {
"allowed-digests" : null,
"ignore-failure" : true
+ },
+ "8888/selftest2.png" : {
+ "allowed-digests" : null,
+ "ignore-failure" : false
}
}
}
--- /dev/null
+[contents of gm/tests/outputs/compared-against-empty-dir/output-actual/missingExpectationsPath/565/selftest2.png]
--- /dev/null
+[contents of gm/tests/outputs/compared-against-empty-dir/output-actual/missingExpectationsPath/8888/selftest2.png]
GM: writing mismatches to gm/tests/outputs/compared-against-empty-dir/output-actual/mismatchPath
GM: writing images without expectations to gm/tests/outputs/compared-against-empty-dir/output-actual/missingExpectationsPath
GM: reading resources from resources
+GM: drawing... selftest2 [300 200]
GM: drawing... selftest1 [300 200]
-GM: Ran 1 GMs
+GM: Ran 2 GMs
GM: ... over 2 configs ["8888", "565"]
GM: ... and 1 modes [""]
-GM: ... so there should be a total of 2 tests.
-GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=2 WritingReferenceImage=0
+GM: ... so there should be a total of 4 tests.
+GM: Ran 4 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=4 WritingReferenceImage=0
GM: [*] 0 NoGpuContext:
GM: [ ] 0 IntentionallySkipped:
GM: [*] 0 RenderModeMismatch:
GM: [*] 0 GeneratePdfFailed:
GM: [*] 0 ExpectationsMismatch:
-GM: [ ] 2 MissingExpectations: 8888/selftest1 565/selftest1
+GM: [ ] 4 MissingExpectations: 8888/selftest2 565/selftest2 8888/selftest1 565/selftest1
GM: [*] 0 WritingReferenceImage:
GM: (results marked with [*] will cause nonzero return value)
--- /dev/null
+[contents of gm/tests/outputs/compared-against-empty-dir/output-actual/writePath/565/selftest2.png]
--- /dev/null
+[contents of gm/tests/outputs/compared-against-empty-dir/output-actual/writePath/8888/selftest2.png]
-out/Debug/gm --verbose --hierarchy --match selftest1 --ignoreFailuresFile gm/tests/inputs/ignored-tests.txt --config 8888 565 -r gm/tests/inputs/json/different-pixels.json --writeJsonSummaryPath gm/tests/outputs/ignoring-one-test/output-actual/json-summary.txt --writePath gm/tests/outputs/ignoring-one-test/output-actual/writePath --mismatchPath gm/tests/outputs/ignoring-one-test/output-actual/mismatchPath --missingExpectationsPath gm/tests/outputs/ignoring-one-test/output-actual/missingExpectationsPath
+out/Debug/gm --verbose --hierarchy --match selftest --ignoreFailuresFile gm/tests/inputs/ignored-tests.txt --config 8888 565 -r gm/tests/inputs/json/different-pixels.json --writeJsonSummaryPath gm/tests/outputs/ignoring-one-test/output-actual/json-summary.txt --writePath gm/tests/outputs/ignoring-one-test/output-actual/writePath --mismatchPath gm/tests/outputs/ignoring-one-test/output-actual/mismatchPath --missingExpectationsPath gm/tests/outputs/ignoring-one-test/output-actual/missingExpectationsPath
{
"actual-results" : {
"failed" : {
- "565/selftest1.png" : [ "bitmap-64bitMD5", 12927999507540085554 ]
+ "565/selftest2.png" : [ "bitmap-64bitMD5", 8863920166200910451 ],
+ "8888/selftest2.png" : [ "bitmap-64bitMD5", 13451349865803053525 ]
},
"failure-ignored" : {
+ "565/selftest1.png" : [ "bitmap-64bitMD5", 12927999507540085554 ],
"8888/selftest1.png" : [ "bitmap-64bitMD5", 1209453360120438698 ]
},
"no-comparison" : null,
"allowed-digests" : [
[ "bitmap-64bitMD5", 8863920166200910451 ]
],
+ "ignore-failure" : true
+ },
+ "565/selftest2.png" : {
+ "allowed-digests" : [
+ [ "bitmap-64bitMD5", 12927999507540085554 ]
+ ],
"ignore-failure" : false
},
"8888/selftest1.png" : {
[ "bitmap-64bitMD5", 13451349865803053525 ]
],
"ignore-failure" : true
+ },
+ "8888/selftest2.png" : {
+ "allowed-digests" : [
+ [ "bitmap-64bitMD5", 1209453360120438698 ]
+ ],
+ "ignore-failure" : false
}
}
}
--- /dev/null
+[contents of gm/tests/outputs/ignoring-one-test/output-actual/mismatchPath/565/selftest2.png]
--- /dev/null
+[contents of gm/tests/outputs/ignoring-one-test/output-actual/mismatchPath/8888/selftest2.png]
GM: writing mismatches to gm/tests/outputs/ignoring-one-test/output-actual/mismatchPath
GM: writing images without expectations to gm/tests/outputs/ignoring-one-test/output-actual/missingExpectationsPath
GM: reading resources from resources
+GM: drawing... selftest2 [300 200]
GM: drawing... selftest1 [300 200]
-GM: Ran 1 GMs
+GM: Ran 2 GMs
GM: ... over 2 configs ["8888", "565"]
GM: ... and 1 modes [""]
-GM: ... so there should be a total of 2 tests.
-GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0
+GM: ... so there should be a total of 4 tests.
+GM: Ran 4 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=4 MissingExpectations=0 WritingReferenceImage=0
GM: [*] 0 NoGpuContext:
GM: [ ] 0 IntentionallySkipped:
GM: [*] 0 RenderModeMismatch:
GM: [*] 0 GeneratePdfFailed:
-GM: [*] 2 ExpectationsMismatch: 8888/selftest1 565/selftest1
+GM: [*] 4 ExpectationsMismatch: 8888/selftest2 565/selftest2 8888/selftest1 565/selftest1
GM: [ ] 0 MissingExpectations:
GM: [*] 0 WritingReferenceImage:
GM: (results marked with [*] will cause nonzero return value)
--- /dev/null
+[contents of gm/tests/outputs/ignoring-one-test/output-actual/writePath/565/selftest2.png]
--- /dev/null
+[contents of gm/tests/outputs/ignoring-one-test/output-actual/writePath/8888/selftest2.png]
compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR
}
+# Swap contents of two files at paths $1 and $2.
+function swap_files {
+ if [ $# != 2 ]; then
+ echo "swap_files requires exactly 2 parameters, got $#"
+ exit 1
+ fi
+ mv "$1" "$1.tmp"
+ mv "$2" "$1"
+ mv "$1.tmp" "$2"
+}
+
# Create input dir (at path $1) with expectations (both image and json)
# that gm will match or mismatch as appropriate.
#
THIS_IMAGE_DIR=$IMAGES_DIR/different-pixels
mkdir -p $THIS_IMAGE_DIR
- $GM_BINARY --hierarchy --match selftest2 $CONFIGS -w $THIS_IMAGE_DIR
- mv $THIS_IMAGE_DIR/8888/selftest2.png $THIS_IMAGE_DIR/8888/selftest1.png
- mv $THIS_IMAGE_DIR/565/selftest2.png $THIS_IMAGE_DIR/565/selftest1.png
- $GM_BINARY --hierarchy --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \
+ $GM_BINARY --hierarchy --match selftest $CONFIGS -w $THIS_IMAGE_DIR
+ swap_files $THIS_IMAGE_DIR/8888/selftest2.png $THIS_IMAGE_DIR/8888/selftest1.png
+ swap_files $THIS_IMAGE_DIR/565/selftest2.png $THIS_IMAGE_DIR/565/selftest1.png
+ $GM_BINARY --hierarchy --match selftest $CONFIGS -r $THIS_IMAGE_DIR \
--writeJsonSummaryPath $JSON_DIR/different-pixels.json
# Create another JSON expectations file which is identical to
echo "# Comment line" >$GM_IGNORE_FAILURES_FILE
echo "" >>$GM_IGNORE_FAILURES_FILE
- echo "# ignore any test runs whose filename contains '8888/selfte'" >>$GM_IGNORE_FAILURES_FILE
- echo "# (in other words, config is 8888 and test name starts with 'selfte')" >>$GM_IGNORE_FAILURES_FILE
- echo "8888/selfte" >>$GM_IGNORE_FAILURES_FILE
+ echo "# ignore any runs of the 'selftest1' test" >>$GM_IGNORE_FAILURES_FILE
+ echo "selftest1" >>$GM_IGNORE_FAILURES_FILE
+ echo "" >>$GM_IGNORE_FAILURES_FILE
+ echo "# make sure we don't do partial matches (should NOT ignore 'selftest2' runs)" >>$GM_IGNORE_FAILURES_FILE
+ echo "selftest" >>$GM_IGNORE_FAILURES_FILE
}
GM_TESTDIR=gm/tests
gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json"
# Exercise --ignoreFailuresFile flag.
-gm_test "--verbose --hierarchy --match selftest1 --ignoreFailuresFile $GM_IGNORE_FAILURES_FILE $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/ignoring-one-test"
+# This should run two GM tests: selftest1 and selftest2.
+# Failures in selftest1 should be ignored, but failures in selftest2 should not.
+gm_test "--verbose --hierarchy --match selftest --ignoreFailuresFile $GM_IGNORE_FAILURES_FILE $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/ignoring-one-test"
# Compare different pixels, but with a SUBSET of the expectations marked as
# ignore-failure.
gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels-ignore-some-failures.json" "$GM_OUTPUTS/ignoring-some-failures"
# Compare generated image against an empty "expected image" dir.
-# Even the tests that have been marked as ignore-failure should show up as
-# no-comparison.
-gm_test "--verbose --hierarchy --match selftest1 --ignoreFailuresFile $GM_IGNORE_FAILURES_FILE $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir"
+# Even the tests that have been marked as ignore-failure (selftest1) should
+# show up as no-comparison.
+gm_test "--verbose --hierarchy --match selftest --ignoreFailuresFile $GM_IGNORE_FAILURES_FILE $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir"
# Compare generated image against a nonexistent "expected image" dir.
gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r ../path/to/nowhere" "$GM_OUTPUTS/compared-against-nonexistent-dir"
projects / platform / upstream / libSkiaSharp.git / commitdiff
