From caac3db48f74676889006b5520b766b572b5e9b2 Mon Sep 17 00:00:00 2001 From: "epoger@google.com" Date: Thu, 4 Apr 2013 19:23:11 +0000 Subject: [PATCH] re-land r8535, now that src.chromium.org is back up Review URL: https://codereview.chromium.org/13662002 git-svn-id: http://skia.googlecode.com/svn/trunk@8538 2bbb7eff-a529-9590-31e7-b0007b416f81 --- gm/gm_error.h | 9 ++-- gm/gmmain.cpp | 56 +++++++++++----------- .../output-expected/stdout | 5 +- .../output-expected/stdout | 5 +- .../output-expected/stdout | 5 +- .../output-expected/stdout | 5 +- .../output-expected/stdout | 5 +- .../output-expected/stdout | 5 +- .../output-expected/stdout | 5 +- .../outputs/no-readpath/output-expected/stdout | 5 +- .../output-expected/command_line | 1 + .../output-expected/json-summary.txt | 33 +++++++++++++ .../output-expected/return_value | 1 + .../pipe-playback-failure/output-expected/stderr | 1 + .../pipe-playback-failure/output-expected/stdout | 13 +++++ gm/tests/run.sh | 3 ++ 16 files changed, 110 insertions(+), 47 deletions(-) create mode 100644 gm/tests/outputs/pipe-playback-failure/output-expected/command_line create mode 100644 gm/tests/outputs/pipe-playback-failure/output-expected/json-summary.txt create mode 100644 gm/tests/outputs/pipe-playback-failure/output-expected/return_value create mode 100644 gm/tests/outputs/pipe-playback-failure/output-expected/stderr create mode 100644 gm/tests/outputs/pipe-playback-failure/output-expected/stdout diff --git a/gm/gm_error.h b/gm/gm_error.h index ced3391..aab0ec0 100644 --- a/gm/gm_error.h +++ b/gm/gm_error.h @@ -26,7 +26,8 @@ namespace skiagm { // or off (as long as the number of these errors is 0). kNoGpuContext_ErrorType, - kImageMismatch_ErrorType, + kRenderModeMismatch_ErrorType, + kExpectationsMismatch_ErrorType, kMissingExpectations_ErrorType, kWritingReferenceImage_ErrorType, kLast_ErrorType = kWritingReferenceImage_ErrorType @@ -39,8 +40,10 @@ namespace skiagm { switch(type) { case kNoGpuContext_ErrorType: return "NoGpuContext"; - case kImageMismatch_ErrorType: - return "ImageMismatch"; + case kRenderModeMismatch_ErrorType: + return "RenderModeMismatch"; + case kExpectationsMismatch_ErrorType: + return "ExpectationsMismatch"; case kMissingExpectations_ErrorType: return "MissingExpectations"; case kWritingReferenceImage_ErrorType: diff --git a/gm/gmmain.cpp b/gm/gmmain.cpp index 6cb47aa..8501064 100644 --- a/gm/gmmain.cpp +++ b/gm/gmmain.cpp @@ -272,7 +272,8 @@ public: } // Things to do only if there is some error condition. - SkString fullName = make_name(name.c_str(), renderModeDescriptor); + SkString fullName = name; + fullName.append(renderModeDescriptor); for (int typeInt = 0; typeInt <= kLast_ErrorType; typeInt++) { ErrorType type = static_cast(typeInt); if (errorCombination.includes(type)) { @@ -683,18 +684,17 @@ public: * @param baseNameString name of test without renderModeDescriptor added * @param renderModeDescriptor e.g., "-rtree", "-deferred" * @param addToJsonSummary whether to add these results (both actual and - * expected) to the JSON summary - * - * TODO: For now, addToJsonSummary is only set to true within - * compare_test_results_to_stored_expectations(), so results of our - * in-memory comparisons (Rtree vs regular, etc.) are not written to the - * JSON summary. We may wish to change that. + * expected) to the JSON summary. Regardless of this setting, if + * we find an image mismatch in this test, we will write these + * results to the JSON summary. (This is so that we will always + * report errors across rendering modes, such as pipe vs tiled. + * See https://codereview.chromium.org/13650002/ ) */ ErrorCombination compare_to_expectations(Expectations expectations, const SkBitmap& actualBitmap, const SkString& baseNameString, const char renderModeDescriptor[], - bool addToJsonSummary=false) { + bool addToJsonSummary) { ErrorCombination errors; Checksum actualChecksum = SkBitmapChecksummer::Compute64(actualBitmap); SkString completeNameString = baseNameString; @@ -704,7 +704,14 @@ public: if (expectations.empty()) { errors.add(kMissingExpectations_ErrorType); } else if (!expectations.match(actualChecksum)) { - errors.add(kImageMismatch_ErrorType); + addToJsonSummary = true; + // The error mode we record depends on whether this was running + // in a non-standard renderMode. + if ('\0' == *renderModeDescriptor) { + errors.add(kExpectationsMismatch_ErrorType); + } else { + errors.add(kRenderModeMismatch_ErrorType); + } // Write out the "actuals" for any mismatches, if we have // been directed to do so. @@ -753,7 +760,7 @@ public: // (where we can set ignore-failure to either true or // false), add test cases that exercise ignored // failures (both for kMissingExpectations_ErrorType - // and kImageMismatch_ErrorType). + // and kExpectationsMismatch_ErrorType). this->fJsonActualResults_FailureIgnored[testName] = actualResults; } else { @@ -772,7 +779,8 @@ public: this->fJsonActualResults_NoComparison[testName] = actualResults; } - if (result.includes(kImageMismatch_ErrorType)) { + if (result.includes(kExpectationsMismatch_ErrorType) || + result.includes(kRenderModeMismatch_ErrorType)) { this->fJsonActualResults_Failed[testName] = actualResults; } } @@ -866,18 +874,11 @@ public: GM* gm, const ConfigData& gRec, const char renderModeDescriptor [], SkBitmap& actualBitmap, const SkBitmap* referenceBitmap) { - // TODO(epoger): This method is run to compare results across - // different rendering modes (as opposed to - // compare_test_results_to_stored_expectations(), which - // compares results against expectations stored on disk). If - // we would like the GenerateGMs step to distinguish between - // those two types of mismatches, we should report image - // mismatches in here with a different ErrorType. SkASSERT(referenceBitmap); SkString name = make_name(gm->shortName(), gRec.fName); Expectations expectations(*referenceBitmap); return compare_to_expectations(expectations, actualBitmap, - name, renderModeDescriptor); + name, renderModeDescriptor, false); } static SkPicture* generate_new_picture(GM* gm, BbhType bbhType, uint32_t recordFlags, @@ -996,9 +997,8 @@ public: return kEmpty_ErrorCombination; } - ErrorCombination test_pipe_playback(GM* gm, - const ConfigData& gRec, - const SkBitmap& referenceBitmap) { + ErrorCombination test_pipe_playback(GM* gm, const ConfigData& gRec, + const SkBitmap& referenceBitmap, bool simulateFailure) { ErrorCombination errors; for (size_t i = 0; i < SK_ARRAY_COUNT(gPipeWritingFlagCombos); ++i) { SkBitmap bitmap; @@ -1010,7 +1010,9 @@ public: SkGPipeWriter writer; SkCanvas* pipeCanvas = writer.startRecording( &pipeController, gPipeWritingFlagCombos[i].flags); - invokeGM(gm, pipeCanvas, false, false); + if (!simulateFailure) { + invokeGM(gm, pipeCanvas, false, false); + } complete_bitmap(&bitmap); writer.endRecording(); SkString string("-pipe"); @@ -1177,6 +1179,7 @@ DEFINE_bool(replay, true, "Exercise the SkPicture replay test pass."); DEFINE_string2(resourcePath, i, "", "Directory that stores image resources."); DEFINE_bool(rtree, true, "Exercise the R-Tree variant of SkPicture test pass."); DEFINE_bool(serialize, true, "Exercise the SkPicture serialization & deserialization test pass."); +DEFINE_bool(simulatePipePlaybackFailure, false, "Simulate a rendering failure in pipe mode only."); DEFINE_bool(tiledPipe, false, "Exercise tiled SkGPipe replay."); DEFINE_bool(tileGrid, true, "Exercise the tile grid variant of SkPicture."); DEFINE_string(tileGridReplayScales, "", "Space separated list of floating-point scale " @@ -1411,10 +1414,6 @@ ErrorCombination run_multiple_modes(GMMain &gmmain, GM *gm, const ConfigData &co errorsForAllModes.add(pictErrors); } - // TODO: add a test in which the RTree rendering results in a - // different bitmap than the standard rendering. It should - // show up as failed in the JSON summary, and should be listed - // in the stdout also. if (!(gmFlags & GM::kSkipPicture_Flag) && FLAGS_rtree) { SkPicture* pict = gmmain.generate_new_picture( gm, kRTree_BbhType, SkPicture::kUsePathBoundsForClip_RecordingFlag); @@ -1459,7 +1458,8 @@ ErrorCombination run_multiple_modes(GMMain &gmmain, GM *gm, const ConfigData &co ErrorCombination pipeErrors; if (FLAGS_pipe) { - pipeErrors.add(gmmain.test_pipe_playback(gm, compareConfig, comparisonBitmap)); + pipeErrors.add(gmmain.test_pipe_playback(gm, compareConfig, comparisonBitmap, + FLAGS_simulatePipePlaybackFailure)); } if ((pipeErrors.isEmpty()) && diff --git a/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout index 3067409..8b8e61d 100644 --- a/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout +++ b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout @@ -4,9 +4,10 @@ GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] GM: ... and 7 modes ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"] GM: ... so there should be a total of 9 tests. -GM: Ran 9 tests: NoGpuContext=0 ImageMismatch=2 MissingExpectations=0 WritingReferenceImage=0 +GM: Ran 9 tests: NoGpuContext=0 RenderModeMismatch=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: -GM: [*] 2 ImageMismatch: 8888/selftest1 565/selftest1 +GM: [*] 0 RenderModeMismatch: +GM: [*] 2 ExpectationsMismatch: 8888/selftest1 565/selftest1 GM: [ ] 0 MissingExpectations: GM: [*] 0 WritingReferenceImage: GM: (results marked with [*] will cause nonzero return value) diff --git a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout index 84b2737..2fd0269 100644 --- a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout +++ b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout @@ -4,9 +4,10 @@ GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] GM: ... and 7 modes ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"] GM: ... so there should be a total of 9 tests. -GM: Ran 9 tests: NoGpuContext=0 ImageMismatch=2 MissingExpectations=0 WritingReferenceImage=0 +GM: Ran 9 tests: NoGpuContext=0 RenderModeMismatch=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: -GM: [*] 2 ImageMismatch: 8888/selftest1 565/selftest1 +GM: [*] 0 RenderModeMismatch: +GM: [*] 2 ExpectationsMismatch: 8888/selftest1 565/selftest1 GM: [ ] 0 MissingExpectations: GM: [*] 0 WritingReferenceImage: GM: (results marked with [*] will cause nonzero return value) diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout index a57916e..829c05e 100644 --- a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout +++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout @@ -4,9 +4,10 @@ GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] GM: ... and 7 modes ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"] GM: ... so there should be a total of 9 tests. -GM: Ran 9 tests: NoGpuContext=0 ImageMismatch=0 MissingExpectations=2 WritingReferenceImage=0 +GM: Ran 9 tests: NoGpuContext=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=2 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: -GM: [*] 0 ImageMismatch: +GM: [*] 0 RenderModeMismatch: +GM: [*] 0 ExpectationsMismatch: GM: [ ] 2 MissingExpectations: 8888/selftest1 565/selftest1 GM: [*] 0 WritingReferenceImage: GM: (results marked with [*] will cause nonzero return value) diff --git a/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout index c67e5bc..5788d69 100644 --- a/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout +++ b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout @@ -4,9 +4,10 @@ GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] GM: ... and 7 modes ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"] GM: ... so there should be a total of 9 tests. -GM: Ran 9 tests: NoGpuContext=0 ImageMismatch=0 MissingExpectations=0 WritingReferenceImage=0 +GM: Ran 9 tests: NoGpuContext=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: -GM: [*] 0 ImageMismatch: +GM: [*] 0 RenderModeMismatch: +GM: [*] 0 ExpectationsMismatch: GM: [ ] 0 MissingExpectations: GM: [*] 0 WritingReferenceImage: GM: (results marked with [*] will cause nonzero return value) diff --git a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout index 25f51b5..303a23f 100644 --- a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout +++ b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout @@ -4,9 +4,10 @@ GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] GM: ... and 7 modes ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"] GM: ... so there should be a total of 9 tests. -GM: Ran 9 tests: NoGpuContext=0 ImageMismatch=0 MissingExpectations=0 WritingReferenceImage=0 +GM: Ran 9 tests: NoGpuContext=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: -GM: [*] 0 ImageMismatch: +GM: [*] 0 RenderModeMismatch: +GM: [*] 0 ExpectationsMismatch: GM: [ ] 0 MissingExpectations: GM: [*] 0 WritingReferenceImage: GM: (results marked with [*] will cause nonzero return value) diff --git a/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout index d23ed86..4c4edba 100644 --- a/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout +++ b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout @@ -4,9 +4,10 @@ GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] GM: ... and 7 modes ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"] GM: ... so there should be a total of 9 tests. -GM: Ran 9 tests: NoGpuContext=0 ImageMismatch=0 MissingExpectations=0 WritingReferenceImage=0 +GM: Ran 9 tests: NoGpuContext=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: -GM: [*] 0 ImageMismatch: +GM: [*] 0 RenderModeMismatch: +GM: [*] 0 ExpectationsMismatch: GM: [ ] 0 MissingExpectations: GM: [*] 0 WritingReferenceImage: GM: (results marked with [*] will cause nonzero return value) diff --git a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout index 147c383..9a9d91b 100644 --- a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout +++ b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout @@ -4,9 +4,10 @@ GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] GM: ... and 7 modes ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"] GM: ... so there should be a total of 9 tests. -GM: Ran 9 tests: NoGpuContext=0 ImageMismatch=0 MissingExpectations=0 WritingReferenceImage=0 +GM: Ran 9 tests: NoGpuContext=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: -GM: [*] 0 ImageMismatch: +GM: [*] 0 RenderModeMismatch: +GM: [*] 0 ExpectationsMismatch: GM: [ ] 0 MissingExpectations: GM: [*] 0 WritingReferenceImage: GM: (results marked with [*] will cause nonzero return value) diff --git a/gm/tests/outputs/no-readpath/output-expected/stdout b/gm/tests/outputs/no-readpath/output-expected/stdout index e3b101e..1bdc86d 100644 --- a/gm/tests/outputs/no-readpath/output-expected/stdout +++ b/gm/tests/outputs/no-readpath/output-expected/stdout @@ -3,9 +3,10 @@ GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] GM: ... and 7 modes ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"] GM: ... so there should be a total of 9 tests. -GM: Ran 9 tests: NoGpuContext=0 ImageMismatch=0 MissingExpectations=2 WritingReferenceImage=0 +GM: Ran 9 tests: NoGpuContext=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=2 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: -GM: [*] 0 ImageMismatch: +GM: [*] 0 RenderModeMismatch: +GM: [*] 0 ExpectationsMismatch: GM: [ ] 2 MissingExpectations: 8888/selftest1 565/selftest1 GM: [*] 0 WritingReferenceImage: GM: (results marked with [*] will cause nonzero return value) diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/command_line b/gm/tests/outputs/pipe-playback-failure/output-expected/command_line new file mode 100644 index 0000000..950339d --- /dev/null +++ b/gm/tests/outputs/pipe-playback-failure/output-expected/command_line @@ -0,0 +1 @@ +out/Debug/gm --simulatePipePlaybackFailure --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/identical-pixels.json --writeJsonSummaryPath gm/tests/outputs/pipe-playback-failure/output-actual/json-summary.txt diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/json-summary.txt b/gm/tests/outputs/pipe-playback-failure/output-expected/json-summary.txt new file mode 100644 index 0000000..249fe09 --- /dev/null +++ b/gm/tests/outputs/pipe-playback-failure/output-expected/json-summary.txt @@ -0,0 +1,33 @@ +{ + "actual-results" : { + "failed" : { + "comparison/selftest1-pipe" : { + "checksum" : 4259036727585789440 + } + }, + "failure-ignored" : null, + "no-comparison" : null, + "succeeded" : { + "565/selftest1" : { + "checksum" : 9512553915271796906 + }, + "8888/selftest1" : { + "checksum" : 14022967492765711532 + } + } + }, + "expected-results" : { + "565/selftest1" : { + "checksums" : [ 9512553915271796906 ], + "ignore-failure" : false + }, + "8888/selftest1" : { + "checksums" : [ 14022967492765711532 ], + "ignore-failure" : false + }, + "comparison/selftest1-pipe" : { + "checksums" : [ 14022967492765711532 ], + "ignore-failure" : false + } + } +} diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/return_value b/gm/tests/outputs/pipe-playback-failure/output-expected/return_value new file mode 100644 index 0000000..ace9d03 --- /dev/null +++ b/gm/tests/outputs/pipe-playback-failure/output-expected/return_value @@ -0,0 +1 @@ +255 diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/stderr b/gm/tests/outputs/pipe-playback-failure/output-expected/stderr new file mode 100644 index 0000000..e65d374 --- /dev/null +++ b/gm/tests/outputs/pipe-playback-failure/output-expected/stderr @@ -0,0 +1 @@ +GM: ---- comparison/selftest1-pipe: 60000 (of 60000) differing pixels, max per-channel mismatch R=135 G=246 B=135 A=0 diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/stdout b/gm/tests/outputs/pipe-playback-failure/output-expected/stdout new file mode 100644 index 0000000..52fc54f --- /dev/null +++ b/gm/tests/outputs/pipe-playback-failure/output-expected/stdout @@ -0,0 +1,13 @@ +GM: reading expectations from JSON summary file gm/tests/inputs/json/identical-pixels.json +GM: drawing... selftest1 [300 200] +GM: Ran 1 GMs +GM: ... over 2 configs ["8888", "565"] +GM: ... and 5 modes ["pipe", "replay", "rtree", "serialize", "tilegrid"] +GM: ... so there should be a total of 7 tests. +GM: Ran 7 tests: NoGpuContext=0 RenderModeMismatch=1 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0 +GM: [*] 0 NoGpuContext: +GM: [*] 1 RenderModeMismatch: comparison/selftest1-pipe +GM: [*] 0 ExpectationsMismatch: +GM: [ ] 0 MissingExpectations: +GM: [*] 0 WritingReferenceImage: +GM: (results marked with [*] will cause nonzero return value) diff --git a/gm/tests/run.sh b/gm/tests/run.sh index 7ec08be..093f7d7 100755 --- a/gm/tests/run.sh +++ b/gm/tests/run.sh @@ -156,4 +156,7 @@ gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir" # section should be empty. gm_test "--hierarchy --match selftest1 $CONFIGS" "$GM_OUTPUTS/no-readpath" +# Test what happens if a subset of the renderModes fail (e.g. pipe) +gm_test "--simulatePipePlaybackFailure --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/pipe-playback-failure" + echo "All tests passed." -- 2.7.4