gm: Add ability to compare against checksums (as opposed to image files)
authorepoger@google.com <epoger@google.com@2bbb7eff-a529-9590-31e7-b0007b416f81>
Wed, 13 Feb 2013 18:14:48 +0000 (18:14 +0000)
committerepoger@google.com <epoger@google.com@2bbb7eff-a529-9590-31e7-b0007b416f81>
Wed, 13 Feb 2013 18:14:48 +0000 (18:14 +0000)
Review URL: https://codereview.appspot.com/7306071

git-svn-id: http://skia.googlecode.com/svn/trunk@7724 2bbb7eff-a529-9590-31e7-b0007b416f81

15 files changed:
gm/gm_expectations.h
gm/gmmain.cpp
gm/tests/outputs/compared-against-different-pixels-json/output-expected/command_line [new file with mode: 0644]
gm/tests/outputs/compared-against-different-pixels-json/output-expected/json-summary.txt [new file with mode: 0644]
gm/tests/outputs/compared-against-different-pixels-json/output-expected/return_value [new file with mode: 0644]
gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout [new file with mode: 0644]
gm/tests/outputs/compared-against-identical-bytes-json/output-expected/command_line [new file with mode: 0644]
gm/tests/outputs/compared-against-identical-bytes-json/output-expected/json-summary.txt [new file with mode: 0644]
gm/tests/outputs/compared-against-identical-bytes-json/output-expected/return_value [new file with mode: 0644]
gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout [new file with mode: 0644]
gm/tests/outputs/compared-against-identical-pixels-json/output-expected/command_line [new file with mode: 0644]
gm/tests/outputs/compared-against-identical-pixels-json/output-expected/json-summary.txt [new file with mode: 0644]
gm/tests/outputs/compared-against-identical-pixels-json/output-expected/return_value [new file with mode: 0644]
gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout [new file with mode: 0644]
gm/tests/run.sh

index 07f0e60..acff9d4 100644 (file)
 #include "gm.h"
 #include "SkBitmap.h"
 #include "SkBitmapChecksummer.h"
+#include "SkData.h"
 #include "SkImageDecoder.h"
 #include "SkOSFile.h"
 #include "SkRefCnt.h"
+#include "SkStream.h"
 #include "SkTArray.h"
 
 #ifdef SK_BUILD_FOR_WIN
     #pragma warning(push)
     #pragma warning(disable : 4530)
 #endif
+#include "json/reader.h"
 #include "json/value.h"
 #ifdef SK_BUILD_FOR_WIN
     #pragma warning(pop)
 #endif
 
+#define DEBUGFAIL_SEE_STDERR SkDEBUGFAIL("see stderr for message")
+
+const static char kJsonKey_ActualResults[]   = "actual-results";
+const static char kJsonKey_ActualResults_Failed[]        = "failed";
+const static char kJsonKey_ActualResults_FailureIgnored[]= "failure-ignored";
+const static char kJsonKey_ActualResults_NoComparison[]  = "no-comparison";
+const static char kJsonKey_ActualResults_Succeeded[]     = "succeeded";
+const static char kJsonKey_ActualResults_AnyStatus_Checksum[]    = "checksum";
+
+const static char kJsonKey_ExpectedResults[] = "expected-results";
+const static char kJsonKey_ExpectedResults_Checksums[]     = "checksums";
+const static char kJsonKey_ExpectedResults_IgnoreFailure[] = "ignore-failure";
+
 namespace skiagm {
 
     // The actual type we use to represent a checksum is hidden in here.
@@ -33,6 +49,9 @@ namespace skiagm {
     static inline Json::Value asJsonValue(Checksum checksum) {
         return checksum;
     }
+    static inline Checksum asChecksum(Json::Value jsonValue) {
+        return jsonValue.asUInt64();
+    }
 
     static SkString make_filename(const char path[],
                                   const char renderModeDescriptor[],
@@ -54,27 +73,69 @@ namespace skiagm {
     public:
         /**
          * No expectations at all.
-         *
-         * We set ignoreFailure to false by default, but it doesn't really
-         * matter... the result will always be "no-comparison" anyway.
          */
-        Expectations(bool ignoreFailure=false) {
+        Expectations(bool ignoreFailure=kDefaultIgnoreFailure) {
             fIgnoreFailure = ignoreFailure;
         }
 
         /**
          * Expect exactly one image (appropriate for the case when we
          * are comparing against a single PNG file).
-         *
-         * By default, DO NOT ignore failures.
          */
-        Expectations(const SkBitmap& bitmap, bool ignoreFailure=false) {
+        Expectations(const SkBitmap& bitmap, bool ignoreFailure=kDefaultIgnoreFailure) {
             fBitmap = bitmap;
             fIgnoreFailure = ignoreFailure;
             fAllowedChecksums.push_back() = SkBitmapChecksummer::Compute64(bitmap);
         }
 
         /**
+         * Create Expectations from a JSON element as found within the
+         * kJsonKey_ExpectedResults section.
+         *
+         * It's fine if the jsonElement is null or empty; in that case, we just
+         * don't have any expectations.
+         */
+        Expectations(Json::Value jsonElement) {
+            if (jsonElement.empty()) {
+                fIgnoreFailure = kDefaultIgnoreFailure;
+            } else {
+                Json::Value ignoreFailure = jsonElement[kJsonKey_ExpectedResults_IgnoreFailure];
+                if (ignoreFailure.isNull()) {
+                    fIgnoreFailure = kDefaultIgnoreFailure;
+                } else if (!ignoreFailure.isBool()) {
+                    fprintf(stderr, "found non-boolean json value for key '%s' in element '%s'\n",
+                            kJsonKey_ExpectedResults_IgnoreFailure,
+                            jsonElement.toStyledString().c_str());
+                    DEBUGFAIL_SEE_STDERR;
+                    fIgnoreFailure = kDefaultIgnoreFailure;
+                } else {
+                    fIgnoreFailure = ignoreFailure.asBool();
+                }
+
+                Json::Value allowedChecksums = jsonElement[kJsonKey_ExpectedResults_Checksums];
+                if (allowedChecksums.isNull()) {
+                    // ok, we'll just assume there aren't any expected checksums to compare against
+                } else if (!allowedChecksums.isArray()) {
+                    fprintf(stderr, "found non-array json value for key '%s' in element '%s'\n",
+                            kJsonKey_ExpectedResults_Checksums,
+                            jsonElement.toStyledString().c_str());
+                    DEBUGFAIL_SEE_STDERR;
+                } else {
+                    for (Json::ArrayIndex i=0; i<allowedChecksums.size(); i++) {
+                        Json::Value checksumElement = allowedChecksums[i];
+                        if (!checksumElement.isIntegral()) {
+                            fprintf(stderr, "found non-integer checksum in json element '%s'\n",
+                                    jsonElement.toStyledString().c_str());
+                            DEBUGFAIL_SEE_STDERR;
+                        } else {
+                            fAllowedChecksums.push_back() = asChecksum(checksumElement);
+                        }
+                    }
+                }
+            }
+        }
+
+        /**
          * Returns true iff we want to ignore failed expectations.
          */
         bool ignoreFailure() const { return this->fIgnoreFailure; }
@@ -126,6 +187,8 @@ namespace skiagm {
         }
 
     private:
+        const static bool kDefaultIgnoreFailure = false;
+
         SkTArray<Checksum> fAllowedChecksums;
         bool fIgnoreFailure;
         SkBitmap fBitmap;
@@ -181,5 +244,119 @@ namespace skiagm {
         const bool fNotifyOfMissingFiles;
     };
 
+    /**
+     * Return Expectations based on JSON summary file.
+     */
+    class JsonExpectationsSource : public ExpectationsSource {
+    public:
+        /**
+         * Create an ExpectationsSource that will return Expectations based on
+         * a JSON file.
+         *
+         * jsonPath: path to JSON file to read
+         */
+        JsonExpectationsSource(const char *jsonPath) {
+            parse(jsonPath, &fJsonRoot);
+            fJsonExpectedResults = fJsonRoot[kJsonKey_ExpectedResults];
+        }
+
+        Expectations get(const char *testName) SK_OVERRIDE {
+            return Expectations(fJsonExpectedResults[testName]);
+        }
+
+    private:
+
+        /**
+         * Read as many bytes as possible (up to maxBytes) from the stream into
+         * an SkData object.
+         *
+         * If the returned SkData contains fewer than maxBytes, then EOF has been
+         * reached and no more data would be available from subsequent calls.
+         * (If EOF has already been reached, then this call will return an empty
+         * SkData object immediately.)
+         *
+         * If there are fewer than maxBytes bytes available to read from the
+         * stream, but the stream has not been closed yet, this call will block
+         * until there are enough bytes to read or the stream has been closed.
+         *
+         * It is up to the caller to call unref() on the returned SkData object
+         * once the data is no longer needed, so that the underlying buffer will
+         * be freed.  For example:
+         *
+         * {
+         *   size_t maxBytes = 256;
+         *   SkAutoDataUnref dataRef(readIntoSkData(stream, maxBytes));
+         *   if (NULL != dataRef.get()) {
+         *     size_t bytesActuallyRead = dataRef.get()->size();
+         *     // use the data...
+         *   }
+         * }
+         * // underlying buffer has been freed, thanks to auto unref
+         *
+         */
+        // TODO(epoger): Move this, into SkStream.[cpp|h] as attempted in
+        // https://codereview.appspot.com/7300071 ?
+        // And maybe readFileIntoSkData() also?
+        static SkData* readIntoSkData(SkStream &stream, size_t maxBytes) {
+            if (0 == maxBytes) {
+                return SkData::NewEmpty();
+            }
+            char* bufStart = reinterpret_cast<char *>(sk_malloc_throw(maxBytes));
+            char* bufPtr = bufStart;
+            size_t bytesRemaining = maxBytes;
+            while (bytesRemaining > 0) {
+                size_t bytesReadThisTime = stream.read(bufPtr, bytesRemaining);
+                if (0 == bytesReadThisTime) {
+                    break;
+                }
+                bytesRemaining -= bytesReadThisTime;
+                bufPtr += bytesReadThisTime;
+            }
+            return SkData::NewFromMalloc(bufStart, maxBytes - bytesRemaining);
+        }
+
+        /**
+         * Wrapper around readIntoSkData for files: reads the entire file into
+         * an SkData object.
+         */
+        static SkData* readFileIntoSkData(SkFILEStream &stream) {
+            return readIntoSkData(stream, stream.getLength());
+        }
+
+        /**
+         * Read the file contents from jsonPath and parse them into jsonRoot.
+         *
+         * Returns true if successful.
+         */
+        static bool parse(const char *jsonPath, Json::Value *jsonRoot) {
+            SkFILEStream inFile(jsonPath);
+            if (!inFile.isValid()) {
+                fprintf(stderr, "unable to read JSON file %s\n", jsonPath);
+                DEBUGFAIL_SEE_STDERR;
+                return false;
+            }
+
+            SkAutoDataUnref dataRef(readFileIntoSkData(inFile));
+            if (NULL == dataRef.get()) {
+                fprintf(stderr, "error reading JSON file %s\n", jsonPath);
+                DEBUGFAIL_SEE_STDERR;
+                return false;
+            }
+
+            const char *bytes = reinterpret_cast<const char *>(dataRef.get()->data());
+            size_t size = dataRef.get()->size();
+            Json::Reader reader;
+            if (!reader.parse(bytes, bytes+size, *jsonRoot)) {
+                fprintf(stderr, "error parsing JSON file %s\n", jsonPath);
+                DEBUGFAIL_SEE_STDERR;
+                return false;
+            }
+            return true;
+        }
+
+        Json::Value fJsonRoot;
+        Json::Value fJsonExpectedResults;
+    };
+
 }
 #endif
index da7b79e..a3e2833 100644 (file)
@@ -94,17 +94,6 @@ const static ErrorBitfield ERROR_IMAGE_MISMATCH          = 0x02;
 const static ErrorBitfield ERROR_READING_REFERENCE_IMAGE = 0x08;
 const static ErrorBitfield ERROR_WRITING_REFERENCE_IMAGE = 0x10;
 
-const static char kJsonKey_ActualResults[]   = "actual-results";
-const static char kJsonKey_ActualResults_Failed[]        = "failed";
-const static char kJsonKey_ActualResults_FailureIgnored[]= "failure-ignored";
-const static char kJsonKey_ActualResults_NoComparison[]  = "no-comparison";
-const static char kJsonKey_ActualResults_Succeeded[]     = "succeeded";
-const static char kJsonKey_ActualResults_AnyStatus_Checksum[]    = "checksum";
-
-const static char kJsonKey_ExpectedResults[] = "expected-results";
-const static char kJsonKey_ExpectedResults_Checksums[]     = "checksums";
-const static char kJsonKey_ExpectedResults_IgnoreFailure[] = "ignore-failure";
-
 using namespace skiagm;
 
 struct FailRec {
@@ -1392,10 +1381,10 @@ int tool_main(int argc, char** argv) {
                 IndividualImageExpectationsSource,
                 (readPath, notifyMissingReadReference)));
         } else {
-            fprintf(stderr, "reading expectations from JSON summary file %s ",
+            fprintf(stderr, "reading expectations from JSON summary file %s\n",
                     readPath);
-            fprintf(stderr, "BUT WE DON'T KNOW HOW TO DO THIS YET!\n");
-            return -1;
+            gmmain.fExpectationsSource.reset(SkNEW_ARGS(
+                JsonExpectationsSource, (readPath)));
         }
     }
     if (writePath) {
diff --git a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/command_line b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/command_line
new file mode 100644 (file)
index 0000000..1226cf2
--- /dev/null
@@ -0,0 +1 @@
+out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/json/different-pixels.json --writeJsonSummary gm/tests/outputs/compared-against-different-pixels-json/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/json-summary.txt
new file mode 100644 (file)
index 0000000..37709e1
--- /dev/null
@@ -0,0 +1,25 @@
+{
+   "actual-results" : {
+      "failed" : {
+         "565/dashing2" : {
+            "checksum" : FAKE
+         },
+         "8888/dashing2" : {
+            "checksum" : FAKE
+         }
+      },
+      "failure-ignored" : null,
+      "no-comparison" : null,
+      "succeeded" : null
+   },
+   "expected-results" : {
+      "565/dashing2" : {
+         "checksums" : [ FAKE ],
+         "ignore-failure" : false
+      },
+      "8888/dashing2" : {
+         "checksums" : [ FAKE ],
+         "ignore-failure" : false
+      }
+   }
+}
diff --git a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/return_value b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/return_value
new file mode 100644 (file)
index 0000000..ace9d03
--- /dev/null
@@ -0,0 +1 @@
+255
diff --git a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout
new file mode 100644 (file)
index 0000000..9194b40
--- /dev/null
@@ -0,0 +1,3 @@
+reading expectations from JSON summary file gm/tests/inputs/json/different-pixels.json
+drawing... dashing2 [640 480]
+Ran 1 tests: 0 passed, 1 failed, 0 missing reference images
diff --git a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/command_line b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/command_line
new file mode 100644 (file)
index 0000000..e8fb0b9
--- /dev/null
@@ -0,0 +1 @@
+out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/json/identical-bytes.json --writeJsonSummary gm/tests/outputs/compared-against-identical-bytes-json/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/json-summary.txt
new file mode 100644 (file)
index 0000000..bed3b2d
--- /dev/null
@@ -0,0 +1,25 @@
+{
+   "actual-results" : {
+      "failed" : null,
+      "failure-ignored" : null,
+      "no-comparison" : null,
+      "succeeded" : {
+         "565/dashing2" : {
+            "checksum" : FAKE
+         },
+         "8888/dashing2" : {
+            "checksum" : FAKE
+         }
+      }
+   },
+   "expected-results" : {
+      "565/dashing2" : {
+         "checksums" : [ FAKE ],
+         "ignore-failure" : false
+      },
+      "8888/dashing2" : {
+         "checksums" : [ FAKE ],
+         "ignore-failure" : false
+      }
+   }
+}
diff --git a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/return_value b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/return_value
new file mode 100644 (file)
index 0000000..573541a
--- /dev/null
@@ -0,0 +1 @@
+0
diff --git a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout
new file mode 100644 (file)
index 0000000..afa854c
--- /dev/null
@@ -0,0 +1,3 @@
+reading expectations from JSON summary file gm/tests/inputs/json/identical-bytes.json
+drawing... dashing2 [640 480]
+Ran 1 tests: 1 passed, 0 failed, 0 missing reference images
diff --git a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/command_line b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/command_line
new file mode 100644 (file)
index 0000000..a596064
--- /dev/null
@@ -0,0 +1 @@
+out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/json/identical-pixels.json --writeJsonSummary gm/tests/outputs/compared-against-identical-pixels-json/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/json-summary.txt
new file mode 100644 (file)
index 0000000..bed3b2d
--- /dev/null
@@ -0,0 +1,25 @@
+{
+   "actual-results" : {
+      "failed" : null,
+      "failure-ignored" : null,
+      "no-comparison" : null,
+      "succeeded" : {
+         "565/dashing2" : {
+            "checksum" : FAKE
+         },
+         "8888/dashing2" : {
+            "checksum" : FAKE
+         }
+      }
+   },
+   "expected-results" : {
+      "565/dashing2" : {
+         "checksums" : [ FAKE ],
+         "ignore-failure" : false
+      },
+      "8888/dashing2" : {
+         "checksums" : [ FAKE ],
+         "ignore-failure" : false
+      }
+   }
+}
diff --git a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/return_value b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/return_value
new file mode 100644 (file)
index 0000000..573541a
--- /dev/null
@@ -0,0 +1 @@
+0
diff --git a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout
new file mode 100644 (file)
index 0000000..3b2671a
--- /dev/null
@@ -0,0 +1,3 @@
+reading expectations from JSON summary file gm/tests/inputs/json/identical-pixels.json
+drawing... dashing2 [640 480]
+Ran 1 tests: 1 passed, 0 failed, 0 missing reference images
index e1b6ce7..a90db38 100755 (executable)
@@ -145,12 +145,15 @@ create_inputs_dir $GM_INPUTS
 
 # Compare generated image against an input image file with identical bytes.
 gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/images/identical-bytes" "$GM_OUTPUTS/compared-against-identical-bytes-images"
+gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/json/identical-bytes.json" "$GM_OUTPUTS/compared-against-identical-bytes-json"
 
 # Compare generated image against an input image file with identical pixels but different PNG encoding.
 gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/images/identical-pixels" "$GM_OUTPUTS/compared-against-identical-pixels-images"
+gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/compared-against-identical-pixels-json"
 
 # Compare generated image against an input image file with different pixels.
 gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/images/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images"
+gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json"
 
 # Compare generated image against an empty "expected image" dir.
 gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir"