From e460a4718c0e9878aad2048cc1eddf12e430b71e Mon Sep 17 00:00:00 2001 From: "epoger@google.com" Date: Wed, 6 Feb 2013 18:41:04 +0000 Subject: [PATCH] gm self-tests: generate JSON expectations, in addition to image expectations Review URL: https://codereview.appspot.com/7310050 git-svn-id: http://skia.googlecode.com/svn/trunk@7624 2bbb7eff-a529-9590-31e7-b0007b416f81 --- .../output-expected/command_line | 1 + .../output-expected/json-summary.txt | 0 .../output-expected/return_value | 0 .../output-expected/stdout | 2 +- .../output-expected/command_line | 1 - .../output-expected/command_line | 2 +- .../output-expected/stdout | 6 +-- .../output-expected/command_line | 1 + .../output-expected/json-summary.txt | 0 .../output-expected/return_value | 0 .../output-expected/stdout | 2 +- .../output-expected/command_line | 1 - .../output-expected/command_line | 1 + .../output-expected/json-summary.txt | 0 .../output-expected/return_value | 0 .../output-expected/stdout | 2 +- .../output-expected/command_line | 1 - gm/tests/run.sh | 53 ++++++++++++++-------- 18 files changed, 43 insertions(+), 30 deletions(-) create mode 100644 gm/tests/outputs/compared-against-different-pixels-images/output-expected/command_line rename gm/tests/outputs/{compared-against-different-pixels => compared-against-different-pixels-images}/output-expected/json-summary.txt (100%) rename gm/tests/outputs/{compared-against-different-pixels => compared-against-different-pixels-images}/output-expected/return_value (100%) rename gm/tests/outputs/{compared-against-different-pixels => compared-against-different-pixels-images}/output-expected/stdout (62%) delete mode 100644 gm/tests/outputs/compared-against-different-pixels/output-expected/command_line create mode 100644 gm/tests/outputs/compared-against-identical-bytes-images/output-expected/command_line rename gm/tests/outputs/{compared-against-identical-pixels => compared-against-identical-bytes-images}/output-expected/json-summary.txt (100%) rename gm/tests/outputs/{compared-against-identical-pixels => compared-against-identical-bytes-images}/output-expected/return_value (100%) rename gm/tests/outputs/{compared-against-identical-bytes => compared-against-identical-bytes-images}/output-expected/stdout (63%) delete mode 100644 gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line create mode 100644 gm/tests/outputs/compared-against-identical-pixels-images/output-expected/command_line rename gm/tests/outputs/{compared-against-identical-bytes => compared-against-identical-pixels-images}/output-expected/json-summary.txt (100%) rename gm/tests/outputs/{compared-against-identical-bytes => compared-against-identical-pixels-images}/output-expected/return_value (100%) rename gm/tests/outputs/{compared-against-identical-pixels => compared-against-identical-pixels-images}/output-expected/stdout (62%) delete mode 100644 gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line diff --git a/gm/tests/outputs/compared-against-different-pixels-images/output-expected/command_line b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/command_line new file mode 100644 index 0000000..494eb85 --- /dev/null +++ b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/command_line @@ -0,0 +1 @@ +out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/images/different-pixels --writeJsonSummary gm/tests/outputs/compared-against-different-pixels-images/output-actual/json-summary.txt diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/json-summary.txt similarity index 100% rename from gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt rename to gm/tests/outputs/compared-against-different-pixels-images/output-expected/json-summary.txt diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/return_value b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/return_value similarity index 100% rename from gm/tests/outputs/compared-against-different-pixels/output-expected/return_value rename to gm/tests/outputs/compared-against-different-pixels-images/output-expected/return_value diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/stdout b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout similarity index 62% rename from gm/tests/outputs/compared-against-different-pixels/output-expected/stdout rename to gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout index 885dfd7..a001fc6 100644 --- a/gm/tests/outputs/compared-against-different-pixels/output-expected/stdout +++ b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout @@ -1,3 +1,3 @@ -reading from gm/tests/inputs/different-pixels +reading from gm/tests/inputs/images/different-pixels drawing... dashing2 [640 480] Ran 1 tests: 0 passed, 1 failed, 0 missing reference images diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line b/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line deleted file mode 100644 index d887baa..0000000 --- a/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line +++ /dev/null @@ -1 +0,0 @@ -out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/different-pixels --writeJsonSummary gm/tests/outputs/compared-against-different-pixels/output-actual/json-summary.txt diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line b/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line index d00ec60..637b397 100644 --- a/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line +++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line @@ -1 +1 @@ -out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/empty-dir --writeJsonSummary gm/tests/outputs/compared-against-empty-dir/output-actual/json-summary.txt +out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/images/empty-dir --writeJsonSummary gm/tests/outputs/compared-against-empty-dir/output-actual/json-summary.txt diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout index 246c05b..ee432a1 100644 --- a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout +++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout @@ -1,5 +1,5 @@ -reading from gm/tests/inputs/empty-dir +reading from gm/tests/inputs/images/empty-dir drawing... dashing2 [640 480] -FAILED to read gm/tests/inputs/empty-dir/8888/dashing2.png -FAILED to read gm/tests/inputs/empty-dir/565/dashing2.png +FAILED to read gm/tests/inputs/images/empty-dir/8888/dashing2.png +FAILED to read gm/tests/inputs/images/empty-dir/565/dashing2.png Ran 1 tests: 0 passed, 0 failed, 1 missing reference images diff --git a/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/command_line b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/command_line new file mode 100644 index 0000000..382655a --- /dev/null +++ b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/command_line @@ -0,0 +1 @@ +out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/images/identical-bytes --writeJsonSummary gm/tests/outputs/compared-against-identical-bytes-images/output-actual/json-summary.txt diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/json-summary.txt similarity index 100% rename from gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt rename to gm/tests/outputs/compared-against-identical-bytes-images/output-expected/json-summary.txt diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/return_value b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/return_value similarity index 100% rename from gm/tests/outputs/compared-against-identical-pixels/output-expected/return_value rename to gm/tests/outputs/compared-against-identical-bytes-images/output-expected/return_value diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/stdout b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout similarity index 63% rename from gm/tests/outputs/compared-against-identical-bytes/output-expected/stdout rename to gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout index 03153e6..8055992 100644 --- a/gm/tests/outputs/compared-against-identical-bytes/output-expected/stdout +++ b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout @@ -1,3 +1,3 @@ -reading from gm/tests/inputs/identical-bytes +reading from gm/tests/inputs/images/identical-bytes drawing... dashing2 [640 480] Ran 1 tests: 1 passed, 0 failed, 0 missing reference images diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line b/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line deleted file mode 100644 index d065b13..0000000 --- a/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line +++ /dev/null @@ -1 +0,0 @@ -out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/identical-bytes --writeJsonSummary gm/tests/outputs/compared-against-identical-bytes/output-actual/json-summary.txt diff --git a/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/command_line b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/command_line new file mode 100644 index 0000000..0b0ee91 --- /dev/null +++ b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/command_line @@ -0,0 +1 @@ +out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/images/identical-pixels --writeJsonSummary gm/tests/outputs/compared-against-identical-pixels-images/output-actual/json-summary.txt diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/json-summary.txt similarity index 100% rename from gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt rename to gm/tests/outputs/compared-against-identical-pixels-images/output-expected/json-summary.txt diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/return_value b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/return_value similarity index 100% rename from gm/tests/outputs/compared-against-identical-bytes/output-expected/return_value rename to gm/tests/outputs/compared-against-identical-pixels-images/output-expected/return_value diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/stdout b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout similarity index 62% rename from gm/tests/outputs/compared-against-identical-pixels/output-expected/stdout rename to gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout index a60b4ef..2f8fa86 100644 --- a/gm/tests/outputs/compared-against-identical-pixels/output-expected/stdout +++ b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout @@ -1,3 +1,3 @@ -reading from gm/tests/inputs/identical-pixels +reading from gm/tests/inputs/images/identical-pixels drawing... dashing2 [640 480] Ran 1 tests: 1 passed, 0 failed, 0 missing reference images diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line b/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line deleted file mode 100644 index 086f984..0000000 --- a/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line +++ /dev/null @@ -1 +0,0 @@ -out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/identical-pixels --writeJsonSummary gm/tests/outputs/compared-against-identical-pixels/output-actual/json-summary.txt diff --git a/gm/tests/run.sh b/gm/tests/run.sh index 717262e..e1b6ce7 100755 --- a/gm/tests/run.sh +++ b/gm/tests/run.sh @@ -85,8 +85,8 @@ function gm_test { compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR } -# Create input dir (at path $1) with images that match or mismatch -# as appropriate. +# Create input dir (at path $1) with expectations (both image and json) +# that gm will match or mismatch as appropriate. # # We used to check these files into SVN, but then we needed to rebasline them # when our drawing changed at all... so, as proposed in @@ -98,29 +98,42 @@ function create_inputs_dir { exit 1 fi INPUTS_DIR="$1" - mkdir -p $INPUTS_DIR + IMAGES_DIR=$INPUTS_DIR/images + JSON_DIR=$INPUTS_DIR/json + mkdir -p $IMAGES_DIR $JSON_DIR - mkdir -p $INPUTS_DIR/identical-bytes + mkdir -p $IMAGES_DIR/identical-bytes + # Run GM to write out the images actually generated. $GM_BINARY --hierarchy --match dashing2 $CONFIGS \ - -w $INPUTS_DIR/identical-bytes + -w $IMAGES_DIR/identical-bytes + # Run GM again to read in those images and write them out as a JSON summary. + $GM_BINARY --hierarchy --match dashing2 $CONFIGS \ + -r $IMAGES_DIR/identical-bytes \ + --writeJsonSummary $JSON_DIR/identical-bytes.json - mkdir -p $INPUTS_DIR/identical-pixels + mkdir -p $IMAGES_DIR/identical-pixels $GM_BINARY --hierarchy --match dashing2 $CONFIGS \ - -w $INPUTS_DIR/identical-pixels + -w $IMAGES_DIR/identical-pixels echo "more bytes that do not change the image pixels" \ - >> $INPUTS_DIR/identical-pixels/8888/dashing2.png + >> $IMAGES_DIR/identical-pixels/8888/dashing2.png echo "more bytes that do not change the image pixels" \ - >> $INPUTS_DIR/identical-pixels/565/dashing2.png + >> $IMAGES_DIR/identical-pixels/565/dashing2.png + $GM_BINARY --hierarchy --match dashing2 $CONFIGS \ + -r $IMAGES_DIR/identical-pixels \ + --writeJsonSummary $JSON_DIR/identical-pixels.json - mkdir -p $INPUTS_DIR/different-pixels + mkdir -p $IMAGES_DIR/different-pixels $GM_BINARY --hierarchy --match dashing3 $CONFIGS \ - -w $INPUTS_DIR/different-pixels - mv $INPUTS_DIR/different-pixels/8888/dashing3.png \ - $INPUTS_DIR/different-pixels/8888/dashing2.png - mv $INPUTS_DIR/different-pixels/565/dashing3.png \ - $INPUTS_DIR/different-pixels/565/dashing2.png + -w $IMAGES_DIR/different-pixels + mv $IMAGES_DIR/different-pixels/8888/dashing3.png \ + $IMAGES_DIR/different-pixels/8888/dashing2.png + mv $IMAGES_DIR/different-pixels/565/dashing3.png \ + $IMAGES_DIR/different-pixels/565/dashing2.png + $GM_BINARY --hierarchy --match dashing2 $CONFIGS \ + -r $IMAGES_DIR/different-pixels \ + --writeJsonSummary $JSON_DIR/different-pixels.json - mkdir -p $INPUTS_DIR/empty-dir + mkdir -p $IMAGES_DIR/empty-dir } GM_TESTDIR=gm/tests @@ -131,16 +144,16 @@ GM_TEMPFILES=$GM_TESTDIR/tempfiles create_inputs_dir $GM_INPUTS # Compare generated image against an input image file with identical bytes. -gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/identical-bytes" "$GM_OUTPUTS/compared-against-identical-bytes" +gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/images/identical-bytes" "$GM_OUTPUTS/compared-against-identical-bytes-images" # Compare generated image against an input image file with identical pixels but different PNG encoding. -gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/identical-pixels" "$GM_OUTPUTS/compared-against-identical-pixels" +gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/images/identical-pixels" "$GM_OUTPUTS/compared-against-identical-pixels-images" # Compare generated image against an input image file with different pixels. -gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels" +gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/images/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images" # Compare generated image against an empty "expected image" dir. -gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir" +gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir" # If run without "-r", the JSON's "actual-results" section should contain # actual checksums marked as "failure-ignored", but the "expected-results" -- 2.7.4