From 354195abe304cbf9cdc6a83f89d9e059edc4cead Mon Sep 17 00:00:00 2001 From: Hugues Evrard Date: Wed, 19 Apr 2017 11:03:12 +0100 Subject: [PATCH] Add support for metamorphic testing Add a new test case which compare the renderings of two different yet semantically equivalent fragment shaders. A couple of shadersets are also added as associated data, each contains one recipient fragment shader which is the reference, and a few variants which are semantically equivalent to the recipient. Each variant included here has proven to trigger some issue on an Android device, see comment in the variant source code. Besides comparing the renderings of both recipient and variant shaders, the test case also checks that each shader leads to a deterministic rendering, by performing rendering twice and doing a pixel-per-pixel comparison on the resulting image. When comparing the recipient and variant renderings, a slight difference is tolerated to cope with variations due to floating point arithmetics. Moreover, a sanity check function may be used in order to verify that the recipient image is relevant, and thus avoid a e.g. full black image rendering to pass the test. Change-Id: Id6cbbe7459864c6fb3acf72faa393ee0f4d8315d --- AndroidGen.mk | 1 + android/cts/master/gles3-master.txt | 6 + android/package/Android.mk | 3 +- .../graphicsfuzz/data/gles3/graphicsfuzz/LICENSE | 19 ++ .../graphicsfuzz/bubblesort_flag/recipient.frag | 67 ++++ .../graphicsfuzz/bubblesort_flag/variant_1.frag | 81 +++++ .../graphicsfuzz/bubblesort_flag/variant_2.frag | 77 +++++ .../gles3/graphicsfuzz/synthetic/recipient.frag | 58 ++++ .../gles3/graphicsfuzz/synthetic/variant_1.frag | 75 +++++ .../gles3/graphicsfuzz/synthetic/variant_2.frag | 78 +++++ .../gles3/graphicsfuzz/synthetic/variant_3.frag | 79 +++++ .../gles3/graphicsfuzz/synthetic/variant_4.frag | 67 ++++ .../data/gles3/graphicsfuzz/vertexShader.glsl | 13 + modules/gles3/CMakeLists.txt | 5 +- modules/gles3/functional/CMakeLists.txt | 2 + modules/gles3/functional/es3fFunctionalTests.cpp | 3 +- .../functional/es3fShaderMetamorphicTests.cpp | 355 +++++++++++++++++++++ .../functional/es3fShaderMetamorphicTests.hpp | 52 +++ 18 files changed, 1037 insertions(+), 4 deletions(-) create mode 100644 external/graphicsfuzz/data/gles3/graphicsfuzz/LICENSE create mode 100755 external/graphicsfuzz/data/gles3/graphicsfuzz/bubblesort_flag/recipient.frag create mode 100755 external/graphicsfuzz/data/gles3/graphicsfuzz/bubblesort_flag/variant_1.frag create mode 100755 external/graphicsfuzz/data/gles3/graphicsfuzz/bubblesort_flag/variant_2.frag create mode 100755 external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/recipient.frag create mode 100755 external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/variant_1.frag create mode 100755 external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/variant_2.frag create mode 100755 external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/variant_3.frag create mode 100755 external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/variant_4.frag create mode 100644 external/graphicsfuzz/data/gles3/graphicsfuzz/vertexShader.glsl create mode 100644 modules/gles3/functional/es3fShaderMetamorphicTests.cpp create mode 100644 modules/gles3/functional/es3fShaderMetamorphicTests.hpp diff --git a/AndroidGen.mk b/AndroidGen.mk index ac92f54..ab257d7 100644 --- a/AndroidGen.mk +++ b/AndroidGen.mk @@ -727,6 +727,7 @@ LOCAL_SRC_FILES := \ modules/gles3/functional/es3fShaderInvarianceTests.cpp \ modules/gles3/functional/es3fShaderLoopTests.cpp \ modules/gles3/functional/es3fShaderMatrixTests.cpp \ + modules/gles3/functional/es3fShaderMetamorphicTests.cpp \ modules/gles3/functional/es3fShaderOperatorTests.cpp \ modules/gles3/functional/es3fShaderPackingFunctionTests.cpp \ modules/gles3/functional/es3fShaderPrecisionTests.cpp \ diff --git a/android/cts/master/gles3-master.txt b/android/cts/master/gles3-master.txt index 0fb6599..8e08302 100644 --- a/android/cts/master/gles3-master.txt +++ b/android/cts/master/gles3-master.txt @@ -19248,6 +19248,12 @@ dEQP-GLES3.functional.shaders.constant_expressions.builtin_functions.geometric.n dEQP-GLES3.functional.shaders.constant_expressions.builtin_functions.geometric.normalize_vec3_fragment dEQP-GLES3.functional.shaders.constant_expressions.builtin_functions.geometric.normalize_vec4_vertex dEQP-GLES3.functional.shaders.constant_expressions.builtin_functions.geometric.normalize_vec4_fragment +dEQP-GLES3.functional.shaders.metamorphic.synthetic.variant_1 +dEQP-GLES3.functional.shaders.metamorphic.synthetic.variant_2 +dEQP-GLES3.functional.shaders.metamorphic.synthetic.variant_3 +dEQP-GLES3.functional.shaders.metamorphic.synthetic.variant_4 +dEQP-GLES3.functional.shaders.metamorphic.bubblesort_flag.variant_1 +dEQP-GLES3.functional.shaders.metamorphic.bubblesort_flag.variant_2 dEQP-GLES3.functional.shaders.random.basic_expression.vertex.0 dEQP-GLES3.functional.shaders.random.basic_expression.vertex.1 dEQP-GLES3.functional.shaders.random.basic_expression.vertex.2 diff --git a/android/package/Android.mk b/android/package/Android.mk index 00db989..6b46292 100644 --- a/android/package/Android.mk +++ b/android/package/Android.mk @@ -31,7 +31,8 @@ LOCAL_JNI_SHARED_LIBRARIES := libdeqp LOCAL_ASSET_DIR := \ $(LOCAL_PATH)/../../data \ $(LOCAL_PATH)/../../external/vulkancts/data \ - $(LOCAL_PATH)/../../../../prebuilts/deqp/spirv + $(LOCAL_PATH)/../../../../prebuilts/deqp/spirv \ + $(LOCAL_PATH)/../../external/graphicsfuzz/data LOCAL_PACKAGE_NAME := com.drawelements.deqp LOCAL_MULTILIB := both diff --git a/external/graphicsfuzz/data/gles3/graphicsfuzz/LICENSE b/external/graphicsfuzz/data/gles3/graphicsfuzz/LICENSE new file mode 100644 index 0000000..15ad286 --- /dev/null +++ b/external/graphicsfuzz/data/gles3/graphicsfuzz/LICENSE @@ -0,0 +1,19 @@ +Copyright 2017 Alastair F. Donaldson, Hugues Evrard, Imperial College London + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. diff --git a/external/graphicsfuzz/data/gles3/graphicsfuzz/bubblesort_flag/recipient.frag b/external/graphicsfuzz/data/gles3/graphicsfuzz/bubblesort_flag/recipient.frag new file mode 100755 index 0000000..5e09f06 --- /dev/null +++ b/external/graphicsfuzz/data/gles3/graphicsfuzz/bubblesort_flag/recipient.frag @@ -0,0 +1,67 @@ +#version 100 + +// Copyright Alastair F. Donaldson and Hugues Evrard, Imperial College London, 2017 + +#ifdef GL_ES +#ifdef GL_FRAGMENT_PRECISION_HIGH +precision highp float; +precision highp int; +#else +precision mediump float; +precision mediump int; +#endif +#endif + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +bool checkSwap(float a, float b) +{ + return gl_FragCoord.y < resolution.y / 2.0 ? a > b : a < b; +} +void main() +{ + float data[10]; + for( + int i = 0; + i < 10; + i ++ + ) + { + data[i] = float(10 - i) * injectionSwitch.y; + } + for( + int i = 0; + i < 9; + i ++ + ) + { + for( + int j = 0; + j < 10; + j ++ + ) + { + if(j < i + 1) + { + continue; + } + bool doSwap = checkSwap(data[i], data[j]); + if(doSwap) + { + float temp = data[i]; + data[i] = data[j]; + data[j] = temp; + } + } + } + if(gl_FragCoord.x < resolution.x / 2.0) + { + gl_FragColor = vec4(data[0] / 10.0, data[5] / 10.0, data[9] / 10.0, 1.0); + } + else + { + gl_FragColor = vec4(data[5] / 10.0, data[9] / 10.0, data[0] / 10.0, 1.0); + } +} diff --git a/external/graphicsfuzz/data/gles3/graphicsfuzz/bubblesort_flag/variant_1.frag b/external/graphicsfuzz/data/gles3/graphicsfuzz/bubblesort_flag/variant_1.frag new file mode 100755 index 0000000..5ac2db3 --- /dev/null +++ b/external/graphicsfuzz/data/gles3/graphicsfuzz/bubblesort_flag/variant_1.frag @@ -0,0 +1,81 @@ +#version 100 + +// Copyright Alastair F. Donaldson and Hugues Evrard, Imperial College London, 2017 +// Defect found using GLFuzz - https://www.graphicsfuzz.com/ +// +// Gives wrong image on: +// NVIDIA SHIELD Android TV +// Model number: P2571 +// GL_VERSION: OpenGL ES 3.2 NVIDIA 361.00 +// GL_VENDOR: NVIDIA Corporation +// GL_RENDERER: NVIDIA Tegra +// Android version: 7.0 + +#ifdef GL_ES +#ifdef GL_FRAGMENT_PRECISION_HIGH +precision highp float; +precision highp int; +#else +precision mediump float; +precision mediump int; +#endif +#endif + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +bool checkSwap(float a, float b) +{ + return gl_FragCoord.y < resolution.y / 2.0 ? a > b : a < b; +} +void main() +{ + float data[10]; + for( + int i = 0; + i < 10; + i ++ + ) + { + data[i] = float(10 - i) * injectionSwitch.y; + } + for( + int i = 0; + i < 9; + i ++ + ) + { + for( + int j = 0; + j < 10; + j ++ + ) + { + if(j < i + 1) + { + if(false) + { + continue; + } + continue; + discard; + } + bool doSwap = checkSwap(data[i], data[j]); + if(doSwap) + { + float temp = data[i]; + data[i] = data[j]; + data[j] = temp; + } + } + } + if(gl_FragCoord.x < resolution.x / 2.0) + { + gl_FragColor = vec4(data[0] / 10.0, data[5] / 10.0, data[9] / 10.0, 1.0); + } + else + { + gl_FragColor = vec4(data[5] / 10.0, data[9] / 10.0, data[0] / 10.0, 1.0); + } +} diff --git a/external/graphicsfuzz/data/gles3/graphicsfuzz/bubblesort_flag/variant_2.frag b/external/graphicsfuzz/data/gles3/graphicsfuzz/bubblesort_flag/variant_2.frag new file mode 100755 index 0000000..4338f70 --- /dev/null +++ b/external/graphicsfuzz/data/gles3/graphicsfuzz/bubblesort_flag/variant_2.frag @@ -0,0 +1,77 @@ +#version 100 + +// Copyright Alastair F. Donaldson and Hugues Evrard, Imperial College London, 2017 +// Defect found using GLFuzz - https://www.graphicsfuzz.com/ +// +// Gives wrong image on: +// NVIDIA SHIELD Android TV +// Model number: P2571 +// GL_VERSION: OpenGL ES 3.2 NVIDIA 361.00 +// GL_VENDOR: NVIDIA Corporation +// GL_RENDERER: NVIDIA Tegra +// Android version: 7.0 + +#ifdef GL_ES +#ifdef GL_FRAGMENT_PRECISION_HIGH +precision highp float; +precision highp int; +#else +precision mediump float; +precision mediump int; +#endif +#endif + +uniform vec2 injectionSwitch; + +uniform vec2 resolution; + +bool checkSwap(float a, float b) +{ + (true ? 1.0 : 1.0) + vec4(1.0); + return gl_FragCoord.y < resolution.y / 2.0 ? a > b : a < b; +} +void main() +{ + float data[10]; + for( + int i = 0; + i < 10; + i ++ + ) + { + data[i] = float(10 - i) * injectionSwitch.y; + } + for( + int i = 0; + i < 9; + i ++ + ) + { + for( + int j = 0; + j < 10; + j ++ + ) + { + if(j < i + 1) + { + continue; + } + bool doSwap = checkSwap(data[i], data[j]); + if(doSwap) + { + float temp = data[i]; + data[i] = data[j]; + data[j] = temp; + } + } + } + if(gl_FragCoord.x < resolution.x / 2.0) + { + gl_FragColor = vec4(data[0] / 10.0, data[5] / 10.0, data[9] / 10.0, 1.0); + } + else + { + gl_FragColor = vec4(data[5] / 10.0, data[9] / 10.0, data[0] / 10.0, 1.0); + } +} diff --git a/external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/recipient.frag b/external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/recipient.frag new file mode 100755 index 0000000..1a17e85 --- /dev/null +++ b/external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/recipient.frag @@ -0,0 +1,58 @@ +#version 100 + +// Copyright Alastair F. Donaldson and Hugues Evrard, Imperial College London, 2017 + +#ifdef GL_ES +#ifdef GL_FRAGMENT_PRECISION_HIGH +precision highp float; +precision highp int; +#else +precision mediump float; +precision mediump int; +#endif +#endif + +uniform vec2 injectionSwitch; + +void main() +{ + int r; + int g; + int b; + int a; + r = 100 * int(injectionSwitch.y); + g = int(injectionSwitch.x) * int(injectionSwitch.y); + b = 2 * int(injectionSwitch.x); + a = g - int(injectionSwitch.x); + for( + int i = 0; + i < 10; + i ++ + ) + { + r --; + g ++; + b ++; + a ++; + for( + int j = 1; + j < 10; + j ++ + ) + { + a ++; + b ++; + g ++; + r --; + } + } + float fr; + float fg; + float fb; + float fa; + fr = float(r / 100); + fg = float(g / 100); + fb = float(b / 100); + fa = float(a / 100); + gl_FragColor = vec4(r, g, b, a); +} diff --git a/external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/variant_1.frag b/external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/variant_1.frag new file mode 100755 index 0000000..e87c5c7 --- /dev/null +++ b/external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/variant_1.frag @@ -0,0 +1,75 @@ +#version 100 + +// Copyright Alastair F. Donaldson and Hugues Evrard, Imperial College London, 2017 +// Defect found using GLFuzz - https://www.graphicsfuzz.com/ +// +// Gives link error on: +// Samsung Galaxy S7 +// Model number: SM-G930F +// GL_VERSION: OpenGL ES 3.2 v1.r12p1-03dev0.92cea07ab1b80f9f42b782b097e01409 +// GL_VENDOR: ARM +// GL_RENDERER: Mali-T880 +// Android version: 7.0 + +#ifdef GL_ES +#ifdef GL_FRAGMENT_PRECISION_HIGH +precision highp float; +precision highp int; +#else +precision mediump float; +precision mediump int; +#endif +#endif + +uniform vec2 injectionSwitch; + +void main() +{ + int r; + int g; + int b; + int a; + r = 100 * int(injectionSwitch.y); + g = int(injectionSwitch.x) * int(injectionSwitch.y); + b = 2 * int(injectionSwitch.x); + a = g - int(injectionSwitch.x); + for( + int i = 0; + i < 10; + i ++ + ) + { + r --; + g ++; + b ++; + a ++; + for( + int j = 1; + j < 10; + j ++ + ) + { + if(injectionSwitch.x > injectionSwitch.y) + { + continue; + } + if(injectionSwitch.x > injectionSwitch.y) + { + discard; + } + a ++; + b ++; + g ++; + r --; + } + } + float fr; + float fg; + float fb; + float fa; + fr = float(r / 100); + fg = float(g / 100); + fb = float(b / 100); + fa = float(a / 100); + gl_FragColor = vec4(r, g, b, a); +} diff --git a/external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/variant_2.frag b/external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/variant_2.frag new file mode 100755 index 0000000..820fa5d --- /dev/null +++ b/external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/variant_2.frag @@ -0,0 +1,78 @@ +#version 100 + +// Copyright Alastair F. Donaldson and Hugues Evrard, Imperial College London, 2017 +// Defect found using GLFuzz - https://www.graphicsfuzz.com/ +// +// Causes crash on: +// Samsung Galaxy S7 +// Model number: SM-G930P +// GL_VERSION: OpenGL ES 3.2 V@145.0 (GIT@I86b60582e4) +// GL_VENDOR: Qualcomm +// GL_RENDERER: Adreno (TM) 530 +// Android version: 7.0 + +#ifdef GL_ES +#ifdef GL_FRAGMENT_PRECISION_HIGH +precision highp float; +precision highp int; +#else +precision mediump float; +precision mediump int; +#endif +#endif + +uniform vec2 injectionSwitch; + +vec4 f() +{ + if(injectionSwitch.x < injectionSwitch.y + ? injectionSwitch.x > injectionSwitch.y + : false) + { + return vec4(1.0); + } + return vec4(1.0); +} +void main() +{ + int r; + int g; + int b; + int a; + r = 100 * int(injectionSwitch.y); + g = int(injectionSwitch.x) * int(injectionSwitch.y); + b = 2 * int(injectionSwitch.x); + a = g - int(injectionSwitch.x); + for( + int i = 0; + i < 10; + i ++ + ) + { + r --; + g ++; + b ++; + a ++; + for( + int j = 1; + j < 10; + j ++ + ) + { + f(); + a ++; + b ++; + g ++; + r --; + } + } + float fr; + float fg; + float fb; + float fa; + fr = float(r / 100); + fg = float(g / 100); + fb = float(b / 100); + fa = float(a / 100); + gl_FragColor = vec4(r, g, b, a); +} diff --git a/external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/variant_3.frag b/external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/variant_3.frag new file mode 100755 index 0000000..78236b7 --- /dev/null +++ b/external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/variant_3.frag @@ -0,0 +1,79 @@ +#version 100 + +// Copyright Alastair F. Donaldson and Hugues Evrard, Imperial College London, 2017 +// Defect found using GLFuzz - https://www.graphicsfuzz.com/ +// +// Causes black image to be rendered on: +// ASUS ZenPad 3S 10 +// Model number: P027 +// GL_VERSION: OpenGL ES 3.1 build 1.5@3830101 +// GL_VENDOR: Imagination Technologies +// GL_RENDERER: PowerVR Rogue GX6250 +// Android version: 6.0 + +#ifdef GL_ES +#ifdef GL_FRAGMENT_PRECISION_HIGH +precision highp float; +precision highp int; +#else +precision mediump float; +precision mediump int; +#endif +#endif + +uniform vec2 injectionSwitch; + +void main() +{ + int r; + int g; + int b; + int a; + r = 100 * int(injectionSwitch.y); + g = int(injectionSwitch.x) * int(injectionSwitch.y); + b = 2 * int(injectionSwitch.x); + a = g - int(injectionSwitch.x); + for( + int i = 0; + i < 10; + i ++ + ) + { + r --; + g ++; + b ++; + a ++; + for( + int j = 1; + j < 10; + j ++ + ) + { + if(injectionSwitch.x > injectionSwitch.y) + { + break; + } + a ++; + if(injectionSwitch.x > injectionSwitch.y) + { + break; + } + b ++; + g ++; + r --; + if(injectionSwitch.x > injectionSwitch.y) + { + return; + } + } + } + float fr; + float fg; + float fb; + float fa; + fr = float(r / 100); + fg = float(g / 100); + fb = float(b / 100); + fa = float(a / 100); + gl_FragColor = vec4(r, g, b, a); +} diff --git a/external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/variant_4.frag b/external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/variant_4.frag new file mode 100755 index 0000000..9a60a96 --- /dev/null +++ b/external/graphicsfuzz/data/gles3/graphicsfuzz/synthetic/variant_4.frag @@ -0,0 +1,67 @@ +#version 100 + +// Copyright Alastair F. Donaldson and Hugues Evrard, Imperial College London, 2017 +// Defect found using GLFuzz - https://www.graphicsfuzz.com/ +// +// Gives compile error on: +// NVIDIA SHIELD Android TV +// Model number: P2571 +// GL_VERSION: OpenGL ES 3.2 NVIDIA 361.00 +// GL_VENDOR: NVIDIA Corporation +// GL_RENDERER: NVIDIA Tegra +// Android version: 7.0 + +#ifdef GL_ES +#ifdef GL_FRAGMENT_PRECISION_HIGH +precision highp float; +precision highp int; +#else +precision mediump float; +precision mediump int; +#endif +#endif + +uniform vec2 injectionSwitch; + +void main() +{ + int r; + int g; + int b; + int a; + r = 100 * int(injectionSwitch.y); + g = int(injectionSwitch.x) * int(injectionSwitch.y); + b = 2 * int(injectionSwitch.x); + a = g - int(injectionSwitch.x); + for( + int i = 0; + i < 10; + i ++ + ) + { + r --; + g ++; + b ++; + a ++; + for( + int j = 1; + j < 10; + j ++ + ) + { + a ++; + b ++; + g ++; + r --; + } + } + float fr; + float fg; + float fb; + float fa; + fr = float(r / 100); + fg = float(g / 100); + fb = float(b / 100); + fa = float(a / 100); + gl_FragColor = vec4(r, g, b, a) * ((false ? 1.0 : 1.0) * vec4(1.0)); +} diff --git a/external/graphicsfuzz/data/gles3/graphicsfuzz/vertexShader.glsl b/external/graphicsfuzz/data/gles3/graphicsfuzz/vertexShader.glsl new file mode 100644 index 0000000..3ba9e74 --- /dev/null +++ b/external/graphicsfuzz/data/gles3/graphicsfuzz/vertexShader.glsl @@ -0,0 +1,13 @@ +#version 100 + +// Copyright Alastair F. Donaldson and Hugues Evrard, Imperial College London, 2017 + +attribute vec2 coord2d; + +varying vec2 surfacePosition; + +void main(void) +{ + gl_Position = vec4(coord2d, 0.0, 1.0); + surfacePosition = coord2d; +} diff --git a/modules/gles3/CMakeLists.txt b/modules/gles3/CMakeLists.txt index fee4280..013486c 100644 --- a/modules/gles3/CMakeLists.txt +++ b/modules/gles3/CMakeLists.txt @@ -41,5 +41,6 @@ set(DEQP_GLES3_LIBS add_deqp_module(deqp-gles3 "${DEQP_GLES3_SRCS}" "${DEQP_GLES3_LIBS}" tes3TestPackageEntry.cpp) # Data directories -add_data_dir(deqp-gles3 ../../data/gles3/data gles3/data) -add_data_dir(deqp-gles3 ../../data/gles3/shaders gles3/shaders) +add_data_dir(deqp-gles3 ../../data/gles3/data gles3/data) +add_data_dir(deqp-gles3 ../../data/gles3/shaders gles3/shaders) +add_data_dir(deqp-gles3 ../../external/graphicsfuzz/data/gles3/graphicsfuzz gles3/graphicsfuzz) diff --git a/modules/gles3/functional/CMakeLists.txt b/modules/gles3/functional/CMakeLists.txt index 1adc70e..258e4ba 100644 --- a/modules/gles3/functional/CMakeLists.txt +++ b/modules/gles3/functional/CMakeLists.txt @@ -75,6 +75,8 @@ set(DEQP_GLES3_FUNCTIONAL_SRCS es3fShaderLoopTests.hpp es3fShaderMatrixTests.cpp es3fShaderMatrixTests.hpp + es3fShaderMetamorphicTests.cpp + es3fShaderMetamorphicTests.hpp es3fShaderOperatorTests.cpp es3fShaderOperatorTests.hpp es3fShaderPrecisionTests.cpp diff --git a/modules/gles3/functional/es3fFunctionalTests.cpp b/modules/gles3/functional/es3fFunctionalTests.cpp index 60607c0..96453b5 100644 --- a/modules/gles3/functional/es3fFunctionalTests.cpp +++ b/modules/gles3/functional/es3fFunctionalTests.cpp @@ -82,6 +82,7 @@ #include "es3fShaderInvarianceTests.hpp" #include "es3fShaderFragDataTests.hpp" #include "es3fBuiltinPrecisionTests.hpp" +#include "es3fShaderMetamorphicTests.hpp" // Texture tests #include "es3fTextureFormatTests.hpp" @@ -226,7 +227,7 @@ public: addChild(new ShaderInvarianceTests (m_context)); addChild(new ShaderFragDataTests (m_context)); addChild(new ShaderConstExprTests (m_context)); - + addChild(new ShaderMetamorphicTests (m_context)); addChild(new RandomShaderTests (m_context)); } }; diff --git a/modules/gles3/functional/es3fShaderMetamorphicTests.cpp b/modules/gles3/functional/es3fShaderMetamorphicTests.cpp new file mode 100644 index 0000000..7b5df9e --- /dev/null +++ b/modules/gles3/functional/es3fShaderMetamorphicTests.cpp @@ -0,0 +1,355 @@ +/*------------------------------------------------------------------------- + * drawElements Quality Program OpenGL ES 3.0 Module + * ------------------------------------------------- + * + * Copyright 2017 Hugues Evrard, Imperial College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + *//*! + * \file + * \brief Shader metamorphic tests. + *//*--------------------------------------------------------------------*/ + +#include "es3fShaderMetamorphicTests.hpp" +#include "glsShaderRenderCase.hpp" +#include "deUniquePtr.hpp" +#include "deFilePath.hpp" +#include "tcuTestContext.hpp" +#include "tcuTestLog.hpp" +#include "tcuRenderTarget.hpp" +#include "tcuImageCompare.hpp" +#include "tcuVectorUtil.hpp" +#include "tcuResource.hpp" +#include "gluPixelTransfer.hpp" +#include "gluDrawUtil.hpp" + +#include "glwFunctions.hpp" + +using std::vector; +using tcu::TestLog; + +namespace deqp +{ +namespace gles3 +{ +namespace Functional +{ + +static const int MAX_RENDER_WIDTH = 256; +static const int MAX_RENDER_HEIGHT = 256; + +typedef bool (*SanityCheckFunc)(const tcu::ConstPixelBufferAccess&); + +/*--------------------------------------------------------------------*//*! + * \brief ShaderMetamorphicVariant + * + * ShaderMetamorphicVariant aims at rendering a recipient shader and a + * variant shader, and compare whether the resulting images are the + * approximately the same. It also checks non-deterministic renderings, + * by rendering each fragment shader a couple of times. + *//*--------------------------------------------------------------------*/ +class ShaderMetamorphicVariant : public TestCase +{ +public: + ShaderMetamorphicVariant (Context& context, const char* name, const std::string& vertexFilename, const std::string& recipientFilename, const std::string& variantFilename, SanityCheckFunc sanityCheck); + ~ShaderMetamorphicVariant (void); + IterateResult iterate (void); + +private: + const std::string m_vertexFilename; + const std::string m_recipientFilename; + const std::string m_variantFilename; + SanityCheckFunc m_sanityCheck; + + std::string fileContents (const std::string& filename); + void render (const tcu::PixelBufferAccess& img, const std::string& vertexSrc, const std::string& fragmentSrc); + void checkNondet (const tcu::Surface& refImg, const std::string& vertexSrc, const std::string& fragmentSrc); +}; + +ShaderMetamorphicVariant::ShaderMetamorphicVariant (Context& context, const char* name, const std::string& vertexFilename, const std::string& recipientFilename, const std::string& variantFilename, SanityCheckFunc sanityCheck) + : TestCase (context, name, "Test a given variant") + , m_vertexFilename (vertexFilename) + , m_recipientFilename (recipientFilename) + , m_variantFilename (variantFilename) + , m_sanityCheck (sanityCheck) +{ +} + +ShaderMetamorphicVariant::~ShaderMetamorphicVariant (void) +{ +} + +std::string ShaderMetamorphicVariant::fileContents (const std::string& filename) +{ + de::UniquePtr resource (m_testCtx.getArchive().getResource(filename.c_str())); + int size = resource->getSize(); + std::vector data; + + data.resize(size + 1); + resource->read(&data[0], size); + data[size] = '\0'; + std::string contents = std::string((const char*)(&data[0])); + return contents; +} + +void ShaderMetamorphicVariant::render (const tcu::PixelBufferAccess& img, const std::string& vertexSrc, const std::string& fragmentSrc) +{ + TestLog& log = m_testCtx.getLog(); + const glw::Functions& gl = m_context.getRenderContext().getFunctions(); + + // Positions, shared between shaders + const float positions[] = + { + -1.0f, 1.0f, // top-left + -1.0f, -1.0f, // bottom-left + 1.0f, -1.0f, // bottom-right + 1.0f, 1.0f, // top-right + }; + + const deUint16 indices[] = + { + 0, 1, 2, // bottom-left triangle + 0, 3, 2, // top-right triangle + }; + + glu::VertexArrayBinding posBinding = glu::va::Float("coord2d", 2, 6, 0, &positions[0]); + + const glu::ShaderProgram program(m_context.getRenderContext(), glu::makeVtxFragSources(vertexSrc, fragmentSrc)); + log << program; + + if (!program.isOk()) + throw tcu::TestError("Compile failed"); + + // Set uniforms expected in GraphicsFuzz generated programs + gl.useProgram(program.getProgram()); + // Uniform: injectionSwitch + int uniformLoc = gl.getUniformLocation(program.getProgram(), "injectionSwitch"); + if (uniformLoc != -1) + gl.uniform2f(uniformLoc, 0.0f, 1.0f); + // Uniform: resolution + uniformLoc = gl.getUniformLocation(program.getProgram(), "resolution"); + if (uniformLoc != -1) + gl.uniform2f(uniformLoc, glw::GLfloat(img.getWidth()), glw::GLfloat(img.getHeight())); + // Uniform: mouse + uniformLoc = gl.getUniformLocation(program.getProgram(), "mouse"); + if (uniformLoc != -1) + gl.uniform2f(uniformLoc, 0.0f, 0.0f); + // Uniform: time + uniformLoc = gl.getUniformLocation(program.getProgram(), "time"); + if (uniformLoc != -1) + gl.uniform1f(uniformLoc, 0.0f); + + // Render two times to check nondeterministic renderings + glu::draw(m_context.getRenderContext(), program.getProgram(), 1, &posBinding, glu::pr::Triangles(DE_LENGTH_OF_ARRAY(indices), &indices[0])); + glu::readPixels(m_context.getRenderContext(), 0, 0, img); + GLU_EXPECT_NO_ERROR(gl.getError(), "Draw"); +} + +void ShaderMetamorphicVariant::checkNondet (const tcu::Surface& refImg, const std::string& vertexSrc, const std::string& fragmentSrc) +{ + TestLog& log = m_testCtx.getLog(); + tcu::Surface img = tcu::Surface(refImg.getWidth(), refImg.getHeight()); + + render(img.getAccess(), vertexSrc, fragmentSrc); + bool same = tcu::pixelThresholdCompare(log, "Result", "Image comparison result", img, refImg, tcu::RGBA(0,0,0,0), tcu::COMPARE_LOG_RESULT); + if (!same) + throw tcu::TestError("Nondeterministic rendering"); +} + +ShaderMetamorphicVariant::IterateResult ShaderMetamorphicVariant::iterate (void) +{ + TestLog& log = m_testCtx.getLog(); + const tcu::RGBA threshold = tcu::RGBA(1,1,1,1) + m_context.getRenderTarget().getPixelFormat().getColorThreshold(); + std::string vertexSrc = fileContents(m_vertexFilename); + std::string recipientSrc = fileContents(m_recipientFilename); + std::string variantSrc = fileContents(m_variantFilename); + const int width = deMin32(m_context.getRenderTarget().getWidth(), MAX_RENDER_WIDTH); + const int height = deMin32(m_context.getRenderTarget().getHeight(), MAX_RENDER_HEIGHT); + tcu::Surface recipientImg = tcu::Surface(width, height); + tcu::Surface variantImg = tcu::Surface(width, height); + + render(recipientImg.getAccess(), vertexSrc, recipientSrc); + render(variantImg.getAccess(), vertexSrc, variantSrc); + + checkNondet(recipientImg, vertexSrc, recipientSrc); + checkNondet(variantImg, vertexSrc, variantSrc); + + if (m_sanityCheck != DE_NULL) + { + bool isSane = m_sanityCheck(recipientImg.getAccess()); + if (!isSane) + throw tcu::TestError("Sanity check fails on recipient"); + } + + bool isOk = tcu::pixelThresholdCompare(log, "Result", "Image comparison result", recipientImg, variantImg, threshold, tcu::COMPARE_LOG_RESULT); + + m_testCtx.setTestResult(isOk ? QP_TEST_RESULT_PASS : QP_TEST_RESULT_FAIL, + isOk ? "Pass" : "Image comparison failed"); + + return STOP; +} + +/*--------------------------------------------------------------------*//*! + * \brief ShaderMetamorphicShaderset + * + * ShaderMetamorphicShaderset gathers a set of ShaderMetamorphicVariant + * for a similar recipient. + *//*--------------------------------------------------------------------*/ +class ShaderMetamorphicShaderset : public TestCaseGroup +{ +public: + ShaderMetamorphicShaderset (Context& context, const char* name, const std::string& vertexFilename, const std::string& recipientFilename, std::vector variantFilenames, SanityCheckFunc sanityCheck); + ~ShaderMetamorphicShaderset (void); + virtual void init (void); + +private: + const std::string m_vertexFilename; + const std::string m_recipientFilename; + std::vector m_variantFilenames; + SanityCheckFunc m_sanityCheck; + + ShaderMetamorphicShaderset (const ShaderMetamorphicShaderset&); // Not allowed! + ShaderMetamorphicShaderset& operator= (const ShaderMetamorphicShaderset&); // Not allowed! +}; + +ShaderMetamorphicShaderset::ShaderMetamorphicShaderset (Context& context, const char *name, const std::string& vertexFilename, const std::string& recipientFilename, std::vector variantFilenames, SanityCheckFunc sanityCheck) + : TestCaseGroup (context, name, "Metamorphic Shader Set") + , m_vertexFilename (vertexFilename) + , m_recipientFilename (recipientFilename) + , m_variantFilenames (variantFilenames) + , m_sanityCheck (sanityCheck) +{ +} + +ShaderMetamorphicShaderset::~ShaderMetamorphicShaderset (void) +{ +} + +void ShaderMetamorphicShaderset::init(void) +{ + for (size_t variantNdx = 0; variantNdx < m_variantFilenames.size(); variantNdx++) + { + std::string variantName = de::FilePath(m_variantFilenames[variantNdx]).getBaseName(); + // Remove extension + size_t pos = variantName.find_last_of("."); + variantName = variantName.substr(0, pos); + + addChild(new ShaderMetamorphicVariant(m_context, variantName.c_str(), m_vertexFilename, m_recipientFilename, m_variantFilenames[variantNdx], m_sanityCheck)); + } +} + +/*--------------------------------------------------------------------*//*! + * \brief SanityPixel + * + * A place holder to store info on reference pixel for sanity checking. + *//*--------------------------------------------------------------------*/ +class SanityPixel +{ +public: + float m_xRelative; + float m_yRelative; + tcu::Vec4 m_RGBA; + + SanityPixel (float xRelative, float yRelative, tcu::Vec4 RGBA); +}; + +SanityPixel::SanityPixel (float xRelative, float yRelative, tcu::Vec4 RGBA) + : m_xRelative (xRelative) + , m_yRelative (yRelative) + , m_RGBA (RGBA) +{ +} + +static bool sanityComparePixels (const tcu::ConstPixelBufferAccess& img, std::vector sanityPixels) +{ + const int depth = 0; + const tcu::Vec4 threshold = tcu::Vec4(0.01f, 0.01f, 0.01f, 0.01f); + + for (deUint32 i = 0; i < sanityPixels.size(); i++) + { + SanityPixel sanPix = sanityPixels[i]; + int x = (int)((float)img.getWidth() * sanPix.m_xRelative); + int y = (int)((float)img.getHeight() * sanPix.m_yRelative); + tcu::Vec4 RGBA = img.getPixel(x, y, depth); + tcu::Vec4 diff = abs(RGBA - sanPix.m_RGBA); + for (int j = 0; j < 4; j++) + if (diff[j] >= threshold[j]) + return false; + } + return true; +} + +static bool sanityCheck_synthetic (const tcu::ConstPixelBufferAccess& img) +{ + std::vector sanityPixels; + bool isOK; + + sanityPixels.push_back(SanityPixel(0.5f, 0.5f, tcu::Vec4(0.0f, 1.0f, 1.0f, 1.0f))); + + isOK = sanityComparePixels(img, sanityPixels); + return isOK; +} + +static bool sanityCheck_bubblesort_flag (const tcu::ConstPixelBufferAccess& img) +{ + std::vector sanityPixels; + bool isOK; + + sanityPixels.push_back(SanityPixel(0.25f, 0.25f, tcu::Vec4(0.1f, 0.6f, 1.0f, 1.0f))); + sanityPixels.push_back(SanityPixel(0.25f, 0.75f, tcu::Vec4(1.0f, 0.5f, 0.1f, 1.0f))); + sanityPixels.push_back(SanityPixel(0.75f, 0.25f, tcu::Vec4(0.6f, 1.0f, 0.1f, 1.0f))); + sanityPixels.push_back(SanityPixel(0.75f, 0.75f, tcu::Vec4(0.5f, 0.1f, 1.0f, 1.0f))); + + isOK = sanityComparePixels(img, sanityPixels); + return isOK; +} + +/*--------------------------------------------------------------------*//*! + * \brief ShaderMetamorphicTests + * + * ShaderMetamorphicTests regroups metamorphic shadersets. + *//*--------------------------------------------------------------------*/ +ShaderMetamorphicTests::ShaderMetamorphicTests (Context& context) +: TestCaseGroup(context, "metamorphic", "Shader Metamorphic Tests") +{ +} + +ShaderMetamorphicTests::~ShaderMetamorphicTests (void) +{ +} + +void ShaderMetamorphicTests::init (void) +{ + std::vector fragNames; + std::string vertexFilename = "graphicsfuzz/vertexShader.glsl"; + + // synthetic + fragNames.clear(); + fragNames.push_back("graphicsfuzz/synthetic/variant_1.frag"); + fragNames.push_back("graphicsfuzz/synthetic/variant_2.frag"); + fragNames.push_back("graphicsfuzz/synthetic/variant_3.frag"); + fragNames.push_back("graphicsfuzz/synthetic/variant_4.frag"); + addChild(new ShaderMetamorphicShaderset (m_context, "synthetic", vertexFilename, "graphicsfuzz/synthetic/recipient.frag", fragNames, sanityCheck_synthetic)); + + // bubblesort_flag + fragNames.clear(); + fragNames.push_back("graphicsfuzz/bubblesort_flag/variant_1.frag"); + fragNames.push_back("graphicsfuzz/bubblesort_flag/variant_2.frag"); + addChild(new ShaderMetamorphicShaderset (m_context, "bubblesort_flag", vertexFilename, "graphicsfuzz/bubblesort_flag/recipient.frag", fragNames, sanityCheck_bubblesort_flag)); + +} + +} // Functional +} // gles3 +} // deqp diff --git a/modules/gles3/functional/es3fShaderMetamorphicTests.hpp b/modules/gles3/functional/es3fShaderMetamorphicTests.hpp new file mode 100644 index 0000000..be89bbe --- /dev/null +++ b/modules/gles3/functional/es3fShaderMetamorphicTests.hpp @@ -0,0 +1,52 @@ +#ifndef _ES3FSHADERMETAMORPHICTESTS_HPP +#define _ES3FSHADERMETAMORPHICTESTS_HPP +/*------------------------------------------------------------------------- + * drawElements Quality Program OpenGL ES 3.0 Module + * ------------------------------------------------- + * + * Copyright 2017 Hugues Evrard, Imperial College London + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + *//*! + * \file + * \brief Shader metamorphic tests. + *//*--------------------------------------------------------------------*/ + +#include "tes3TestCase.hpp" + +namespace deqp +{ +namespace gles3 +{ +namespace Functional +{ + +class ShaderMetamorphicTests : public TestCaseGroup +{ +public: + ShaderMetamorphicTests (Context& context); + virtual ~ShaderMetamorphicTests (void); + + virtual void init (void); + +private: + ShaderMetamorphicTests (const ShaderMetamorphicTests&); // not allowed! + ShaderMetamorphicTests& operator= (const ShaderMetamorphicTests&); // not allowed! +}; + +} // Functional +} // gles3 +} // deqp + +#endif // _ES3FSHADERMETAMORPHICTESTS_HPP -- 2.7.4