CP: Require 'master' config to be supported in Android CTS

All Android devices should support rgba8888d24s8 for compatiblity and
test coverage.

Bug: 30938718

(cherry picked from commit e232a6e83c482b219ee1e7678c8ccf91c64b3086)

Change-Id: Ia924a13e5e62b3b8e51981b200314b911aa57bc4

diff --git a/android/cts/AndroidTest.xml b/android/cts/AndroidTest.xml
index 75ee8f1..0484be6 100644 (file)
--- a/android/cts/AndroidTest.xml
+++ b/android/cts/AndroidTest.xml
@@ -25,6 +25,7 @@
                <option name="deqp-surface-type" value="window"/>
                <option name="deqp-screen-rotation" value="unspecified"/>
                <option name="runtime-hint" value="24m"/>
+               <option name="deqp-config-required" value="true"/>
        </test>
        <test class="com.drawelements.deqp.runner.DeqpTestRunner">
                <option name="deqp-package" value="dEQP-GLES2"/>
@@ -33,6 +34,7 @@
                <option name="deqp-surface-type" value="window"/>
                <option name="deqp-screen-rotation" value="unspecified"/>
                <option name="runtime-hint" value="40m"/>
+               <option name="deqp-config-required" value="true"/>
        </test>
        <test class="com.drawelements.deqp.runner.DeqpTestRunner">
                <option name="deqp-package" value="dEQP-GLES3"/>
@@ -41,6 +43,7 @@
                <option name="deqp-surface-type" value="window"/>
                <option name="deqp-screen-rotation" value="unspecified"/>
                <option name="runtime-hint" value="1h15m"/>
+               <option name="deqp-config-required" value="true"/>
        </test>
        <test class="com.drawelements.deqp.runner.DeqpTestRunner">
                <option name="deqp-package" value="dEQP-GLES3"/>
@@ -97,6 +100,7 @@
                <option name="deqp-surface-type" value="window"/>
                <option name="deqp-screen-rotation" value="unspecified"/>
                <option name="runtime-hint" value="7h30m"/>
+               <option name="deqp-config-required" value="true"/>
        </test>
        <test class="com.drawelements.deqp.runner.DeqpTestRunner">
                <option name="deqp-package" value="dEQP-GLES31"/>

diff --git a/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java b/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java
index 280d76d..ca8de1c 100644 (file)
--- a/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java
+++ b/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java
@@ -115,6 +115,10 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
             description="Surface type ('window', 'pbuffer', 'fbo'). Defaults to 'window'",
             importance=Option.Importance.NEVER)
     private String mSurfaceType = "window";
+    @Option(name="deqp-config-required",
+            description="Is current config required if API is supported? Defaults to false.",
+            importance=Option.Importance.NEVER)
+    private boolean mConfigRequired = false;
     @Option(name = "include-filter",
             description="Test include filter. '*' is zero or more letters. '.' has no special meaning.")
     private List<String> mIncludeFilters = new ArrayList<>();
@@ -254,19 +258,21 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
         private final String mGlConfig;
         private final String mRotation;
         private final String mSurfaceType;
+        private final boolean mRequired;
 
-        public BatchRunConfiguration(String glConfig, String rotation, String surfaceType) {
+        public BatchRunConfiguration(String glConfig, String rotation, String surfaceType, boolean required) {
             mGlConfig = glConfig;
             mRotation = rotation;
             mSurfaceType = surfaceType;
+            mRequired = required;
         }
 
         /**
          * Get string that uniquely identifies this config
          */
         public String getId() {
-            return String.format("{glformat=%s,rotation=%s,surfacetype=%s}",
-                    mGlConfig, mRotation, mSurfaceType);
+            return String.format("{glformat=%s,rotation=%s,surfacetype=%s,required=%b}",
+                    mGlConfig, mRotation, mSurfaceType, mRequired);
         }
 
         /**
@@ -290,6 +296,13 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
             return mSurfaceType;
         }
 
+        /**
+         * Is this configuration mandatory to support, if target API is supported?
+         */
+        public boolean isRequired() {
+            return mRequired;
+        }
+
         @Override
         public boolean equals(Object other) {
             if (other == null) {
@@ -1073,7 +1086,7 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
     }
 
     private static Map<TestIdentifier, Set<BatchRunConfiguration>> generateTestInstances(
-            Reader testlist, String configName, String screenRotation, String surfaceType) throws FileNotFoundException {
+            Reader testlist, String configName, String screenRotation, String surfaceType, boolean required) throws FileNotFoundException {
         // Note: This is specifically a LinkedHashMap to guarantee that tests are iterated
         // in the insertion order.
         final Map<TestIdentifier, Set<BatchRunConfiguration>> instances = new LinkedHashMap<>();
@@ -1083,7 +1096,7 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
             while ((testName = testlistReader.readLine()) != null) {
                 // Test name -> testId -> only one config -> done.
                 final Set<BatchRunConfiguration> testInstanceSet = new LinkedHashSet<>();
-                BatchRunConfiguration config = new BatchRunConfiguration(configName, screenRotation, surfaceType);
+                BatchRunConfiguration config = new BatchRunConfiguration(configName, screenRotation, surfaceType, required);
                 testInstanceSet.add(config);
                 TestIdentifier test = pathToIdentifier(testName);
                 instances.put(test, testInstanceSet);
@@ -1329,7 +1342,11 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
         if (isSupportedRunConfiguration(batch.config)) {
             executeTestRunBatch(batch);
         } else {
-            fakePassTestRunBatch(batch);
+            if (batch.config.isRequired()) {
+                fakeFailTestRunBatch(batch);
+            } else {
+                fakePassTestRunBatch(batch);
+            }
         }
     }
 
@@ -1633,13 +1650,24 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
      */
     private void fakePassTestRunBatch(TestBatch batch) {
         for (TestIdentifier test : batch.tests) {
-            CLog.d("Skipping test '%s' invocation in config '%s'", test.toString(),
+            CLog.d("Marking '%s' invocation in config '%s' as passed without running", test.toString(),
                     batch.config.getId());
             mInstanceListerner.skipTest(test);
         }
     }
 
     /**
+     * Fail given batch tests without running it
+     */
+    private void fakeFailTestRunBatch(TestBatch batch) {
+        for (TestIdentifier test : batch.tests) {
+            CLog.d("Marking '%s' invocation in config '%s' as failed without running", test.toString(),
+                    batch.config.getId());
+            mInstanceListerner.abortTest(test, "Required config not supported");
+        }
+    }
+
+    /**
      * Pass all remaining tests without running them
      */
     private void fakePassTests(ITestInvocationListener listener) {
@@ -1960,7 +1988,7 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
                 }
                 reader = new FileReader(testlist);
             }
-            mTestInstances = generateTestInstances(reader, mConfigName, mScreenRotation, mSurfaceType);
+            mTestInstances = generateTestInstances(reader, mConfigName, mScreenRotation, mSurfaceType, mConfigRequired);
             mCaselistReader = null;
             reader.close();
         }
@@ -2083,6 +2111,7 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
         destination.mCaselistFile = source.mCaselistFile;
         destination.mScreenRotation = source.mScreenRotation;
         destination.mSurfaceType = source.mSurfaceType;
+        destination.mConfigRequired = source.mConfigRequired;
         destination.mIncludeFilters = new ArrayList<>(source.mIncludeFilters);
         destination.mExcludeFilters = new ArrayList<>(source.mExcludeFilters);
         destination.mAbi = source.mAbi;

diff --git a/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java b/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java
index 249fcf9..7718d22 100644 (file)
--- a/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java
+++ b/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java
@@ -407,7 +407,7 @@ public class DeqpTestRunnerTest extends TestCase {
 
         if (!pass) {
             mockListener.testFailed(testId,
-                    "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window} ===\n"
+                    "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n"
                     + resultCode + ": Detail" + resultCode);
 
             EasyMock.expectLastCall().once();
@@ -931,7 +931,7 @@ public class DeqpTestRunnerTest extends TestCase {
             EasyMock.expectLastCall().once();
 
             mockListener.testFailed(EasyMock.eq(testIds[i]),
-                    EasyMock.eq("=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window} ===\n"
+                    EasyMock.eq("=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n"
                     + "Abort: Test cannot be executed"));
             EasyMock.expectLastCall().once();
 

diff --git a/scripts/build_android_mustpass.py b/scripts/build_android_mustpass.py
index a20da2c..4139e10 100644 (file)
--- a/scripts/build_android_mustpass.py
+++ b/scripts/build_android_mustpass.py
@@ -208,6 +208,7 @@ MASTER_EGL_PKG                                      = Package(module = EGL_MODULE, configurations = [
                                          glconfig              = "rgba8888d24s8ms0",
                                          rotation              = "unspecified",
                                          surfacetype   = "window",
+                                         required              = True,
                                          filters               = MASTER_EGL_COMMON_FILTERS,
                                      runtime           = "24m"),
        ])
@@ -223,6 +224,7 @@ MASTER_GLES2_PKG                            = Package(module = GLES2_MODULE, configurations = [
                                          glconfig              = "rgba8888d24s8ms0",
                                          rotation              = "unspecified",
                                          surfacetype   = "window",
+                                         required              = True,
                                          filters               = MASTER_GLES2_COMMON_FILTERS,
                                          runtime               = "40m"),
        ])
@@ -240,6 +242,7 @@ MASTER_GLES3_PKG                            = Package(module = GLES3_MODULE, configurations = [
                                          glconfig              = "rgba8888d24s8ms0",
                                          rotation              = "unspecified",
                                          surfacetype   = "window",
+                                         required              = True,
                                          filters               = MASTER_GLES3_COMMON_FILTERS,
                                          runtime               = "1h15m"),
                # Rotations
@@ -300,6 +303,7 @@ MASTER_GLES31_PKG                           = Package(module = GLES31_MODULE, configurations = [
                                          glconfig              = "rgba8888d24s8ms0",
                                          rotation              = "unspecified",
                                          surfacetype   = "window",
+                                         required              = True,
                                          filters               = MASTER_GLES31_COMMON_FILTERS,
                                          runtime               = "7h30m"),
 

diff --git a/scripts/mustpass.py b/scripts/mustpass.py
index 0a5f7dd..755faa0 100644 (file)
--- a/scripts/mustpass.py
+++ b/scripts/mustpass.py
@@ -42,13 +42,14 @@ class Project:
                self.copyright  = copyright
 
 class Configuration:
-       def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, runtime = None):
-               self.name                        = name
-               self.glconfig            = glconfig
-               self.rotation            = rotation
-               self.surfacetype         = surfacetype
-               self.filters             = filters
-               self.expectedRuntime = runtime
+       def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None):
+               self.name                               = name
+               self.glconfig                   = glconfig
+               self.rotation                   = rotation
+               self.surfacetype                = surfacetype
+               self.required                   = required
+               self.filters                    = filters
+               self.expectedRuntime    = runtime
 
 class Package:
        def __init__ (self, module, configurations):
@@ -387,6 +388,9 @@ def genAndroidTestXml (mustpass):
                        if config.expectedRuntime != None:
                                addOptionElement(testElement, "runtime-hint", config.expectedRuntime)
 
+                       if config.required:
+                               addOptionElement(testElement, "deqp-config-required", "true")
+
        insertXMLHeaders(mustpass, configElement)
 
        return configElement