Updates to Android test list generation

Fixes to reflect changes on the Android side.

Change-Id: I4eece1d987c35e27046061adc79f9ecde755a56a

diff --git a/android/cts/AndroidTest.xml b/android/cts/AndroidTest.xml
index 7464c3c..5d29fb7 100644 (file)
--- a/android/cts/AndroidTest.xml
+++ b/android/cts/AndroidTest.xml
@@ -24,6 +24,12 @@
        </target_preparer>
        <option name="test-suite-tag" value="cts"/>
        <option key="component" name="config-descriptor:metadata" value="deqp"/>
+       <option key="parameter" name="config-descriptor:metadata" value="not_instant_app"/>
+       <option key="parameter" name="config-descriptor:metadata" value="multi_abi"/>
+       <option key="parameter" name="config-descriptor:metadata" value="secondary_user"/>
+       <object class="com.android.tradefed.testtype.suite.module.TestFailureModuleController" type="module_controller">
+               <option name="screenshot-on-failure" value="false"/>
+       </object>
        <test class="com.drawelements.deqp.runner.DeqpTestRunner">
                <option name="deqp-package" value="dEQP-EGL"/>
                <option name="deqp-caselist-file" value="egl-master.txt"/>
@@ -34,6 +40,15 @@
                <option name="deqp-config-required" value="true"/>
        </test>
        <test class="com.drawelements.deqp.runner.DeqpTestRunner">
+               <option name="deqp-package" value="dEQP-EGL"/>
+               <option name="deqp-caselist-file" value="egl-master-risky.txt"/>
+               <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
+               <option name="deqp-surface-type" value="window"/>
+               <option name="deqp-screen-rotation" value="unspecified"/>
+               <option name="runtime-hint" value="2m"/>
+               <option name="deqp-config-required" value="true"/>
+       </test>
+       <test class="com.drawelements.deqp.runner.DeqpTestRunner">
                <option name="deqp-package" value="dEQP-GLES2"/>
                <option name="deqp-caselist-file" value="gles2-master.txt"/>
                <option name="deqp-gl-config-name" value="rgba8888d24s8ms0"/>
@@ -182,11 +197,6 @@
        </test>
        <test class="com.drawelements.deqp.runner.DeqpTestRunner">
                <option name="deqp-package" value="dEQP-VK"/>
-               <option name="deqp-caselist-file" value="vk-master.txt"/>
-               <option name="runtime-hint" value="2h39m"/>
-       </test>
-       <test class="com.drawelements.deqp.runner.DeqpTestRunner">
-               <option name="deqp-package" value="dEQP-VK"/>
                <option name="deqp-caselist-file" value="vk-master-2019-03-01.txt"/>
                <option name="runtime-hint" value="2h29m"/>
        </test>

diff --git a/android/cts/master/egl-master-risky.txt b/android/cts/master/egl-master-risky.txt
new file mode 100644 (file)
index 0000000..8b13789
--- /dev/null
+++ b/android/cts/master/egl-master-risky.txt
@@ -0,0 +1 @@
+

diff --git a/android/cts/master/mustpass.xml b/android/cts/master/mustpass.xml
index d7673da..66581be 100644 (file)
--- a/android/cts/master/mustpass.xml
+++ b/android/cts/master/mustpass.xml
@@ -20,6 +20,7 @@
      -->
        <TestPackage name="dEQP-EGL">
                <Configuration caseListFile="egl-master.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="master"/>
+               <Configuration caseListFile="egl-master-risky.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="master-risky"/>
        </TestPackage>
        <TestPackage name="dEQP-GLES2">
                <Configuration caseListFile="gles2-master.txt" commandLine="--deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-type=window --deqp-watchdog=enable" name="master"/>

diff --git a/android/cts/master/src/egl-temp-excluded.txt b/android/cts/master/src/egl-temp-excluded.txt
new file mode 100644 (file)
index 0000000..2974117
--- /dev/null
+++ b/android/cts/master/src/egl-temp-excluded.txt
@@ -0,0 +1 @@
+# Tests to be temporarily skipped for Android CI, but still enforced in CTS.

diff --git a/external/vulkancts/mustpass/AndroidTest.xml b/external/vulkancts/mustpass/AndroidTest.xml
index c125fe5..63fbf68 100644 (file)
--- a/external/vulkancts/mustpass/AndroidTest.xml
+++ b/external/vulkancts/mustpass/AndroidTest.xml
@@ -22,6 +22,12 @@
        </target_preparer>
        <option name="test-suite-tag" value="cts"/>
        <option key="component" name="config-descriptor:metadata" value="deqp"/>
+       <option key="parameter" name="config-descriptor:metadata" value="not_instant_app"/>
+       <option key="parameter" name="config-descriptor:metadata" value="multi_abi"/>
+       <option key="parameter" name="config-descriptor:metadata" value="secondary_user"/>
+       <object class="com.android.tradefed.testtype.suite.module.TestFailureModuleController" type="module_controller">
+               <option name="screenshot-on-failure" value="false"/>
+       </object>
        <test class="com.drawelements.deqp.runner.DeqpTestRunner">
                <option name="deqp-package" value="dEQP-VK"/>
                <option name="deqp-caselist-file" value="vk-default.txt"/>

diff --git a/scripts/build_android_mustpass.py b/scripts/build_android_mustpass.py
index 5abe4a5..d0d4f8e 100644 (file)
--- a/scripts/build_android_mustpass.py
+++ b/scripts/build_android_mustpass.py
@@ -59,7 +59,8 @@ MASTER_EGL_COMMON_FILTERS             = [include("egl-master.txt"),
                                                                   exclude("egl-test-issues.txt"),
                                                                   exclude("egl-internal-api-tests.txt"),
                                                                   exclude("egl-manual-robustness.txt"),
-                                                                  exclude("egl-driver-issues.txt")]
+                                                                  exclude("egl-driver-issues.txt"),
+                                                                  exclude("egl-temp-excluded.txt")]
 MASTER_EGL_PKG                                 = Package(module = EGL_MODULE, configurations = [
                # Master
                Configuration(name                      = "master",
@@ -69,6 +70,14 @@ MASTER_EGL_PKG                                       = Package(module = EGL_MODULE, configurations = [
                                          required              = True,
                                          filters               = MASTER_EGL_COMMON_FILTERS,
                                      runtime           = "23m"),
+               # Risky subset
+               Configuration(name                      = "master-risky",
+                                         glconfig              = "rgba8888d24s8ms0",
+                                         rotation              = "unspecified",
+                                         surfacetype   = "window",
+                                         required              = True,
+                                         filters               = [include("egl-temp-excluded.txt")],
+                                     runtime           = "2m"),
        ])
 
 MASTER_GLES2_COMMON_FILTERS            = [
@@ -243,7 +252,8 @@ MASTER_VULKAN_FILTERS                       = [
 MASTER_VULKAN_PKG                              = Package(module = VULKAN_MODULE, configurations = [
                Configuration(name                      = "master",
                                          filters               = MASTER_VULKAN_FILTERS,
-                                         runtime               = "2h39m"),
+                                         runtime               = "2h39m",
+                                         runByDefault          = False),
                Configuration(name                      = "master-2019-03-01",
                                          filters               = [include("vk-master-2019-03-01.txt")],
                                          runtime               = "2h29m"),

diff --git a/scripts/mustpass.py b/scripts/mustpass.py
index 12949db..1c17273 100644 (file)
--- a/scripts/mustpass.py
+++ b/scripts/mustpass.py
@@ -43,7 +43,7 @@ class Project:
                self.copyright  = copyright
 
 class Configuration:
-       def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None):
+       def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None, runByDefault = True):
                self.name                               = name
                self.glconfig                   = glconfig
                self.rotation                   = rotation
@@ -51,6 +51,7 @@ class Configuration:
                self.required                   = required
                self.filters                    = filters
                self.expectedRuntime    = runtime
+               self.runByDefault               = runByDefault
 
 class Package:
        def __init__ (self, module, configurations):
@@ -314,9 +315,19 @@ def genAndroidTestXml (mustpass):
        # add in metadata option for component name
        ElementTree.SubElement(configElement, "option", name="test-suite-tag", value="cts")
        ElementTree.SubElement(configElement, "option", name="config-descriptor:metadata", key="component", value="deqp")
+       ElementTree.SubElement(configElement, "option", name="config-descriptor:metadata", key="parameter", value="not_instant_app")
+       ElementTree.SubElement(configElement, "option", name="config-descriptor:metadata", key="parameter", value="multi_abi")
+       ElementTree.SubElement(configElement, "option", name="config-descriptor:metadata", key="parameter", value="secondary_user")
+       controllerElement = ElementTree.SubElement(configElement, "object")
+       controllerElement.set("type", "module_controller")
+       controllerElement.set("class", "com.android.tradefed.testtype.suite.module.TestFailureModuleController")
+       addOptionElement(controllerElement, "screenshot-on-failure", "false")
 
        for package in mustpass.packages:
                for config in package.configurations:
+                       if not config.runByDefault:
+                               continue
+
                        testElement = ElementTree.SubElement(configElement, "test")
                        testElement.set("class", RUNNER_CLASS)
                        addOptionElement(testElement, "deqp-package", package.module.name)
@@ -406,4 +417,4 @@ def parseCmdLineArgs ():
 
 def parseBuildConfigFromCmdLineArgs ():
        args = parseCmdLineArgs()
-       return getBuildConfig(args.buildDir, args.targetName, args.buildType)
\ No newline at end of file
+       return getBuildConfig(args.buildDir, args.targetName, args.buildType)