Add support for waiver mechanism to cts-runner

Affects: cts-runner

Change-Id: I147cbb49114386b939a6563a1b4b96f620310736
Components: Framework

diff --git a/external/openglcts/README.md b/external/openglcts/README.md
index b71b926..9c0f360 100644 (file)
--- a/external/openglcts/README.md
+++ b/external/openglcts/README.md
@@ -439,6 +439,7 @@ Full list of parameters for the `cts-runner` binary:
 --type=[esN[M]|glNM] Conformance test run type. Choose from
                                         ES: es2, es3, es31, es32
                                         GL: gl30, gl31, gl32, gl33, gl40, gl41, gl42, gl43, gl44, gl45, gl46
+--waivers=[path]     Path to xml file containing waived tests
 --logdir=[path]      Destination directory for log files
 --summary            Print summary without running the tests
 --verbose            Print out and log more information
@@ -462,6 +463,8 @@ See Section [Test Logs](#test-logs) for instructions on how to view the images.
 
 To direct logs to a directory, add `--logdir=[path]` parameter.
 
+To specify waived tests, add `--waivers=[path]` parameter.
+
 **NOTE**: Due to the lack of support for run-time selection of API context in the
 Khronos Confidential CTS, a conformance run may fail if it is executed for an API
 version that doesn't match the `GLCTS_GTF_TARGET` value used during the build step.
@@ -494,6 +497,10 @@ string extra. See the following example:
 
        am start -n org.khronos.gl_cts/org.khronos.cts.ES32Activity -e logdir "/sdcard/logs" -e summary "true"
 
+Waivers can be specified by supplying a `waivers` string extra. See the following example:
+
+       am start -n org.khronos.gl_cts/org.khronos.cts.ES32Activity -e logdir "/sdcard/logs" -e waivers "/sdcard/waivers.xml"
+
 **NOTE**: Supplying a `summary` = `"true"` string extra will result in the `cts-run-summary.xml` file
 being written out but no tests will be executed.
 
@@ -656,6 +663,9 @@ Full list of parameters for the `glcts` binary:
 
   --deqp-egl-config-name=<value>
     Legacy name for --deqp-gl-config-name
+
+  --deqp-waiver-file=<value>
+    Path to xml file containing waived tests
 ```
 
 ### Understanding the Results

diff --git a/external/openglcts/data/mustpass/gles/khronos_mustpass/master/waivers.xml b/external/openglcts/data/mustpass/waivers/waivers.xml
similarity index 95%
rename from external/openglcts/data/mustpass/gles/khronos_mustpass/master/waivers.xml
rename to external/openglcts/data/mustpass/waivers/waivers.xml
index 460b067..90f4d5d 100644 (file)
--- a/external/openglcts/data/mustpass/gles/khronos_mustpass/master/waivers.xml
+++ b/external/openglcts/data/mustpass/waivers/waivers.xml
@@ -54,7 +54,7 @@
 
        <waiver vendor="ATI Technologies Inc." url="https://gitlab.khronos.org/Tracker/vk-gl-cts/issues/1141 , https://gitlab.khronos.org/Tracker/vk-gl-cts/issues/1190">
                <description></description>
-               <!--TODO: plase let me know for which renderers this waiver should be defined -->
+               <!--TODO: this waiver is not applied - please specify for which renderers it should be defined -->
                <renderer_list>
                        <r>???</r>
                </renderer_list>

diff --git a/external/openglcts/modules/runner/glcAndroidTestActivity.cpp b/external/openglcts/modules/runner/glcAndroidTestActivity.cpp
index b324bde..9896d09 100644 (file)
--- a/external/openglcts/modules/runner/glcAndroidTestActivity.cpp
+++ b/external/openglcts/modules/runner/glcAndroidTestActivity.cpp
@@ -42,6 +42,11 @@ using tcu::Android::NativeActivity;
 
 static const char* DEFAULT_LOG_PATH = "/sdcard";
 
+static std::string getWaiverPath(ANativeActivity* activity)
+{
+       return tcu::Android::getIntentStringExtra(activity, "waivers");
+}
+
 static std::string getLogPath(ANativeActivity* activity)
 {
        std::string path = tcu::Android::getIntentStringExtra(activity, "logdir");
@@ -58,12 +63,12 @@ static deUint32 getFlags(ANativeActivity* activity)
        return flags;
 }
 
-TestThread::TestThread(NativeActivity& activity, tcu::Android::AssetArchive& archive, const std::string& logPath,
-                                          glu::ApiType runType, deUint32 runFlags)
+TestThread::TestThread(NativeActivity& activity, tcu::Android::AssetArchive& archive, const std::string& waiverPath,
+                                          const std::string& logPath, glu::ApiType runType, deUint32 runFlags)
        : RenderThread(activity)
        , m_platform(activity)
        , m_archive(archive)
-       , m_app(m_platform, m_archive, logPath.c_str(), runType, runFlags)
+       , m_app(m_platform, m_archive, waiverPath.c_str(), logPath.c_str(), runType, runFlags)
        , m_finished(false)
 {
 }
@@ -108,7 +113,7 @@ TestActivity::TestActivity(ANativeActivity* activity, glu::ApiType runType)
        : RenderActivity(activity)
        , m_archive(activity->assetManager)
        , m_cmdLine(tcu::Android::getIntentStringExtra(activity, "cmdLine"))
-       , m_testThread(*this, m_archive, getLogPath(activity), runType, getFlags(activity))
+       , m_testThread(*this, m_archive, getWaiverPath(activity), getLogPath(activity), runType, getFlags(activity))
        , m_started(false)
 {
        // Set initial orientation.

diff --git a/external/openglcts/modules/runner/glcAndroidTestActivity.hpp b/external/openglcts/modules/runner/glcAndroidTestActivity.hpp
index 1330442..495150a 100644 (file)
--- a/external/openglcts/modules/runner/glcAndroidTestActivity.hpp
+++ b/external/openglcts/modules/runner/glcAndroidTestActivity.hpp
@@ -42,7 +42,7 @@ class TestThread : public tcu::Android::RenderThread
 {
 public:
        TestThread(tcu::Android::NativeActivity& nativeActivity, tcu::Android::AssetArchive& archive,
-                          const std::string& logPath, glu::ApiType runType, deUint32 runFlags);
+                          const std::string& waiverPath, const std::string& logPath, glu::ApiType runType, deUint32 runFlags);
        ~TestThread(void);
 
        void run(void);

diff --git a/external/openglcts/modules/runner/glcTestRunner.cpp b/external/openglcts/modules/runner/glcTestRunner.cpp
index a151312..c955d57 100644 (file)
--- a/external/openglcts/modules/runner/glcTestRunner.cpp
+++ b/external/openglcts/modules/runner/glcTestRunner.cpp
@@ -665,10 +665,11 @@ static void writeRunSummary(const TestRunSummary& summary, const char* filename)
 
 #undef XML_CHECK
 
-TestRunner::TestRunner(tcu::Platform& platform, tcu::Archive& archive, const char* logDirPath, glu::ApiType type,
-                                          deUint32 flags)
+TestRunner::TestRunner(tcu::Platform& platform, tcu::Archive& archive, const char* waiverPath,
+                                          const char* logDirPath, glu::ApiType type, deUint32 flags)
        : m_platform(platform)
        , m_archive(archive)
+       , m_waiverPath(waiverPath)
        , m_logDirPath(logDirPath)
        , m_type(type)
        , m_flags(flags)
@@ -797,6 +798,9 @@ void TestRunner::initSession(const TestRunParams& runParams)
        if (!(m_flags & VERBOSE_SHADERS))
                args.push_back("--deqp-log-shader-sources=disable");
 
+       if (!m_waiverPath.empty())
+               args.push_back(string("--deqp-waiver-file=") + m_waiverPath);
+
        std::ostringstream                        ostr;
        std::ostream_iterator<string> out_it(ostr, ", ");
        std::copy(args.begin(), args.end(), out_it);

diff --git a/external/openglcts/modules/runner/glcTestRunner.hpp b/external/openglcts/modules/runner/glcTestRunner.hpp
index 962d334..f584f89 100644 (file)
--- a/external/openglcts/modules/runner/glcTestRunner.hpp
+++ b/external/openglcts/modules/runner/glcTestRunner.hpp
@@ -97,8 +97,8 @@ public:
                PRINT_SUMMARY = (1 << 3)
        };
 
-       TestRunner(tcu::Platform& platform, tcu::Archive& archive, const char* logDirPath, glu::ApiType type,
-                          deUint32 flags);
+       TestRunner(tcu::Platform& platform, tcu::Archive& archive, const char* waiverPath, const char* logDirPath,
+                          glu::ApiType type, deUint32 flags);
        ~TestRunner(void);
 
        bool iterate(void);
@@ -129,7 +129,8 @@ private:
 
        tcu::Platform& m_platform;
        tcu::Archive&  m_archive;
-       std::string     m_logDirPath;
+       std::string    m_waiverPath;
+       std::string    m_logDirPath;
        glu::ApiType   m_type;
        deUint32           m_flags;
 

diff --git a/external/openglcts/modules/runner/glcTestRunnerMain.cpp b/external/openglcts/modules/runner/glcTestRunnerMain.cpp
index 44c5c85..2e5f009 100644 (file)
--- a/external/openglcts/modules/runner/glcTestRunnerMain.cpp
+++ b/external/openglcts/modules/runner/glcTestRunnerMain.cpp
@@ -40,6 +40,7 @@ struct CommandLine
        }
 
        glu::ApiType runType;
+       std::string  waiverPath;
        std::string  dstLogDir;
        deUint32         flags;
 };
@@ -80,6 +81,11 @@ static bool parseCommandLine(CommandLine& cmdLine, int argc, const char* const*
                        if (ndx >= DE_LENGTH_OF_ARRAY(runTypes))
                                return false;
                }
+               else if (deStringBeginsWith(arg, "--waivers="))
+               {
+                       const char* value = arg + 10;
+                       cmdLine.waiverPath = value;
+               }
                else if (deStringBeginsWith(arg, "--logdir="))
                {
                        const char* value = arg + 9;
@@ -104,6 +110,7 @@ static void printHelp(const char* binName)
        printf("  --type=[esN[M]|glNM] Conformance test run type. Choose from\n");
        printf("                       ES: es2, es3, es31, es32\n");
        printf("                       GL: gl30, gl31, gl32, gl33, gl40, gl41, gl42, gl43, gl44, gl45, gl46\n");
+       printf("  --waivers=[path]     Path to xml file containing waived tests\n");
        printf("  --logdir=[path]      Destination directory for log files\n");
        printf("  --summary            Print summary without running the tests\n");
        printf("  --verbose            Print out and log more information\n");
@@ -124,8 +131,8 @@ int main(int argc, char** argv)
        {
                de::UniquePtr<tcu::Platform> platform(createPlatform());
                tcu::DirArchive                          archive(".");
-               glcts::TestRunner runner(static_cast<tcu::Platform&>(*platform.get()), archive, cmdLine.dstLogDir.c_str(),
-                                                                cmdLine.runType, cmdLine.flags);
+               glcts::TestRunner runner(static_cast<tcu::Platform&>(*platform.get()), archive, cmdLine.waiverPath.c_str(),
+                                                                cmdLine.dstLogDir.c_str(), cmdLine.runType, cmdLine.flags);
 
                for (;;)
                {