1 /*-------------------------------------------------------------------------
2 * OpenGL Conformance Test Suite
3 * -----------------------------
5 * Copyright (c) 2016 Google Inc.
6 * Copyright (c) 2016 The Khronos Group Inc.
8 * Licensed under the Apache License, Version 2.0 (the "License");
9 * you may not use this file except in compliance with the License.
10 * You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
23 */ /*-------------------------------------------------------------------*/
25 #include "glcTestRunner.hpp"
26 #include "deFilePath.hpp"
27 #include "deStringUtil.hpp"
28 #include "deUniquePtr.hpp"
29 #include "glcConfigList.hpp"
30 #include "qpXmlWriter.h"
32 #include "tcuCommandLine.hpp"
33 #include "tcuTestLog.hpp"
34 #include "tcuTestSessionExecutor.hpp"
49 RunSession(tcu::Platform& platform, tcu::Archive& archive, const int numArgs, const char* const* args)
50 : m_cmdLine(numArgs, args)
51 , m_log(m_cmdLine.getLogFileName(), m_cmdLine.getLogFlags())
52 , m_app(platform, archive, m_log, m_cmdLine)
56 inline bool iterate(void)
58 return m_app.iterate();
61 inline const tcu::TestRunStatus& getResult(void) const
63 return m_app.getResult();
67 tcu::CommandLine m_cmdLine;
72 static void appendConfigArgs(const Config& config, std::vector<std::string>& args, const char* fboConfig)
74 if (fboConfig != NULL)
76 args.push_back(string("--deqp-gl-config-name=") + fboConfig);
77 args.push_back("--deqp-surface-type=fbo");
80 if (config.type != CONFIGTYPE_DEFAULT)
82 // \todo [2013-05-06 pyry] Test all surface types for some configs?
83 if (fboConfig == NULL)
85 if (config.surfaceTypes & SURFACETYPE_WINDOW)
86 args.push_back("--deqp-surface-type=window");
87 else if (config.surfaceTypes & SURFACETYPE_PBUFFER)
88 args.push_back("--deqp-surface-type=pbuffer");
89 else if (config.surfaceTypes & SURFACETYPE_PIXMAP)
90 args.push_back("--deqp-surface-type=pixmap");
93 args.push_back(string("--deqp-gl-config-id=") + de::toString(config.id));
95 if (config.type == CONFIGTYPE_EGL)
96 args.push_back("--deqp-gl-context-type=egl");
97 else if (config.type == CONFIGTYPE_WGL)
98 args.push_back("--deqp-gl-context-type=wgl");
102 typedef struct configInfo
113 static configInfo parseConfigBitsFromName(const char* configName)
124 { "rgba8888", 8, 8, 8, 8 }, { "rgb565", 5, 6, 5, 0 },
126 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(colorCfgs); ndx++)
128 if (!strncmp(configName, colorCfgs[ndx].name, strlen(colorCfgs[ndx].name)))
130 cfgInfo.redBits = colorCfgs[ndx].redBits;
131 cfgInfo.greenBits = colorCfgs[ndx].greenBits;
132 cfgInfo.blueBits = colorCfgs[ndx].blueBits;
133 cfgInfo.alphaBits = colorCfgs[ndx].alphaBits;
135 configName += strlen(colorCfgs[ndx].name);
145 { "d0", 0 }, { "d24", 24 },
147 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(depthCfgs); ndx++)
149 if (!strncmp(configName, depthCfgs[ndx].name, strlen(depthCfgs[ndx].name)))
151 cfgInfo.depthBits = depthCfgs[ndx].depthBits;
153 configName += strlen(depthCfgs[ndx].name);
163 { "s0", 0 }, { "s8", 8 },
165 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(stencilCfgs); ndx++)
167 if (!strncmp(configName, stencilCfgs[ndx].name, strlen(stencilCfgs[ndx].name)))
169 cfgInfo.stencilBits = stencilCfgs[ndx].stencilBits;
171 configName += strlen(stencilCfgs[ndx].name);
180 } multiSampleCfgs[] = {
181 { "ms0", 0 }, { "ms4", 4 },
183 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(multiSampleCfgs); ndx++)
185 if (!strncmp(configName, multiSampleCfgs[ndx].name, strlen(multiSampleCfgs[ndx].name)))
187 cfgInfo.samples = multiSampleCfgs[ndx].samples;
189 configName += strlen(multiSampleCfgs[ndx].name);
197 static string getApiName(glu::ApiType apiType)
199 if (apiType == glu::ApiType::es(2, 0))
201 else if (apiType == glu::ApiType::es(3, 0))
203 else if (apiType == glu::ApiType::es(3, 1))
205 else if (apiType == glu::ApiType::es(3, 2))
208 throw std::runtime_error("Unknown context type");
211 static const string getCaseListFileOption(const string mustpassDir, const string apiName, const string mustpassName)
213 #if DE_OS == DE_OS_ANDROID
214 const string case_list_option = "--deqp-caselist-resource=";
216 const string case_list_option = "--deqp-caselist-file=";
218 return case_list_option + mustpassDir + apiName + "-" + mustpassName + ".txt";
221 static const string getLogFileName(const string apiName, const string configName, const int iterId, const int runId,
222 const int width, const int height, const int seed)
224 string res = string("config-") + apiName + "-" + configName + "-cfg-" + de::toString(iterId) + "-run-" +
225 de::toString(runId) + "-width-" + de::toString(width) + "-height-" + de::toString(height);
228 res += "-seed-" + de::toString(seed);
235 static void getBaseOptions(std::vector<std::string>& args, const string mustpassDir, const string apiName,
236 const string configName, const string screenRotation, int width, int height)
238 args.push_back(getCaseListFileOption(mustpassDir, apiName, configName));
239 args.push_back(string("--deqp-screen-rotation=") + screenRotation);
240 args.push_back(string("--deqp-surface-width=") + de::toString(width));
241 args.push_back(string("--deqp-surface-height=") + de::toString(height));
242 args.push_back("--deqp-watchdog=disable");
245 static bool isGLConfigCompatible(configInfo cfgInfo, const AOSPConfig& config)
247 return cfgInfo.redBits == config.redBits && cfgInfo.greenBits == config.greenBits &&
248 cfgInfo.blueBits == config.blueBits && cfgInfo.alphaBits == config.alphaBits &&
249 cfgInfo.depthBits == config.depthBits && cfgInfo.stencilBits == config.stencilBits &&
250 cfgInfo.samples == config.samples;
253 static void getTestRunsForAOSPEGL(vector<TestRunParams>& runs, const ConfigList& configs)
255 #include "glcAospMustpassEgl.hpp"
257 for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_egl_first_cfg); ++i)
259 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_egl_first_cfg[i].glConfigName);
261 vector<AOSPConfig>::const_iterator cfgIter;
262 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
264 // find first compatible config
265 if ((*cfgIter).type == CONFIGTYPE_EGL && isGLConfigCompatible(cfgInfo, *cfgIter))
271 if (cfgIter == configs.aospConfigs.end())
273 // No suitable configuration found. Skipping EGL tests
277 string apiName = "egl";
278 const int width = aosp_mustpass_egl_first_cfg[i].surfaceWidth;
279 const int height = aosp_mustpass_egl_first_cfg[i].surfaceHeight;
281 TestRunParams params;
283 getLogFileName(apiName, aosp_mustpass_egl_first_cfg[i].configName, 1, i, width, height, -1);
284 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_egl_first_cfg[i].configName,
285 aosp_mustpass_egl_first_cfg[i].screenRotation, width, height);
287 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_egl_first_cfg[i].glConfigName));
289 runs.push_back(params);
293 static void getTestRunsForAOSPES(vector<TestRunParams>& runs, const ConfigList& configs, const glu::ApiType apiType)
295 #include "glcAospMustpassEs.hpp"
297 for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_es_first_cfg); ++i)
299 if (!glu::contextSupports(glu::ContextType(apiType), aosp_mustpass_es_first_cfg[i].apiType))
302 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_es_first_cfg[i].glConfigName);
304 vector<AOSPConfig>::const_iterator cfgIter;
305 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
307 // find first compatible config
308 if (isGLConfigCompatible(cfgInfo, *cfgIter))
314 if (cfgIter == configs.aospConfigs.end())
316 TCU_FAIL(("No suitable configuration found for GL config " +
317 de::toString(aosp_mustpass_es_first_cfg[i].glConfigName))
322 string apiName = getApiName(aosp_mustpass_es_first_cfg[i].apiType);
323 const int width = aosp_mustpass_es_first_cfg[i].surfaceWidth;
324 const int height = aosp_mustpass_es_first_cfg[i].surfaceHeight;
326 TestRunParams params;
327 params.logFilename = getLogFileName(apiName, aosp_mustpass_es_first_cfg[i].configName, 1, i, width, height, -1);
328 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_es_first_cfg[i].configName,
329 aosp_mustpass_es_first_cfg[i].screenRotation, width, height);
331 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_es_first_cfg[i].glConfigName));
334 if ((*cfgIter).surfaceTypes & SURFACETYPE_WINDOW)
335 params.args.push_back("--deqp-surface-type=window");
336 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PBUFFER)
337 params.args.push_back("--deqp-surface-type=pbuffer");
338 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PIXMAP)
339 params.args.push_back("--deqp-surface-type=pixmap");
340 runs.push_back(params);
344 static void getTestRunsForES(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
346 getTestRunsForAOSPEGL(runs, configs);
347 getTestRunsForAOSPES(runs, configs, type);
349 #include "glcKhronosMustpassEs.hpp"
351 for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
353 const bool isFirst = cfgIter == configs.configs.begin();
354 const int numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_es_first_cfg) :
355 DE_LENGTH_OF_ARRAY(khronos_mustpass_es_other_cfg);
356 const RunParams* runParams = isFirst ? khronos_mustpass_es_first_cfg : khronos_mustpass_es_other_cfg;
358 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
360 if (!glu::contextSupports(glu::ContextType(type), runParams[runNdx].apiType))
363 string apiName = getApiName(runParams[runNdx].apiType);
364 const int width = runParams[runNdx].surfaceWidth;
365 const int height = runParams[runNdx].surfaceHeight;
366 const int seed = runParams[runNdx].baseSeed;
368 TestRunParams params;
371 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed);
373 getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName,
374 runParams[runNdx].screenRotation, width, height);
375 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
377 appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
379 runs.push_back(params);
384 static void getTestRunsForGL(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
386 const char* packageName = DE_NULL;
388 if (type == glu::ApiType::core(3, 0))
389 packageName = "GL30-CTS";
390 else if (type == glu::ApiType::core(3, 1))
391 packageName = "GL31-CTS";
392 else if (type == glu::ApiType::core(3, 2))
393 packageName = "GL32-CTS";
394 else if (type == glu::ApiType::core(3, 3))
395 packageName = "GL33-CTS";
396 else if (type == glu::ApiType::core(4, 0))
397 packageName = "GL40-CTS";
398 else if (type == glu::ApiType::core(4, 1))
399 packageName = "GL41-CTS";
400 else if (type == glu::ApiType::core(4, 2))
401 packageName = "GL42-CTS";
402 else if (type == glu::ApiType::core(4, 3))
403 packageName = "GL43-CTS";
404 else if (type == glu::ApiType::core(4, 4))
405 packageName = "GL44-CTS";
406 else if (type == glu::ApiType::core(4, 5))
407 packageName = "GL45-CTS";
409 throw std::runtime_error("Unknown context type");
411 DE_ASSERT(packageName);
418 const char* fboConfig; //!< Fbo config arg, or null if not used.
421 static const RunParams firstConfigParams[] = { { 64, 64, 1, DE_NULL },
422 { 113, 47, 2, DE_NULL },
423 { 64, -1, 3, "rgba8888d24s8" },
424 { -1, 64, 3, "rgba8888d24s8" } };
426 static const RunParams otherConfigParams[] = { { 64, 64, 1, DE_NULL }, { 113, 47, 2, DE_NULL } };
428 for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
430 const bool isFirst = cfgIter == configs.configs.begin();
431 const int numRunParams =
432 isFirst ? DE_LENGTH_OF_ARRAY(firstConfigParams) : DE_LENGTH_OF_ARRAY(otherConfigParams);
433 const RunParams* runParams = isFirst ? firstConfigParams : otherConfigParams;
435 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
437 const std::string width = de::toString(runParams[runNdx].width);
438 const std::string height = de::toString(runParams[runNdx].height);
439 const std::string seed = de::toString(runParams[runNdx].baseSeed);
441 TestRunParams params;
443 params.logFilename = string("config-") + de::toString(cfgIter->id) + "-run-" + de::toString(runNdx) +
444 "-width-" + de::toString(width) + "-height-" + de::toString(height) + "-seed-" + seed +
447 params.args.push_back(string("--deqp-case=") + packageName + ".*");
448 params.args.push_back(string("--deqp-surface-width=") + width);
449 params.args.push_back(string("--deqp-surface-height=") + height);
450 params.args.push_back(string("--deqp-base-seed=") + seed);
451 appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
453 runs.push_back(params);
458 static void getTestRunParams(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
460 switch (type.getProfile())
462 case glu::PROFILE_CORE:
463 getTestRunsForGL(type, configs, runs);
465 case glu::PROFILE_ES:
466 getTestRunsForES(type, configs, runs);
469 throw std::runtime_error("Unknown context type");
475 void operator()(FILE* file) const
482 struct XmlWriterDeleter
484 void operator()(qpXmlWriter* writer) const
487 qpXmlWriter_destroy(writer);
491 static const char* getRunTypeName(glu::ApiType type)
493 if (type == glu::ApiType::es(2, 0))
495 else if (type == glu::ApiType::es(3, 0))
497 else if (type == glu::ApiType::es(3, 1))
499 else if (type == glu::ApiType::es(3, 2))
501 else if (type == glu::ApiType::core(3, 0))
503 else if (type == glu::ApiType::core(3, 1))
505 else if (type == glu::ApiType::core(3, 2))
507 else if (type == glu::ApiType::core(3, 3))
509 else if (type == glu::ApiType::core(4, 0))
511 else if (type == glu::ApiType::core(4, 1))
513 else if (type == glu::ApiType::core(4, 2))
515 else if (type == glu::ApiType::core(4, 3))
517 else if (type == glu::ApiType::core(4, 4))
519 else if (type == glu::ApiType::core(4, 5))
525 #define XML_CHECK(X) \
527 throw tcu::Exception("Writing XML failed")
529 static void writeRunSummary(const TestRunSummary& summary, const char* filename)
531 de::UniquePtr<FILE, FileDeleter> out(fopen(filename, "wb"));
533 throw tcu::Exception(string("Failed to open ") + filename);
535 de::UniquePtr<qpXmlWriter, XmlWriterDeleter> writer(qpXmlWriter_createFileWriter(out.get(), DE_FALSE, DE_FALSE));
537 throw std::bad_alloc();
539 XML_CHECK(qpXmlWriter_startDocument(writer.get()));
542 qpXmlAttribute attribs[2];
544 attribs[0] = qpSetStringAttrib("Type", getRunTypeName(summary.runType));
545 attribs[1] = qpSetBoolAttrib("Conformant", summary.isConformant ? DE_TRUE : DE_FALSE);
547 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Summary", DE_LENGTH_OF_ARRAY(attribs), attribs));
552 qpXmlAttribute attribs[1];
553 attribs[0] = qpSetStringAttrib("FileName", summary.configLogFilename.c_str());
554 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Configs", DE_LENGTH_OF_ARRAY(attribs), attribs) &&
555 qpXmlWriter_endElement(writer.get(), "Configs"));
558 // Record test run parameters (log filename & command line).
559 for (vector<TestRunParams>::const_iterator runIter = summary.runParams.begin(); runIter != summary.runParams.end();
563 qpXmlAttribute attribs[2];
565 for (vector<string>::const_iterator argIter = runIter->args.begin(); argIter != runIter->args.end(); ++argIter)
567 if (argIter != runIter->args.begin())
572 attribs[0] = qpSetStringAttrib("FileName", runIter->logFilename.c_str());
573 attribs[1] = qpSetStringAttrib("CmdLine", cmdLine.c_str());
575 XML_CHECK(qpXmlWriter_startElement(writer.get(), "TestRun", DE_LENGTH_OF_ARRAY(attribs), attribs) &&
576 qpXmlWriter_endElement(writer.get(), "TestRun"));
579 XML_CHECK(qpXmlWriter_endElement(writer.get(), "Summary"));
580 XML_CHECK(qpXmlWriter_endDocument(writer.get()));
585 TestRunner::TestRunner(tcu::Platform& platform, tcu::Archive& archive, const char* logDirPath, glu::ApiType type,
587 : m_platform(platform)
589 , m_logDirPath(logDirPath)
592 , m_iterState(ITERATE_INIT)
593 , m_curSession(DE_NULL)
594 , m_sessionsExecuted(0)
595 , m_sessionsPassed(0)
596 , m_sessionsFailed(0)
600 TestRunner::~TestRunner(void)
605 bool TestRunner::iterate(void)
611 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
616 m_iterState = ITERATE_INIT;
619 case ITERATE_INIT_SESSION:
620 DE_ASSERT(m_sessionIter != m_runSessions.end());
621 initSession(*m_sessionIter);
622 if (m_flags & PRINT_SUMMARY)
623 m_iterState = ITERATE_DEINIT_SESSION;
625 m_iterState = ITERATE_ITERATE_SESSION;
628 case ITERATE_DEINIT_SESSION:
631 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
634 case ITERATE_ITERATE_SESSION:
635 if (!iterateSession())
636 m_iterState = ITERATE_DEINIT_SESSION;
645 void TestRunner::init(void)
647 DE_ASSERT(m_runSessions.empty() && m_summary.runParams.empty());
649 tcu::print("Running %s conformance\n", glu::getApiTypeDescription(m_type));
651 m_summary.runType = m_type;
653 // Get list of configs to test.
654 ConfigList configList;
655 getDefaultConfigList(m_platform, m_type, configList);
657 tcu::print(" found %d compatible and %d excluded configs\n", (int)configList.configs.size(),
658 (int)configList.excludedConfigs.size());
662 const char* configLogFilename = "configs.qpa";
663 TestRunParams configRun;
665 configRun.logFilename = configLogFilename;
666 configRun.args.push_back("--deqp-case=CTS-Configs.*");
667 m_runSessions.push_back(configRun);
669 m_summary.configLogFilename = configLogFilename;
672 // Conformance test type specific runs
673 getTestRunParams(m_type, configList, m_runSessions);
675 // Record run params for summary.
676 for (std::vector<TestRunParams>::const_iterator runIter = m_runSessions.begin() + 1; runIter != m_runSessions.end();
678 m_summary.runParams.push_back(*runIter);
681 m_sessionIter = m_runSessions.begin();
684 void TestRunner::deinit(void)
687 bool isConformant = m_sessionsExecuted == m_sessionsPassed;
688 DE_ASSERT(m_sessionsExecuted == m_sessionsPassed + m_sessionsFailed);
689 tcu::print("\n%d/%d sessions passed, conformance test %s\n", m_sessionsPassed, m_sessionsExecuted,
690 isConformant ? "PASSED" : "FAILED");
692 m_summary.isConformant = isConformant;
694 // Write out summary.
695 writeRunSummary(m_summary, de::FilePath::join(m_logDirPath, "cts-run-summary.xml").getPath());
697 m_runSessions.clear();
701 void TestRunner::initSession(const TestRunParams& runParams)
703 DE_ASSERT(!m_curSession);
705 tcu::print("\n Test run %d / %d\n", (int)(m_sessionIter - m_runSessions.begin() + 1), (int)m_runSessions.size());
707 // Compute final args for run.
708 vector<string> args(runParams.args);
709 args.push_back(string("--deqp-log-filename=") + de::FilePath::join(m_logDirPath, runParams.logFilename).getPath());
711 if (!(m_flags & VERBOSE_IMAGES))
712 args.push_back("--deqp-log-images=disable");
714 if (!(m_flags & VERBOSE_SHADERS))
715 args.push_back("--deqp-log-shader-sources=disable");
717 std::ostringstream ostr;
718 std::ostream_iterator<string> out_it(ostr, ", ");
719 std::copy(args.begin(), args.end(), out_it);
720 tcu::print("\n Config: %s \n\n", ostr.str().c_str());
722 // Translate to argc, argv
723 vector<const char*> argv;
724 argv.push_back("cts-runner"); // Dummy binary name
725 for (vector<string>::const_iterator i = args.begin(); i != args.end(); i++)
726 argv.push_back(i->c_str());
729 m_curSession = new RunSession(m_platform, m_archive, (int)argv.size(), &argv[0]);
732 void TestRunner::deinitSession(void)
734 DE_ASSERT(m_curSession);
737 // \note NotSupported is treated as pass.
738 const tcu::TestRunStatus& result = m_curSession->getResult();
740 result.numExecuted == (result.numPassed + result.numNotSupported + result.numWarnings) && result.isComplete;
742 DE_ASSERT(result.numExecuted == result.numPassed + result.numFailed + result.numNotSupported + result.numWarnings);
744 m_sessionsExecuted += 1;
745 (isOk ? m_sessionsPassed : m_sessionsFailed) += 1;
748 m_curSession = DE_NULL;
751 inline bool TestRunner::iterateSession(void)
753 return m_curSession->iterate();