external/openglcts/modules/runner/glcTestRunner.cpp

   1 /*-------------------------------------------------------------------------
   2  * OpenGL Conformance Test Suite
   3  * -----------------------------
   4  *
   5  * Copyright (c) 2016 Google Inc.
   6  * Copyright (c) 2016 The Khronos Group Inc.
   7  *
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *      http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  *
  20  */ /*!
  21  * \file
  22  * \brief CTS runner.
  23  */ /*-------------------------------------------------------------------*/
  24
  25 #include "glcTestRunner.hpp"
  26 #include "deFilePath.hpp"
  27 #include "deStringUtil.hpp"
  28 #include "deUniquePtr.hpp"
  29 #include "glcConfigList.hpp"
  30 #include "qpXmlWriter.h"
  31 #include "tcuApp.hpp"
  32 #include "tcuCommandLine.hpp"
  33 #include "tcuTestLog.hpp"
  34 #include "tcuTestSessionExecutor.hpp"
  35
  36 #include <iterator>
  37
  38 namespace glcts
  39 {
  40
  41 using std::vector;
  42 using std::string;
  43
  44 // RunSession
  45
  46 class RunSession
  47 {
  48 public:
  49         RunSession(tcu::Platform& platform, tcu::Archive& archive, const int numArgs, const char* const* args)
  50                 : m_cmdLine(numArgs, args)
  51                 , m_log(m_cmdLine.getLogFileName(), m_cmdLine.getLogFlags())
  52                 , m_app(platform, archive, m_log, m_cmdLine)
  53         {
  54         }
  55
  56         inline bool iterate(void)
  57         {
  58                 return m_app.iterate();
  59         }
  60
  61         inline const tcu::TestRunStatus& getResult(void) const
  62         {
  63                 return m_app.getResult();
  64         }
  65
  66 private:
  67         tcu::CommandLine m_cmdLine;
  68         tcu::TestLog     m_log;
  69         tcu::App                 m_app;
  70 };
  71
  72 static void appendConfigArgs(const Config& config, std::vector<std::string>& args, const char* fboConfig)
  73 {
  74         if (fboConfig != NULL)
  75         {
  76                 args.push_back(string("--deqp-gl-config-name=") + fboConfig);
  77                 args.push_back("--deqp-surface-type=fbo");
  78         }
  79
  80         if (config.type != CONFIGTYPE_DEFAULT)
  81         {
  82                 // \todo [2013-05-06 pyry] Test all surface types for some configs?
  83                 if (fboConfig == NULL)
  84                 {
  85                         if (config.surfaceTypes & SURFACETYPE_WINDOW)
  86                                 args.push_back("--deqp-surface-type=window");
  87                         else if (config.surfaceTypes & SURFACETYPE_PBUFFER)
  88                                 args.push_back("--deqp-surface-type=pbuffer");
  89                         else if (config.surfaceTypes & SURFACETYPE_PIXMAP)
  90                                 args.push_back("--deqp-surface-type=pixmap");
  91                 }
  92
  93                 args.push_back(string("--deqp-gl-config-id=") + de::toString(config.id));
  94
  95                 if (config.type == CONFIGTYPE_EGL)
  96                         args.push_back("--deqp-gl-context-type=egl");
  97                 else if (config.type == CONFIGTYPE_WGL)
  98                         args.push_back("--deqp-gl-context-type=wgl");
  99         }
 100 }
 101
 102 typedef struct configInfo
 103 {
 104         deInt32 redBits;
 105         deInt32 greenBits;
 106         deInt32 blueBits;
 107         deInt32 alphaBits;
 108         deInt32 depthBits;
 109         deInt32 stencilBits;
 110         deInt32 samples;
 111 } configInfo;
 112
 113 static configInfo parseConfigBitsFromName(const char* configName)
 114 {
 115         configInfo cfgInfo;
 116         static const struct
 117         {
 118                 const char* name;
 119                 int                     redBits;
 120                 int                     greenBits;
 121                 int                     blueBits;
 122                 int                     alphaBits;
 123         } colorCfgs[] = {
 124                 { "rgba8888", 8, 8, 8, 8 }, { "rgb565", 5, 6, 5, 0 },
 125         };
 126         for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(colorCfgs); ndx++)
 127         {
 128                 if (!strncmp(configName, colorCfgs[ndx].name, strlen(colorCfgs[ndx].name)))
 129                 {
 130                         cfgInfo.redBits   = colorCfgs[ndx].redBits;
 131                         cfgInfo.greenBits = colorCfgs[ndx].greenBits;
 132                         cfgInfo.blueBits  = colorCfgs[ndx].blueBits;
 133                         cfgInfo.alphaBits = colorCfgs[ndx].alphaBits;
 134
 135                         configName += strlen(colorCfgs[ndx].name);
 136                         break;
 137                 }
 138         }
 139
 140         static const struct
 141         {
 142                 const char* name;
 143                 int                     depthBits;
 144         } depthCfgs[] = {
 145                 { "d0", 0 }, { "d24", 24 },
 146         };
 147         for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(depthCfgs); ndx++)
 148         {
 149                 if (!strncmp(configName, depthCfgs[ndx].name, strlen(depthCfgs[ndx].name)))
 150                 {
 151                         cfgInfo.depthBits = depthCfgs[ndx].depthBits;
 152
 153                         configName += strlen(depthCfgs[ndx].name);
 154                         break;
 155                 }
 156         }
 157
 158         static const struct
 159         {
 160                 const char* name;
 161                 int                     stencilBits;
 162         } stencilCfgs[] = {
 163                 { "s0", 0 }, { "s8", 8 },
 164         };
 165         for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(stencilCfgs); ndx++)
 166         {
 167                 if (!strncmp(configName, stencilCfgs[ndx].name, strlen(stencilCfgs[ndx].name)))
 168                 {
 169                         cfgInfo.stencilBits = stencilCfgs[ndx].stencilBits;
 170
 171                         configName += strlen(stencilCfgs[ndx].name);
 172                         break;
 173                 }
 174         }
 175
 176         static const struct
 177         {
 178                 const char* name;
 179                 int                     samples;
 180         } multiSampleCfgs[] = {
 181                 { "ms0", 0 }, { "ms4", 4 },
 182         };
 183         for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(multiSampleCfgs); ndx++)
 184         {
 185                 if (!strncmp(configName, multiSampleCfgs[ndx].name, strlen(multiSampleCfgs[ndx].name)))
 186                 {
 187                         cfgInfo.samples = multiSampleCfgs[ndx].samples;
 188
 189                         configName += strlen(multiSampleCfgs[ndx].name);
 190                         break;
 191                 }
 192         }
 193
 194         return cfgInfo;
 195 }
 196
 197 static string getApiName(glu::ApiType apiType)
 198 {
 199         if (apiType == glu::ApiType::es(2, 0))
 200                 return "gles2";
 201         else if (apiType == glu::ApiType::es(3, 0))
 202                 return "gles3";
 203         else if (apiType == glu::ApiType::es(3, 1))
 204                 return "gles31";
 205         else if (apiType == glu::ApiType::es(3, 2))
 206                 return "gles32";
 207         else if (apiType == glu::ApiType::core(3, 0))
 208                 return "gl30";
 209         else if (apiType == glu::ApiType::core(3, 1))
 210                 return "gl31";
 211         else if (apiType == glu::ApiType::core(3, 2))
 212                 return "gl32";
 213         else if (apiType == glu::ApiType::core(3, 3))
 214                 return "gl33";
 215         else if (apiType == glu::ApiType::core(4, 0))
 216                 return "gl40";
 217         else if (apiType == glu::ApiType::core(4, 1))
 218                 return "gl41";
 219         else if (apiType == glu::ApiType::core(4, 2))
 220                 return "gl42";
 221         else if (apiType == glu::ApiType::core(4, 3))
 222                 return "gl43";
 223         else if (apiType == glu::ApiType::core(4, 4))
 224                 return "gl44";
 225         else if (apiType == glu::ApiType::core(4, 5))
 226                 return "gl45";
 227         else
 228                 throw std::runtime_error("Unknown context type");
 229 }
 230
 231 static const string getCaseListFileOption(const string mustpassDir, const string apiName, const string mustpassName)
 232 {
 233 #if DE_OS == DE_OS_ANDROID
 234         const string case_list_option = "--deqp-caselist-resource=";
 235 #else
 236         const string case_list_option = "--deqp-caselist-file=";
 237 #endif
 238         return case_list_option + mustpassDir + apiName + "-" + mustpassName + ".txt";
 239 }
 240
 241 static const string getLogFileName(const string apiName, const string configName, const int iterId, const int runId,
 242                                                                    const int width, const int height, const int seed)
 243 {
 244         string res = string("config-") + apiName + "-" + configName + "-cfg-" + de::toString(iterId) + "-run-" +
 245                                  de::toString(runId) + "-width-" + de::toString(width) + "-height-" + de::toString(height);
 246         if (seed != -1)
 247         {
 248                 res += "-seed-" + de::toString(seed);
 249         }
 250         res += ".qpa";
 251
 252         return res;
 253 }
 254
 255 static void getBaseOptions(std::vector<std::string>& args, const string mustpassDir, const string apiName,
 256                                                    const string configName, const string screenRotation, int width, int height)
 257 {
 258         args.push_back(getCaseListFileOption(mustpassDir, apiName, configName));
 259         args.push_back(string("--deqp-screen-rotation=") + screenRotation);
 260         args.push_back(string("--deqp-surface-width=") + de::toString(width));
 261         args.push_back(string("--deqp-surface-height=") + de::toString(height));
 262         args.push_back("--deqp-watchdog=disable");
 263 }
 264
 265 static bool isGLConfigCompatible(configInfo cfgInfo, const AOSPConfig& config)
 266 {
 267         return cfgInfo.redBits == config.redBits && cfgInfo.greenBits == config.greenBits &&
 268                    cfgInfo.blueBits == config.blueBits && cfgInfo.alphaBits == config.alphaBits &&
 269                    cfgInfo.depthBits == config.depthBits && cfgInfo.stencilBits == config.stencilBits &&
 270                    cfgInfo.samples == config.samples;
 271 }
 272
 273 static void getTestRunsForAOSPEGL(vector<TestRunParams>& runs, const ConfigList& configs)
 274 {
 275 #include "glcAospMustpassEgl.hpp"
 276
 277         for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_egl_first_cfg); ++i)
 278         {
 279                 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_egl_first_cfg[i].glConfigName);
 280
 281                 vector<AOSPConfig>::const_iterator cfgIter;
 282                 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
 283                 {
 284                         // find first compatible config
 285                         if ((*cfgIter).type == CONFIGTYPE_EGL && isGLConfigCompatible(cfgInfo, *cfgIter))
 286                         {
 287                                 break;
 288                         }
 289                 }
 290
 291                 if (cfgIter == configs.aospConfigs.end())
 292                 {
 293                         // No suitable configuration found. Skipping EGL tests
 294                         continue;
 295                 }
 296
 297                 string  apiName = "egl";
 298                 const int width   = aosp_mustpass_egl_first_cfg[i].surfaceWidth;
 299                 const int height  = aosp_mustpass_egl_first_cfg[i].surfaceHeight;
 300
 301                 TestRunParams params;
 302                 params.logFilename =
 303                         getLogFileName(apiName, aosp_mustpass_egl_first_cfg[i].configName, 1, i, width, height, -1);
 304                 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_egl_first_cfg[i].configName,
 305                                            aosp_mustpass_egl_first_cfg[i].screenRotation, width, height);
 306
 307                 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_egl_first_cfg[i].glConfigName));
 308
 309                 runs.push_back(params);
 310         }
 311 }
 312
 313 static void getTestRunsForAOSPES(vector<TestRunParams>& runs, const ConfigList& configs, const glu::ApiType apiType)
 314 {
 315 #include "glcAospMustpassEs.hpp"
 316
 317         for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_es_first_cfg); ++i)
 318         {
 319                 if (!glu::contextSupports(glu::ContextType(apiType), aosp_mustpass_es_first_cfg[i].apiType))
 320                         continue;
 321
 322                 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_es_first_cfg[i].glConfigName);
 323
 324                 vector<AOSPConfig>::const_iterator cfgIter;
 325                 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
 326                 {
 327                         // find first compatible config
 328                         if (isGLConfigCompatible(cfgInfo, *cfgIter))
 329                         {
 330                                 break;
 331                         }
 332                 }
 333
 334                 if (cfgIter == configs.aospConfigs.end())
 335                 {
 336                         TCU_FAIL(("No suitable configuration found for GL config " +
 337                                           de::toString(aosp_mustpass_es_first_cfg[i].glConfigName))
 338                                                  .c_str());
 339                         return;
 340                 }
 341
 342                 string  apiName = getApiName(aosp_mustpass_es_first_cfg[i].apiType);
 343                 const int width   = aosp_mustpass_es_first_cfg[i].surfaceWidth;
 344                 const int height  = aosp_mustpass_es_first_cfg[i].surfaceHeight;
 345
 346                 TestRunParams params;
 347                 params.logFilename = getLogFileName(apiName, aosp_mustpass_es_first_cfg[i].configName, 1, i, width, height, -1);
 348                 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_es_first_cfg[i].configName,
 349                                            aosp_mustpass_es_first_cfg[i].screenRotation, width, height);
 350
 351                 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_es_first_cfg[i].glConfigName));
 352
 353                 //set surface type
 354                 if ((*cfgIter).surfaceTypes & SURFACETYPE_WINDOW)
 355                         params.args.push_back("--deqp-surface-type=window");
 356                 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PBUFFER)
 357                         params.args.push_back("--deqp-surface-type=pbuffer");
 358                 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PIXMAP)
 359                         params.args.push_back("--deqp-surface-type=pixmap");
 360                 runs.push_back(params);
 361         }
 362 }
 363
 364 static void getTestRunsForES(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
 365 {
 366         getTestRunsForAOSPEGL(runs, configs);
 367         getTestRunsForAOSPES(runs, configs, type);
 368
 369 #include "glcKhronosMustpassEs.hpp"
 370
 371         for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
 372         {
 373                 const bool isFirst              = cfgIter == configs.configs.begin();
 374                 const int  numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_es_first_cfg) :
 375                                                                                    DE_LENGTH_OF_ARRAY(khronos_mustpass_es_other_cfg);
 376                 const RunParams* runParams = isFirst ? khronos_mustpass_es_first_cfg : khronos_mustpass_es_other_cfg;
 377
 378                 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
 379                 {
 380                         if (!glu::contextSupports(glu::ContextType(type), runParams[runNdx].apiType))
 381                                 continue;
 382
 383                         string  apiName = getApiName(runParams[runNdx].apiType);
 384                         const int width   = runParams[runNdx].surfaceWidth;
 385                         const int height  = runParams[runNdx].surfaceHeight;
 386                         const int seed  = runParams[runNdx].baseSeed;
 387
 388                         TestRunParams params;
 389
 390                         params.logFilename =
 391                                 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed);
 392
 393                         getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName,
 394                                                    runParams[runNdx].screenRotation, width, height);
 395                         params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
 396
 397                         appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
 398
 399                         runs.push_back(params);
 400                 }
 401         }
 402 }
 403
 404 static void getTestRunsForGL(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
 405 {
 406 #include "glcKhronosMustpassGl.hpp"
 407
 408         for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
 409         {
 410                 const bool isFirst              = cfgIter == configs.configs.begin();
 411                 const int  numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_first_cfg) :
 412                                                                                    DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_other_cfg);
 413                 const RunParams* runParams = isFirst ? khronos_mustpass_gl_first_cfg : khronos_mustpass_gl_other_cfg;
 414
 415                 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
 416                 {
 417                         if (type != runParams[runNdx].apiType)
 418                                 continue;
 419
 420                         string  apiName = getApiName(runParams[runNdx].apiType);
 421                         const int width   = runParams[runNdx].surfaceWidth;
 422                         const int height  = runParams[runNdx].surfaceHeight;
 423                         const int seed  = runParams[runNdx].baseSeed;
 424
 425                         TestRunParams params;
 426
 427                         params.logFilename =
 428                                 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed);
 429
 430                         getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName,
 431                                                    runParams[runNdx].screenRotation, width, height);
 432                         params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
 433
 434                         appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
 435
 436                         runs.push_back(params);
 437                 }
 438         }
 439 }
 440
 441 static void getTestRunParams(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
 442 {
 443         switch (type.getProfile())
 444         {
 445         case glu::PROFILE_CORE:
 446                 getTestRunsForGL(type, configs, runs);
 447                 break;
 448         case glu::PROFILE_ES:
 449                 getTestRunsForES(type, configs, runs);
 450                 break;
 451         default:
 452                 throw std::runtime_error("Unknown context type");
 453         }
 454 }
 455
 456 struct FileDeleter
 457 {
 458         void operator()(FILE* file) const
 459         {
 460                 if (file)
 461                         fclose(file);
 462         }
 463 };
 464
 465 struct XmlWriterDeleter
 466 {
 467         void operator()(qpXmlWriter* writer) const
 468         {
 469                 if (writer)
 470                         qpXmlWriter_destroy(writer);
 471         }
 472 };
 473
 474 static const char* getRunTypeName(glu::ApiType type)
 475 {
 476         if (type == glu::ApiType::es(2, 0))
 477                 return "es2";
 478         else if (type == glu::ApiType::es(3, 0))
 479                 return "es3";
 480         else if (type == glu::ApiType::es(3, 1))
 481                 return "es31";
 482         else if (type == glu::ApiType::es(3, 2))
 483                 return "es32";
 484         else if (type == glu::ApiType::core(3, 0))
 485                 return "gl30";
 486         else if (type == glu::ApiType::core(3, 1))
 487                 return "gl31";
 488         else if (type == glu::ApiType::core(3, 2))
 489                 return "gl32";
 490         else if (type == glu::ApiType::core(3, 3))
 491                 return "gl33";
 492         else if (type == glu::ApiType::core(4, 0))
 493                 return "gl40";
 494         else if (type == glu::ApiType::core(4, 1))
 495                 return "gl41";
 496         else if (type == glu::ApiType::core(4, 2))
 497                 return "gl42";
 498         else if (type == glu::ApiType::core(4, 3))
 499                 return "gl43";
 500         else if (type == glu::ApiType::core(4, 4))
 501                 return "gl44";
 502         else if (type == glu::ApiType::core(4, 5))
 503                 return "gl45";
 504         else
 505                 return DE_NULL;
 506 }
 507
 508 #define XML_CHECK(X) \
 509         if (!(X))        \
 510         throw tcu::Exception("Writing XML failed")
 511
 512 static void writeRunSummary(const TestRunSummary& summary, const char* filename)
 513 {
 514         de::UniquePtr<FILE, FileDeleter> out(fopen(filename, "wb"));
 515         if (!out)
 516                 throw tcu::Exception(string("Failed to open ") + filename);
 517
 518         de::UniquePtr<qpXmlWriter, XmlWriterDeleter> writer(qpXmlWriter_createFileWriter(out.get(), DE_FALSE, DE_FALSE));
 519         if (!writer)
 520                 throw std::bad_alloc();
 521
 522         XML_CHECK(qpXmlWriter_startDocument(writer.get()));
 523
 524         {
 525                 qpXmlAttribute attribs[2];
 526
 527                 attribs[0] = qpSetStringAttrib("Type", getRunTypeName(summary.runType));
 528                 attribs[1] = qpSetBoolAttrib("Conformant", summary.isConformant ? DE_TRUE : DE_FALSE);
 529
 530                 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Summary", DE_LENGTH_OF_ARRAY(attribs), attribs));
 531         }
 532
 533         // Config run
 534         {
 535                 qpXmlAttribute attribs[1];
 536                 attribs[0] = qpSetStringAttrib("FileName", summary.configLogFilename.c_str());
 537                 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Configs", DE_LENGTH_OF_ARRAY(attribs), attribs) &&
 538                                   qpXmlWriter_endElement(writer.get(), "Configs"));
 539         }
 540
 541         // Record test run parameters (log filename & command line).
 542         for (vector<TestRunParams>::const_iterator runIter = summary.runParams.begin(); runIter != summary.runParams.end();
 543                  ++runIter)
 544         {
 545                 string             cmdLine;
 546                 qpXmlAttribute attribs[2];
 547
 548                 for (vector<string>::const_iterator argIter = runIter->args.begin(); argIter != runIter->args.end(); ++argIter)
 549                 {
 550                         if (argIter != runIter->args.begin())
 551                                 cmdLine += " ";
 552                         cmdLine += *argIter;
 553                 }
 554
 555                 attribs[0] = qpSetStringAttrib("FileName", runIter->logFilename.c_str());
 556                 attribs[1] = qpSetStringAttrib("CmdLine", cmdLine.c_str());
 557
 558                 XML_CHECK(qpXmlWriter_startElement(writer.get(), "TestRun", DE_LENGTH_OF_ARRAY(attribs), attribs) &&
 559                                   qpXmlWriter_endElement(writer.get(), "TestRun"));
 560         }
 561
 562         XML_CHECK(qpXmlWriter_endElement(writer.get(), "Summary"));
 563         XML_CHECK(qpXmlWriter_endDocument(writer.get()));
 564 }
 565
 566 #undef XML_CHECK
 567
 568 TestRunner::TestRunner(tcu::Platform& platform, tcu::Archive& archive, const char* logDirPath, glu::ApiType type,
 569                                            deUint32 flags)
 570         : m_platform(platform)
 571         , m_archive(archive)
 572         , m_logDirPath(logDirPath)
 573         , m_type(type)
 574         , m_flags(flags)
 575         , m_iterState(ITERATE_INIT)
 576         , m_curSession(DE_NULL)
 577         , m_sessionsExecuted(0)
 578         , m_sessionsPassed(0)
 579         , m_sessionsFailed(0)
 580 {
 581 }
 582
 583 TestRunner::~TestRunner(void)
 584 {
 585         delete m_curSession;
 586 }
 587
 588 bool TestRunner::iterate(void)
 589 {
 590         switch (m_iterState)
 591         {
 592         case ITERATE_INIT:
 593                 init();
 594                 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
 595                 return true;
 596
 597         case ITERATE_DEINIT:
 598                 deinit();
 599                 m_iterState = ITERATE_INIT;
 600                 return false;
 601
 602         case ITERATE_INIT_SESSION:
 603                 DE_ASSERT(m_sessionIter != m_runSessions.end());
 604                 initSession(*m_sessionIter);
 605                 if (m_flags & PRINT_SUMMARY)
 606                         m_iterState = ITERATE_DEINIT_SESSION;
 607                 else
 608                         m_iterState = ITERATE_ITERATE_SESSION;
 609                 return true;
 610
 611         case ITERATE_DEINIT_SESSION:
 612                 deinitSession();
 613                 ++m_sessionIter;
 614                 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
 615                 return true;
 616
 617         case ITERATE_ITERATE_SESSION:
 618                 if (!iterateSession())
 619                         m_iterState = ITERATE_DEINIT_SESSION;
 620                 return true;
 621
 622         default:
 623                 DE_ASSERT(false);
 624                 return false;
 625         }
 626 }
 627
 628 void TestRunner::init(void)
 629 {
 630         DE_ASSERT(m_runSessions.empty() && m_summary.runParams.empty());
 631
 632         tcu::print("Running %s conformance\n", glu::getApiTypeDescription(m_type));
 633
 634         m_summary.runType = m_type;
 635
 636         // Get list of configs to test.
 637         ConfigList configList;
 638         getDefaultConfigList(m_platform, m_type, configList);
 639
 640         tcu::print("  found %d compatible and %d excluded configs\n", (int)configList.configs.size(),
 641                            (int)configList.excludedConfigs.size());
 642
 643         // Config list run.
 644         {
 645                 const char*   configLogFilename = "configs.qpa";
 646                 TestRunParams configRun;
 647
 648                 configRun.logFilename = configLogFilename;
 649                 configRun.args.push_back("--deqp-case=CTS-Configs.*");
 650                 m_runSessions.push_back(configRun);
 651
 652                 m_summary.configLogFilename = configLogFilename;
 653         }
 654
 655         // Conformance test type specific runs
 656         getTestRunParams(m_type, configList, m_runSessions);
 657
 658         // Record run params for summary.
 659         for (std::vector<TestRunParams>::const_iterator runIter = m_runSessions.begin() + 1; runIter != m_runSessions.end();
 660                  ++runIter)
 661                 m_summary.runParams.push_back(*runIter);
 662
 663         // Session iterator
 664         m_sessionIter = m_runSessions.begin();
 665 }
 666
 667 void TestRunner::deinit(void)
 668 {
 669         // Print out totals.
 670         bool isConformant = m_sessionsExecuted == m_sessionsPassed;
 671         DE_ASSERT(m_sessionsExecuted == m_sessionsPassed + m_sessionsFailed);
 672         tcu::print("\n%d/%d sessions passed, conformance test %s\n", m_sessionsPassed, m_sessionsExecuted,
 673                            isConformant ? "PASSED" : "FAILED");
 674
 675         m_summary.isConformant = isConformant;
 676
 677         // Write out summary.
 678         writeRunSummary(m_summary, de::FilePath::join(m_logDirPath, "cts-run-summary.xml").getPath());
 679
 680         m_runSessions.clear();
 681         m_summary.clear();
 682 }
 683
 684 void TestRunner::initSession(const TestRunParams& runParams)
 685 {
 686         DE_ASSERT(!m_curSession);
 687
 688         tcu::print("\n  Test run %d / %d\n", (int)(m_sessionIter - m_runSessions.begin() + 1), (int)m_runSessions.size());
 689
 690         // Compute final args for run.
 691         vector<string> args(runParams.args);
 692         args.push_back(string("--deqp-log-filename=") + de::FilePath::join(m_logDirPath, runParams.logFilename).getPath());
 693
 694         if (!(m_flags & VERBOSE_IMAGES))
 695                 args.push_back("--deqp-log-images=disable");
 696
 697         if (!(m_flags & VERBOSE_SHADERS))
 698                 args.push_back("--deqp-log-shader-sources=disable");
 699
 700         std::ostringstream                        ostr;
 701         std::ostream_iterator<string> out_it(ostr, ", ");
 702         std::copy(args.begin(), args.end(), out_it);
 703         tcu::print("\n  Config: %s \n\n", ostr.str().c_str());
 704
 705         // Translate to argc, argv
 706         vector<const char*> argv;
 707         argv.push_back("cts-runner"); // Dummy binary name
 708         for (vector<string>::const_iterator i = args.begin(); i != args.end(); i++)
 709                 argv.push_back(i->c_str());
 710
 711         // Create session
 712         m_curSession = new RunSession(m_platform, m_archive, (int)argv.size(), &argv[0]);
 713 }
 714
 715 void TestRunner::deinitSession(void)
 716 {
 717         DE_ASSERT(m_curSession);
 718
 719         // Collect results.
 720         // \note NotSupported is treated as pass.
 721         const tcu::TestRunStatus& result = m_curSession->getResult();
 722         bool                                      isOk =
 723                 result.numExecuted == (result.numPassed + result.numNotSupported + result.numWarnings) && result.isComplete;
 724
 725         DE_ASSERT(result.numExecuted == result.numPassed + result.numFailed + result.numNotSupported + result.numWarnings);
 726
 727         m_sessionsExecuted += 1;
 728         (isOk ? m_sessionsPassed : m_sessionsFailed) += 1;
 729
 730         delete m_curSession;
 731         m_curSession = DE_NULL;
 732 }
 733
 734 inline bool TestRunner::iterateSession(void)
 735 {
 736         return m_curSession->iterate();
 737 }
 738
 739 } // glcts