external/openglcts/modules/runner/glcTestRunner.cpp

   1 /*-------------------------------------------------------------------------
   2  * OpenGL Conformance Test Suite
   3  * -----------------------------
   4  *
   5  * Copyright (c) 2016 Google Inc.
   6  * Copyright (c) 2016 The Khronos Group Inc.
   7  *
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *      http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  *
  20  */ /*!
  21  * \file
  22  * \brief CTS runner.
  23  */ /*-------------------------------------------------------------------*/
  24
  25 #include "glcTestRunner.hpp"
  26 #include "deFilePath.hpp"
  27 #include "deStringUtil.hpp"
  28 #include "deUniquePtr.hpp"
  29 #include "glcConfigList.hpp"
  30 #include "qpXmlWriter.h"
  31 #include "tcuApp.hpp"
  32 #include "tcuCommandLine.hpp"
  33 #include "tcuTestLog.hpp"
  34 #include "tcuTestSessionExecutor.hpp"
  35
  36 #include <iterator>
  37
  38 namespace glcts
  39 {
  40
  41 using std::vector;
  42 using std::string;
  43
  44 // RunSession
  45
  46 class RunSession
  47 {
  48 public:
  49         RunSession(tcu::Platform& platform, tcu::Archive& archive, const int numArgs, const char* const* args)
  50                 : m_cmdLine(numArgs, args)
  51                 , m_log(m_cmdLine.getLogFileName(), m_cmdLine.getLogFlags())
  52                 , m_app(platform, archive, m_log, m_cmdLine)
  53         {
  54         }
  55
  56         inline bool iterate(void)
  57         {
  58                 return m_app.iterate();
  59         }
  60
  61         inline const tcu::TestRunStatus& getResult(void) const
  62         {
  63                 return m_app.getResult();
  64         }
  65
  66 private:
  67         tcu::CommandLine m_cmdLine;
  68         tcu::TestLog     m_log;
  69         tcu::App                 m_app;
  70 };
  71
  72 static void appendConfigArgs(const Config& config, std::vector<std::string>& args, const char* fboConfig)
  73 {
  74         if (fboConfig != NULL)
  75         {
  76                 args.push_back(string("--deqp-gl-config-name=") + fboConfig);
  77                 args.push_back("--deqp-surface-type=fbo");
  78         }
  79
  80         if (config.type != CONFIGTYPE_DEFAULT)
  81         {
  82                 // \todo [2013-05-06 pyry] Test all surface types for some configs?
  83                 if (fboConfig == NULL)
  84                 {
  85                         if (config.surfaceTypes & SURFACETYPE_WINDOW)
  86                                 args.push_back("--deqp-surface-type=window");
  87                         else if (config.surfaceTypes & SURFACETYPE_PBUFFER)
  88                                 args.push_back("--deqp-surface-type=pbuffer");
  89                         else if (config.surfaceTypes & SURFACETYPE_PIXMAP)
  90                                 args.push_back("--deqp-surface-type=pixmap");
  91                 }
  92
  93                 args.push_back(string("--deqp-gl-config-id=") + de::toString(config.id));
  94
  95                 if (config.type == CONFIGTYPE_EGL)
  96                         args.push_back("--deqp-gl-context-type=egl");
  97                 else if (config.type == CONFIGTYPE_WGL)
  98                         args.push_back("--deqp-gl-context-type=wgl");
  99         }
 100 }
 101
 102 typedef struct configInfo
 103 {
 104         deInt32 redBits;
 105         deInt32 greenBits;
 106         deInt32 blueBits;
 107         deInt32 alphaBits;
 108         deInt32 depthBits;
 109         deInt32 stencilBits;
 110         deInt32 samples;
 111 } configInfo;
 112
 113 static configInfo parseConfigBitsFromName(const char* configName)
 114 {
 115         configInfo cfgInfo;
 116         static const struct
 117         {
 118                 const char* name;
 119                 int                     redBits;
 120                 int                     greenBits;
 121                 int                     blueBits;
 122                 int                     alphaBits;
 123         } colorCfgs[] = {
 124                 { "rgba8888", 8, 8, 8, 8 }, { "rgb565", 5, 6, 5, 0 },
 125         };
 126         for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(colorCfgs); ndx++)
 127         {
 128                 if (!strncmp(configName, colorCfgs[ndx].name, strlen(colorCfgs[ndx].name)))
 129                 {
 130                         cfgInfo.redBits   = colorCfgs[ndx].redBits;
 131                         cfgInfo.greenBits = colorCfgs[ndx].greenBits;
 132                         cfgInfo.blueBits  = colorCfgs[ndx].blueBits;
 133                         cfgInfo.alphaBits = colorCfgs[ndx].alphaBits;
 134
 135                         configName += strlen(colorCfgs[ndx].name);
 136                         break;
 137                 }
 138         }
 139
 140         static const struct
 141         {
 142                 const char* name;
 143                 int                     depthBits;
 144         } depthCfgs[] = {
 145                 { "d0", 0 }, { "d24", 24 },
 146         };
 147         for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(depthCfgs); ndx++)
 148         {
 149                 if (!strncmp(configName, depthCfgs[ndx].name, strlen(depthCfgs[ndx].name)))
 150                 {
 151                         cfgInfo.depthBits = depthCfgs[ndx].depthBits;
 152
 153                         configName += strlen(depthCfgs[ndx].name);
 154                         break;
 155                 }
 156         }
 157
 158         static const struct
 159         {
 160                 const char* name;
 161                 int                     stencilBits;
 162         } stencilCfgs[] = {
 163                 { "s0", 0 }, { "s8", 8 },
 164         };
 165         for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(stencilCfgs); ndx++)
 166         {
 167                 if (!strncmp(configName, stencilCfgs[ndx].name, strlen(stencilCfgs[ndx].name)))
 168                 {
 169                         cfgInfo.stencilBits = stencilCfgs[ndx].stencilBits;
 170
 171                         configName += strlen(stencilCfgs[ndx].name);
 172                         break;
 173                 }
 174         }
 175
 176         static const struct
 177         {
 178                 const char* name;
 179                 int                     samples;
 180         } multiSampleCfgs[] = {
 181                 { "ms0", 0 }, { "ms4", 4 },
 182         };
 183         for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(multiSampleCfgs); ndx++)
 184         {
 185                 if (!strncmp(configName, multiSampleCfgs[ndx].name, strlen(multiSampleCfgs[ndx].name)))
 186                 {
 187                         cfgInfo.samples = multiSampleCfgs[ndx].samples;
 188
 189                         configName += strlen(multiSampleCfgs[ndx].name);
 190                         break;
 191                 }
 192         }
 193
 194         return cfgInfo;
 195 }
 196
 197 static string getApiName(glu::ApiType apiType)
 198 {
 199         if (apiType == glu::ApiType::es(2, 0))
 200                 return "gles2";
 201         else if (apiType == glu::ApiType::es(3, 0))
 202                 return "gles3";
 203         else if (apiType == glu::ApiType::es(3, 1))
 204                 return "gles31";
 205         else if (apiType == glu::ApiType::es(3, 2))
 206                 return "gles32";
 207         else
 208                 throw std::runtime_error("Unknown context type");
 209 }
 210
 211 static const string getCaseListFileOption(const string mustpassDir, const string apiName, const string mustpassName)
 212 {
 213 #if DE_OS == DE_OS_ANDROID
 214         const string case_list_option = "--deqp-caselist-resource=";
 215 #else
 216         const string case_list_option = "--deqp-caselist-file=";
 217 #endif
 218         return case_list_option + mustpassDir + apiName + "-" + mustpassName + ".txt";
 219 }
 220
 221 static const string getLogFileName(const string apiName, const string configName, const int iterId, const int runId,
 222                                                                    const int width, const int height, const int seed)
 223 {
 224         string res = string("config-") + apiName + "-" + configName + "-cfg-" + de::toString(iterId) + "-run-" +
 225                                  de::toString(runId) + "-width-" + de::toString(width) + "-height-" + de::toString(height);
 226         if (seed != -1)
 227         {
 228                 res += "-seed-" + de::toString(seed);
 229         }
 230         res += ".qpa";
 231
 232         return res;
 233 }
 234
 235 static void getBaseOptions(std::vector<std::string>& args, const string mustpassDir, const string apiName,
 236                                                    const string configName, const string screenRotation, int width, int height)
 237 {
 238         args.push_back(getCaseListFileOption(mustpassDir, apiName, configName));
 239         args.push_back(string("--deqp-screen-rotation=") + screenRotation);
 240         args.push_back(string("--deqp-surface-width=") + de::toString(width));
 241         args.push_back(string("--deqp-surface-height=") + de::toString(height));
 242         args.push_back("--deqp-watchdog=disable");
 243 }
 244
 245 static bool isGLConfigCompatible(configInfo cfgInfo, const AOSPConfig& config)
 246 {
 247         return cfgInfo.redBits == config.redBits && cfgInfo.greenBits == config.greenBits &&
 248                    cfgInfo.blueBits == config.blueBits && cfgInfo.alphaBits == config.alphaBits &&
 249                    cfgInfo.depthBits == config.depthBits && cfgInfo.stencilBits == config.stencilBits &&
 250                    cfgInfo.samples == config.samples;
 251 }
 252
 253 static void getTestRunsForAOSPEGL(vector<TestRunParams>& runs, const ConfigList& configs)
 254 {
 255 #include "glcAospMustpassEgl.hpp"
 256
 257         for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_egl_first_cfg); ++i)
 258         {
 259                 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_egl_first_cfg[i].glConfigName);
 260
 261                 vector<AOSPConfig>::const_iterator cfgIter;
 262                 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
 263                 {
 264                         // find first compatible config
 265                         if ((*cfgIter).type == CONFIGTYPE_EGL && isGLConfigCompatible(cfgInfo, *cfgIter))
 266                         {
 267                                 break;
 268                         }
 269                 }
 270
 271                 if (cfgIter == configs.aospConfigs.end())
 272                 {
 273                         // No suitable configuration found. Skipping EGL tests
 274                         continue;
 275                 }
 276
 277                 string  apiName = "egl";
 278                 const int width   = aosp_mustpass_egl_first_cfg[i].surfaceWidth;
 279                 const int height  = aosp_mustpass_egl_first_cfg[i].surfaceHeight;
 280
 281                 TestRunParams params;
 282                 params.logFilename =
 283                         getLogFileName(apiName, aosp_mustpass_egl_first_cfg[i].configName, 1, i, width, height, -1);
 284                 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_egl_first_cfg[i].configName,
 285                                            aosp_mustpass_egl_first_cfg[i].screenRotation, width, height);
 286
 287                 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_egl_first_cfg[i].glConfigName));
 288
 289                 runs.push_back(params);
 290         }
 291 }
 292
 293 static void getTestRunsForAOSPES(vector<TestRunParams>& runs, const ConfigList& configs, const glu::ApiType apiType)
 294 {
 295 #include "glcAospMustpassEs.hpp"
 296
 297         for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_es_first_cfg); ++i)
 298         {
 299                 if (!glu::contextSupports(glu::ContextType(apiType), aosp_mustpass_es_first_cfg[i].apiType))
 300                         continue;
 301
 302                 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_es_first_cfg[i].glConfigName);
 303
 304                 vector<AOSPConfig>::const_iterator cfgIter;
 305                 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
 306                 {
 307                         // find first compatible config
 308                         if (isGLConfigCompatible(cfgInfo, *cfgIter))
 309                         {
 310                                 break;
 311                         }
 312                 }
 313
 314                 if (cfgIter == configs.aospConfigs.end())
 315                 {
 316                         TCU_FAIL(("No suitable configuration found for GL config " +
 317                                           de::toString(aosp_mustpass_es_first_cfg[i].glConfigName))
 318                                                  .c_str());
 319                         return;
 320                 }
 321
 322                 string  apiName = getApiName(aosp_mustpass_es_first_cfg[i].apiType);
 323                 const int width   = aosp_mustpass_es_first_cfg[i].surfaceWidth;
 324                 const int height  = aosp_mustpass_es_first_cfg[i].surfaceHeight;
 325
 326                 TestRunParams params;
 327                 params.logFilename = getLogFileName(apiName, aosp_mustpass_es_first_cfg[i].configName, 1, i, width, height, -1);
 328                 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_es_first_cfg[i].configName,
 329                                            aosp_mustpass_es_first_cfg[i].screenRotation, width, height);
 330
 331                 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_es_first_cfg[i].glConfigName));
 332
 333                 //set surface type
 334                 if ((*cfgIter).surfaceTypes & SURFACETYPE_WINDOW)
 335                         params.args.push_back("--deqp-surface-type=window");
 336                 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PBUFFER)
 337                         params.args.push_back("--deqp-surface-type=pbuffer");
 338                 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PIXMAP)
 339                         params.args.push_back("--deqp-surface-type=pixmap");
 340                 runs.push_back(params);
 341         }
 342 }
 343
 344 static void getTestRunsForES(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
 345 {
 346         getTestRunsForAOSPEGL(runs, configs);
 347         getTestRunsForAOSPES(runs, configs, type);
 348
 349 #include "glcKhronosMustpassEs.hpp"
 350
 351         for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
 352         {
 353                 const bool isFirst              = cfgIter == configs.configs.begin();
 354                 const int  numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_es_first_cfg) :
 355                                                                                    DE_LENGTH_OF_ARRAY(khronos_mustpass_es_other_cfg);
 356                 const RunParams* runParams = isFirst ? khronos_mustpass_es_first_cfg : khronos_mustpass_es_other_cfg;
 357
 358                 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
 359                 {
 360                         if (!glu::contextSupports(glu::ContextType(type), runParams[runNdx].apiType))
 361                                 continue;
 362
 363                         string  apiName = getApiName(runParams[runNdx].apiType);
 364                         const int width   = runParams[runNdx].surfaceWidth;
 365                         const int height  = runParams[runNdx].surfaceHeight;
 366                         const int seed  = runParams[runNdx].baseSeed;
 367
 368                         TestRunParams params;
 369
 370                         params.logFilename =
 371                                 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed);
 372
 373                         getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName,
 374                                                    runParams[runNdx].screenRotation, width, height);
 375                         params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
 376
 377                         appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
 378
 379                         runs.push_back(params);
 380                 }
 381         }
 382 }
 383
 384 static void getTestRunsForGL(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
 385 {
 386         const char* packageName = DE_NULL;
 387
 388         if (type == glu::ApiType::core(3, 0))
 389                 packageName = "GL30-CTS";
 390         else if (type == glu::ApiType::core(3, 1))
 391                 packageName = "GL31-CTS";
 392         else if (type == glu::ApiType::core(3, 2))
 393                 packageName = "GL32-CTS";
 394         else if (type == glu::ApiType::core(3, 3))
 395                 packageName = "GL33-CTS";
 396         else if (type == glu::ApiType::core(4, 0))
 397                 packageName = "GL40-CTS";
 398         else if (type == glu::ApiType::core(4, 1))
 399                 packageName = "GL41-CTS";
 400         else if (type == glu::ApiType::core(4, 2))
 401                 packageName = "GL42-CTS";
 402         else if (type == glu::ApiType::core(4, 3))
 403                 packageName = "GL43-CTS";
 404         else if (type == glu::ApiType::core(4, 4))
 405                 packageName = "GL44-CTS";
 406         else if (type == glu::ApiType::core(4, 5))
 407                 packageName = "GL45-CTS";
 408         else
 409                 throw std::runtime_error("Unknown context type");
 410
 411         DE_ASSERT(packageName);
 412
 413         struct RunParams
 414         {
 415                 int                     width;
 416                 int                     height;
 417                 int                     baseSeed;
 418                 const char* fboConfig; //!< Fbo config arg, or null if not used.
 419         };
 420
 421         static const RunParams firstConfigParams[] = { { 64, 64, 1, DE_NULL },
 422                                                                                                    { 113, 47, 2, DE_NULL },
 423                                                                                                    { 64, -1, 3, "rgba8888d24s8" },
 424                                                                                                    { -1, 64, 3, "rgba8888d24s8" } };
 425
 426         static const RunParams otherConfigParams[] = { { 64, 64, 1, DE_NULL }, { 113, 47, 2, DE_NULL } };
 427
 428         for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
 429         {
 430                 const bool isFirst = cfgIter == configs.configs.begin();
 431                 const int  numRunParams =
 432                         isFirst ? DE_LENGTH_OF_ARRAY(firstConfigParams) : DE_LENGTH_OF_ARRAY(otherConfigParams);
 433                 const RunParams* runParams = isFirst ? firstConfigParams : otherConfigParams;
 434
 435                 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
 436                 {
 437                         const std::string width  = de::toString(runParams[runNdx].width);
 438                         const std::string height = de::toString(runParams[runNdx].height);
 439                         const std::string seed   = de::toString(runParams[runNdx].baseSeed);
 440
 441                         TestRunParams params;
 442
 443                         params.logFilename = string("config-") + de::toString(cfgIter->id) + "-run-" + de::toString(runNdx) +
 444                                                                  "-width-" + de::toString(width) + "-height-" + de::toString(height) + "-seed-" + seed +
 445                                                                  ".qpa";
 446
 447                         params.args.push_back(string("--deqp-case=") + packageName + ".*");
 448                         params.args.push_back(string("--deqp-surface-width=") + width);
 449                         params.args.push_back(string("--deqp-surface-height=") + height);
 450                         params.args.push_back(string("--deqp-base-seed=") + seed);
 451                         appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
 452
 453                         runs.push_back(params);
 454                 }
 455         }
 456 }
 457
 458 static void getTestRunParams(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
 459 {
 460         switch (type.getProfile())
 461         {
 462         case glu::PROFILE_CORE:
 463                 getTestRunsForGL(type, configs, runs);
 464                 break;
 465         case glu::PROFILE_ES:
 466                 getTestRunsForES(type, configs, runs);
 467                 break;
 468         default:
 469                 throw std::runtime_error("Unknown context type");
 470         }
 471 }
 472
 473 struct FileDeleter
 474 {
 475         void operator()(FILE* file) const
 476         {
 477                 if (file)
 478                         fclose(file);
 479         }
 480 };
 481
 482 struct XmlWriterDeleter
 483 {
 484         void operator()(qpXmlWriter* writer) const
 485         {
 486                 if (writer)
 487                         qpXmlWriter_destroy(writer);
 488         }
 489 };
 490
 491 static const char* getRunTypeName(glu::ApiType type)
 492 {
 493         if (type == glu::ApiType::es(2, 0))
 494                 return "es2";
 495         else if (type == glu::ApiType::es(3, 0))
 496                 return "es3";
 497         else if (type == glu::ApiType::es(3, 1))
 498                 return "es31";
 499         else if (type == glu::ApiType::es(3, 2))
 500                 return "es32";
 501         else if (type == glu::ApiType::core(3, 0))
 502                 return "gl30";
 503         else if (type == glu::ApiType::core(3, 1))
 504                 return "gl31";
 505         else if (type == glu::ApiType::core(3, 2))
 506                 return "gl32";
 507         else if (type == glu::ApiType::core(3, 3))
 508                 return "gl33";
 509         else if (type == glu::ApiType::core(4, 0))
 510                 return "gl40";
 511         else if (type == glu::ApiType::core(4, 1))
 512                 return "gl41";
 513         else if (type == glu::ApiType::core(4, 2))
 514                 return "gl42";
 515         else if (type == glu::ApiType::core(4, 3))
 516                 return "gl43";
 517         else if (type == glu::ApiType::core(4, 4))
 518                 return "gl44";
 519         else if (type == glu::ApiType::core(4, 5))
 520                 return "gl45";
 521         else
 522                 return DE_NULL;
 523 }
 524
 525 #define XML_CHECK(X) \
 526         if (!(X))        \
 527         throw tcu::Exception("Writing XML failed")
 528
 529 static void writeRunSummary(const TestRunSummary& summary, const char* filename)
 530 {
 531         de::UniquePtr<FILE, FileDeleter> out(fopen(filename, "wb"));
 532         if (!out)
 533                 throw tcu::Exception(string("Failed to open ") + filename);
 534
 535         de::UniquePtr<qpXmlWriter, XmlWriterDeleter> writer(qpXmlWriter_createFileWriter(out.get(), DE_FALSE, DE_FALSE));
 536         if (!writer)
 537                 throw std::bad_alloc();
 538
 539         XML_CHECK(qpXmlWriter_startDocument(writer.get()));
 540
 541         {
 542                 qpXmlAttribute attribs[2];
 543
 544                 attribs[0] = qpSetStringAttrib("Type", getRunTypeName(summary.runType));
 545                 attribs[1] = qpSetBoolAttrib("Conformant", summary.isConformant ? DE_TRUE : DE_FALSE);
 546
 547                 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Summary", DE_LENGTH_OF_ARRAY(attribs), attribs));
 548         }
 549
 550         // Config run
 551         {
 552                 qpXmlAttribute attribs[1];
 553                 attribs[0] = qpSetStringAttrib("FileName", summary.configLogFilename.c_str());
 554                 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Configs", DE_LENGTH_OF_ARRAY(attribs), attribs) &&
 555                                   qpXmlWriter_endElement(writer.get(), "Configs"));
 556         }
 557
 558         // Record test run parameters (log filename & command line).
 559         for (vector<TestRunParams>::const_iterator runIter = summary.runParams.begin(); runIter != summary.runParams.end();
 560                  ++runIter)
 561         {
 562                 string             cmdLine;
 563                 qpXmlAttribute attribs[2];
 564
 565                 for (vector<string>::const_iterator argIter = runIter->args.begin(); argIter != runIter->args.end(); ++argIter)
 566                 {
 567                         if (argIter != runIter->args.begin())
 568                                 cmdLine += " ";
 569                         cmdLine += *argIter;
 570                 }
 571
 572                 attribs[0] = qpSetStringAttrib("FileName", runIter->logFilename.c_str());
 573                 attribs[1] = qpSetStringAttrib("CmdLine", cmdLine.c_str());
 574
 575                 XML_CHECK(qpXmlWriter_startElement(writer.get(), "TestRun", DE_LENGTH_OF_ARRAY(attribs), attribs) &&
 576                                   qpXmlWriter_endElement(writer.get(), "TestRun"));
 577         }
 578
 579         XML_CHECK(qpXmlWriter_endElement(writer.get(), "Summary"));
 580         XML_CHECK(qpXmlWriter_endDocument(writer.get()));
 581 }
 582
 583 #undef XML_CHECK
 584
 585 TestRunner::TestRunner(tcu::Platform& platform, tcu::Archive& archive, const char* logDirPath, glu::ApiType type,
 586                                            deUint32 flags)
 587         : m_platform(platform)
 588         , m_archive(archive)
 589         , m_logDirPath(logDirPath)
 590         , m_type(type)
 591         , m_flags(flags)
 592         , m_iterState(ITERATE_INIT)
 593         , m_curSession(DE_NULL)
 594         , m_sessionsExecuted(0)
 595         , m_sessionsPassed(0)
 596         , m_sessionsFailed(0)
 597 {
 598 }
 599
 600 TestRunner::~TestRunner(void)
 601 {
 602         delete m_curSession;
 603 }
 604
 605 bool TestRunner::iterate(void)
 606 {
 607         switch (m_iterState)
 608         {
 609         case ITERATE_INIT:
 610                 init();
 611                 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
 612                 return true;
 613
 614         case ITERATE_DEINIT:
 615                 deinit();
 616                 m_iterState = ITERATE_INIT;
 617                 return false;
 618
 619         case ITERATE_INIT_SESSION:
 620                 DE_ASSERT(m_sessionIter != m_runSessions.end());
 621                 initSession(*m_sessionIter);
 622                 if (m_flags & PRINT_SUMMARY)
 623                         m_iterState = ITERATE_DEINIT_SESSION;
 624                 else
 625                         m_iterState = ITERATE_ITERATE_SESSION;
 626                 return true;
 627
 628         case ITERATE_DEINIT_SESSION:
 629                 deinitSession();
 630                 ++m_sessionIter;
 631                 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
 632                 return true;
 633
 634         case ITERATE_ITERATE_SESSION:
 635                 if (!iterateSession())
 636                         m_iterState = ITERATE_DEINIT_SESSION;
 637                 return true;
 638
 639         default:
 640                 DE_ASSERT(false);
 641                 return false;
 642         }
 643 }
 644
 645 void TestRunner::init(void)
 646 {
 647         DE_ASSERT(m_runSessions.empty() && m_summary.runParams.empty());
 648
 649         tcu::print("Running %s conformance\n", glu::getApiTypeDescription(m_type));
 650
 651         m_summary.runType = m_type;
 652
 653         // Get list of configs to test.
 654         ConfigList configList;
 655         getDefaultConfigList(m_platform, m_type, configList);
 656
 657         tcu::print("  found %d compatible and %d excluded configs\n", (int)configList.configs.size(),
 658                            (int)configList.excludedConfigs.size());
 659
 660         // Config list run.
 661         {
 662                 const char*   configLogFilename = "configs.qpa";
 663                 TestRunParams configRun;
 664
 665                 configRun.logFilename = configLogFilename;
 666                 configRun.args.push_back("--deqp-case=CTS-Configs.*");
 667                 m_runSessions.push_back(configRun);
 668
 669                 m_summary.configLogFilename = configLogFilename;
 670         }
 671
 672         // Conformance test type specific runs
 673         getTestRunParams(m_type, configList, m_runSessions);
 674
 675         // Record run params for summary.
 676         for (std::vector<TestRunParams>::const_iterator runIter = m_runSessions.begin() + 1; runIter != m_runSessions.end();
 677                  ++runIter)
 678                 m_summary.runParams.push_back(*runIter);
 679
 680         // Session iterator
 681         m_sessionIter = m_runSessions.begin();
 682 }
 683
 684 void TestRunner::deinit(void)
 685 {
 686         // Print out totals.
 687         bool isConformant = m_sessionsExecuted == m_sessionsPassed;
 688         DE_ASSERT(m_sessionsExecuted == m_sessionsPassed + m_sessionsFailed);
 689         tcu::print("\n%d/%d sessions passed, conformance test %s\n", m_sessionsPassed, m_sessionsExecuted,
 690                            isConformant ? "PASSED" : "FAILED");
 691
 692         m_summary.isConformant = isConformant;
 693
 694         // Write out summary.
 695         writeRunSummary(m_summary, de::FilePath::join(m_logDirPath, "cts-run-summary.xml").getPath());
 696
 697         m_runSessions.clear();
 698         m_summary.clear();
 699 }
 700
 701 void TestRunner::initSession(const TestRunParams& runParams)
 702 {
 703         DE_ASSERT(!m_curSession);
 704
 705         tcu::print("\n  Test run %d / %d\n", (int)(m_sessionIter - m_runSessions.begin() + 1), (int)m_runSessions.size());
 706
 707         // Compute final args for run.
 708         vector<string> args(runParams.args);
 709         args.push_back(string("--deqp-log-filename=") + de::FilePath::join(m_logDirPath, runParams.logFilename).getPath());
 710
 711         if (!(m_flags & VERBOSE_IMAGES))
 712                 args.push_back("--deqp-log-images=disable");
 713
 714         if (!(m_flags & VERBOSE_SHADERS))
 715                 args.push_back("--deqp-log-shader-sources=disable");
 716
 717         std::ostringstream                        ostr;
 718         std::ostream_iterator<string> out_it(ostr, ", ");
 719         std::copy(args.begin(), args.end(), out_it);
 720         tcu::print("\n  Config: %s \n\n", ostr.str().c_str());
 721
 722         // Translate to argc, argv
 723         vector<const char*> argv;
 724         argv.push_back("cts-runner"); // Dummy binary name
 725         for (vector<string>::const_iterator i = args.begin(); i != args.end(); i++)
 726                 argv.push_back(i->c_str());
 727
 728         // Create session
 729         m_curSession = new RunSession(m_platform, m_archive, (int)argv.size(), &argv[0]);
 730 }
 731
 732 void TestRunner::deinitSession(void)
 733 {
 734         DE_ASSERT(m_curSession);
 735
 736         // Collect results.
 737         // \note NotSupported is treated as pass.
 738         const tcu::TestRunStatus& result = m_curSession->getResult();
 739         bool                                      isOk =
 740                 result.numExecuted == (result.numPassed + result.numNotSupported + result.numWarnings) && result.isComplete;
 741
 742         DE_ASSERT(result.numExecuted == result.numPassed + result.numFailed + result.numNotSupported + result.numWarnings);
 743
 744         m_sessionsExecuted += 1;
 745         (isOk ? m_sessionsPassed : m_sessionsFailed) += 1;
 746
 747         delete m_curSession;
 748         m_curSession = DE_NULL;
 749 }
 750
 751 inline bool TestRunner::iterateSession(void)
 752 {
 753         return m_curSession->iterate();
 754 }
 755
 756 } // glcts