external/openglcts/modules/runner/glcTestRunner.cpp

   1 /*-------------------------------------------------------------------------
   2  * OpenGL Conformance Test Suite
   3  * -----------------------------
   4  *
   5  * Copyright (c) 2016 Google Inc.
   6  * Copyright (c) 2016 The Khronos Group Inc.
   7  *
   8  * Licensed under the Apache License, Version 2.0 (the "License");
   9  * you may not use this file except in compliance with the License.
  10  * You may obtain a copy of the License at
  11  *
  12  *      http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing, software
  15  * distributed under the License is distributed on an "AS IS" BASIS,
  16  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17  * See the License for the specific language governing permissions and
  18  * limitations under the License.
  19  *
  20  */ /*!
  21  * \file
  22  * \brief CTS runner.
  23  */ /*-------------------------------------------------------------------*/
  24
  25 #include "glcTestRunner.hpp"
  26 #include "deFilePath.hpp"
  27 #include "deStringUtil.hpp"
  28 #include "deUniquePtr.hpp"
  29 #include "glcConfigList.hpp"
  30 #include "qpXmlWriter.h"
  31 #include "tcuApp.hpp"
  32 #include "tcuCommandLine.hpp"
  33 #include "tcuTestLog.hpp"
  34 #include "tcuTestSessionExecutor.hpp"
  35
  36 #include <iterator>
  37
  38 namespace glcts
  39 {
  40
  41 using std::vector;
  42 using std::string;
  43
  44 // RunSession
  45
  46 class RunSession
  47 {
  48 public:
  49         RunSession(tcu::Platform& platform, tcu::Archive& archive, const int numArgs, const char* const* args)
  50                 : m_cmdLine(numArgs, args)
  51                 , m_log(m_cmdLine.getLogFileName(), m_cmdLine.getLogFlags())
  52                 , m_app(platform, archive, m_log, m_cmdLine)
  53         {
  54         }
  55
  56         inline bool iterate(void)
  57         {
  58                 return m_app.iterate();
  59         }
  60
  61         inline const tcu::TestRunStatus& getResult(void) const
  62         {
  63                 return m_app.getResult();
  64         }
  65
  66 private:
  67         tcu::CommandLine m_cmdLine;
  68         tcu::TestLog     m_log;
  69         tcu::App                 m_app;
  70 };
  71
  72 static void appendConfigArgs(const Config& config, std::vector<std::string>& args, const char* fboConfig)
  73 {
  74         if (fboConfig != NULL)
  75         {
  76                 args.push_back(string("--deqp-gl-config-name=") + fboConfig);
  77                 args.push_back("--deqp-surface-type=fbo");
  78         }
  79
  80         if (config.type != CONFIGTYPE_DEFAULT)
  81         {
  82                 // \todo [2013-05-06 pyry] Test all surface types for some configs?
  83                 if (fboConfig == NULL)
  84                 {
  85                         if (config.surfaceTypes & SURFACETYPE_WINDOW)
  86                                 args.push_back("--deqp-surface-type=window");
  87                         else if (config.surfaceTypes & SURFACETYPE_PBUFFER)
  88                                 args.push_back("--deqp-surface-type=pbuffer");
  89                         else if (config.surfaceTypes & SURFACETYPE_PIXMAP)
  90                                 args.push_back("--deqp-surface-type=pixmap");
  91                 }
  92
  93                 args.push_back(string("--deqp-gl-config-id=") + de::toString(config.id));
  94
  95                 if (config.type == CONFIGTYPE_EGL)
  96                         args.push_back("--deqp-gl-context-type=egl");
  97                 else if (config.type == CONFIGTYPE_WGL)
  98                         args.push_back("--deqp-gl-context-type=wgl");
  99         }
 100 }
 101
 102 typedef struct configInfo
 103 {
 104         deInt32 redBits;
 105         deInt32 greenBits;
 106         deInt32 blueBits;
 107         deInt32 alphaBits;
 108         deInt32 depthBits;
 109         deInt32 stencilBits;
 110         deInt32 samples;
 111 } configInfo;
 112
 113 static configInfo parseConfigBitsFromName(const char* configName)
 114 {
 115         configInfo cfgInfo;
 116         static const struct
 117         {
 118                 const char* name;
 119                 int                     redBits;
 120                 int                     greenBits;
 121                 int                     blueBits;
 122                 int                     alphaBits;
 123         } colorCfgs[] = {
 124                 { "rgba8888", 8, 8, 8, 8 }, { "rgb565", 5, 6, 5, 0 },
 125         };
 126         for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(colorCfgs); ndx++)
 127         {
 128                 if (!strncmp(configName, colorCfgs[ndx].name, strlen(colorCfgs[ndx].name)))
 129                 {
 130                         cfgInfo.redBits   = colorCfgs[ndx].redBits;
 131                         cfgInfo.greenBits = colorCfgs[ndx].greenBits;
 132                         cfgInfo.blueBits  = colorCfgs[ndx].blueBits;
 133                         cfgInfo.alphaBits = colorCfgs[ndx].alphaBits;
 134
 135                         configName += strlen(colorCfgs[ndx].name);
 136                         break;
 137                 }
 138         }
 139
 140         static const struct
 141         {
 142                 const char* name;
 143                 int                     depthBits;
 144         } depthCfgs[] = {
 145                 { "d0", 0 }, { "d24", 24 },
 146         };
 147         for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(depthCfgs); ndx++)
 148         {
 149                 if (!strncmp(configName, depthCfgs[ndx].name, strlen(depthCfgs[ndx].name)))
 150                 {
 151                         cfgInfo.depthBits = depthCfgs[ndx].depthBits;
 152
 153                         configName += strlen(depthCfgs[ndx].name);
 154                         break;
 155                 }
 156         }
 157
 158         static const struct
 159         {
 160                 const char* name;
 161                 int                     stencilBits;
 162         } stencilCfgs[] = {
 163                 { "s0", 0 }, { "s8", 8 },
 164         };
 165         for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(stencilCfgs); ndx++)
 166         {
 167                 if (!strncmp(configName, stencilCfgs[ndx].name, strlen(stencilCfgs[ndx].name)))
 168                 {
 169                         cfgInfo.stencilBits = stencilCfgs[ndx].stencilBits;
 170
 171                         configName += strlen(stencilCfgs[ndx].name);
 172                         break;
 173                 }
 174         }
 175
 176         static const struct
 177         {
 178                 const char* name;
 179                 int                     samples;
 180         } multiSampleCfgs[] = {
 181                 { "ms0", 0 }, { "ms4", 4 },
 182         };
 183         for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(multiSampleCfgs); ndx++)
 184         {
 185                 if (!strncmp(configName, multiSampleCfgs[ndx].name, strlen(multiSampleCfgs[ndx].name)))
 186                 {
 187                         cfgInfo.samples = multiSampleCfgs[ndx].samples;
 188
 189                         configName += strlen(multiSampleCfgs[ndx].name);
 190                         break;
 191                 }
 192         }
 193
 194         return cfgInfo;
 195 }
 196
 197 static const char* getApiName(glu::ApiType apiType)
 198 {
 199         if (apiType == glu::ApiType::es(2, 0))
 200                 return "gles2";
 201         else if (apiType == glu::ApiType::es(3, 0))
 202                 return "gles3";
 203         else if (apiType == glu::ApiType::es(3, 1))
 204                 return "gles31";
 205         else if (apiType == glu::ApiType::es(3, 2))
 206                 return "gles32";
 207         else if (apiType == glu::ApiType::core(3, 0))
 208                 return "gl30";
 209         else if (apiType == glu::ApiType::core(3, 1))
 210                 return "gl31";
 211         else if (apiType == glu::ApiType::core(3, 2))
 212                 return "gl32";
 213         else if (apiType == glu::ApiType::core(3, 3))
 214                 return "gl33";
 215         else if (apiType == glu::ApiType::core(4, 0))
 216                 return "gl40";
 217         else if (apiType == glu::ApiType::core(4, 1))
 218                 return "gl41";
 219         else if (apiType == glu::ApiType::core(4, 2))
 220                 return "gl42";
 221         else if (apiType == glu::ApiType::core(4, 3))
 222                 return "gl43";
 223         else if (apiType == glu::ApiType::core(4, 4))
 224                 return "gl44";
 225         else if (apiType == glu::ApiType::core(4, 5))
 226                 return "gl45";
 227         else if (apiType == glu::ApiType::core(4, 6))
 228                 return "gl46";
 229         else
 230                 throw std::runtime_error("Unknown context type");
 231 }
 232
 233 static const string getCaseListFileOption(const char* mustpassDir, const char* apiName, const char* mustpassName)
 234 {
 235 #if DE_OS == DE_OS_ANDROID
 236         const string case_list_option = "--deqp-caselist-resource=";
 237 #else
 238         const string case_list_option = "--deqp-caselist-file=";
 239 #endif
 240         return case_list_option + mustpassDir + apiName + "-" + mustpassName + ".txt";
 241 }
 242
 243 static const string getLogFileName(const char* apiName, const char* configName, const int iterId, const int runId,
 244                                                                    const int width, const int height, const int seed)
 245 {
 246         string res = string("config-") + apiName + "-" + configName + "-cfg-" + de::toString(iterId) + "-run-" +
 247                                  de::toString(runId) + "-width-" + de::toString(width) + "-height-" + de::toString(height);
 248         if (seed != -1)
 249         {
 250                 res += "-seed-" + de::toString(seed);
 251         }
 252         res += ".qpa";
 253
 254         return res;
 255 }
 256
 257 static void getBaseOptions(std::vector<std::string>& args, const char* mustpassDir, const char* apiName,
 258                                                    const char* configName, const char* screenRotation, int width, int height)
 259 {
 260         args.push_back(getCaseListFileOption(mustpassDir, apiName, configName));
 261         args.push_back(string("--deqp-screen-rotation=") + screenRotation);
 262         args.push_back(string("--deqp-surface-width=") + de::toString(width));
 263         args.push_back(string("--deqp-surface-height=") + de::toString(height));
 264         args.push_back("--deqp-watchdog=disable");
 265 }
 266
 267 static bool isGLConfigCompatible(configInfo cfgInfo, const AOSPConfig& config)
 268 {
 269         return cfgInfo.redBits == config.redBits && cfgInfo.greenBits == config.greenBits &&
 270                    cfgInfo.blueBits == config.blueBits && cfgInfo.alphaBits == config.alphaBits &&
 271                    cfgInfo.depthBits == config.depthBits && cfgInfo.stencilBits == config.stencilBits &&
 272                    cfgInfo.samples == config.samples;
 273 }
 274
 275 static void getTestRunsForAOSPEGL(vector<TestRunParams>& runs, const ConfigList& configs)
 276 {
 277 #include "glcAospMustpassEgl.hpp"
 278
 279         for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_egl_first_cfg); ++i)
 280         {
 281                 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_egl_first_cfg[i].glConfigName);
 282
 283                 vector<AOSPConfig>::const_iterator cfgIter;
 284                 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
 285                 {
 286                         // find first compatible config
 287                         if ((*cfgIter).type == CONFIGTYPE_EGL && isGLConfigCompatible(cfgInfo, *cfgIter))
 288                         {
 289                                 break;
 290                         }
 291                 }
 292
 293                 if (cfgIter == configs.aospConfigs.end())
 294                 {
 295                         // No suitable configuration found. Skipping EGL tests
 296                         continue;
 297                 }
 298
 299                 const char* apiName = "egl";
 300
 301                 const int width   = aosp_mustpass_egl_first_cfg[i].surfaceWidth;
 302                 const int height  = aosp_mustpass_egl_first_cfg[i].surfaceHeight;
 303
 304                 TestRunParams params;
 305                 params.logFilename =
 306                         getLogFileName(apiName, aosp_mustpass_egl_first_cfg[i].configName, 1, i, width, height, -1);
 307                 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_egl_first_cfg[i].configName,
 308                                            aosp_mustpass_egl_first_cfg[i].screenRotation, width, height);
 309
 310                 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_egl_first_cfg[i].glConfigName));
 311
 312                 runs.push_back(params);
 313         }
 314 }
 315
 316 static void getTestRunsForAOSPES(vector<TestRunParams>& runs, const ConfigList& configs, const glu::ApiType apiType)
 317 {
 318 #include "glcAospMustpassEs.hpp"
 319
 320         for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_es_first_cfg); ++i)
 321         {
 322                 if (!glu::contextSupports(glu::ContextType(apiType), aosp_mustpass_es_first_cfg[i].apiType))
 323                         continue;
 324
 325                 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_es_first_cfg[i].glConfigName);
 326
 327                 vector<AOSPConfig>::const_iterator cfgIter;
 328                 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
 329                 {
 330                         // find first compatible config
 331                         if (isGLConfigCompatible(cfgInfo, *cfgIter))
 332                         {
 333                                 break;
 334                         }
 335                 }
 336
 337                 if (cfgIter == configs.aospConfigs.end())
 338                 {
 339                         TCU_FAIL(("No suitable configuration found for GL config " +
 340                                           de::toString(aosp_mustpass_es_first_cfg[i].glConfigName))
 341                                                  .c_str());
 342                         return;
 343                 }
 344
 345                 const char* apiName = getApiName(aosp_mustpass_es_first_cfg[i].apiType);
 346
 347                 const int width   = aosp_mustpass_es_first_cfg[i].surfaceWidth;
 348                 const int height  = aosp_mustpass_es_first_cfg[i].surfaceHeight;
 349
 350                 TestRunParams params;
 351                 params.logFilename = getLogFileName(apiName, aosp_mustpass_es_first_cfg[i].configName, 1, i, width, height, -1);
 352                 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_es_first_cfg[i].configName,
 353                                            aosp_mustpass_es_first_cfg[i].screenRotation, width, height);
 354
 355                 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_es_first_cfg[i].glConfigName));
 356
 357                 //set surface type
 358                 if ((*cfgIter).surfaceTypes & SURFACETYPE_WINDOW)
 359                         params.args.push_back("--deqp-surface-type=window");
 360                 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PBUFFER)
 361                         params.args.push_back("--deqp-surface-type=pbuffer");
 362                 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PIXMAP)
 363                         params.args.push_back("--deqp-surface-type=pixmap");
 364                 runs.push_back(params);
 365         }
 366 }
 367
 368 static void getTestRunsForNoContext(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs, const RunParams* runParams,
 369                                                                         const int numRunParams, const char* mustpassDir)
 370 {
 371         vector<Config>::const_iterator cfgIter = configs.configs.begin();
 372
 373         for (int i = 0; i < numRunParams; ++i)
 374         {
 375                 if (!glu::contextSupports(glu::ContextType(type), runParams[i].apiType))
 376                                 continue;
 377
 378                 const char* apiName = getApiName(runParams[i].apiType);
 379
 380                 const int width  = runParams[i].surfaceWidth;
 381                 const int height = runParams[i].surfaceHeight;
 382                 const int seed   = runParams[i].baseSeed;
 383
 384                 TestRunParams params;
 385                 params.logFilename = getLogFileName(apiName, runParams[i].configName, 1, i, width, height, seed);
 386
 387                 getBaseOptions(params.args, mustpassDir, apiName, runParams[i].configName, runParams[i].screenRotation, width,
 388                                            height);
 389
 390                 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
 391
 392                 appendConfigArgs(*cfgIter, params.args, runParams[i].fboConfig);
 393
 394                 runs.push_back(params);
 395         }
 396 }
 397
 398 static void getTestRunsForNoContextES(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
 399 {
 400 #include "glcKhronosMustpassEsNocontext.hpp"
 401         getTestRunsForNoContext(type, runs, configs, khronos_mustpass_es_nocontext_first_cfg,
 402                                                         DE_LENGTH_OF_ARRAY(khronos_mustpass_es_nocontext_first_cfg), mustpassDir);
 403 }
 404
 405 static void getTestRunsForES(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
 406 {
 407         getTestRunsForAOSPEGL(runs, configs);
 408         getTestRunsForAOSPES(runs, configs, type);
 409         getTestRunsForNoContextES(type, runs, configs);
 410
 411 #include "glcKhronosMustpassEs.hpp"
 412
 413         for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
 414         {
 415                 const bool isFirst              = cfgIter == configs.configs.begin();
 416                 const int  numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_es_first_cfg) :
 417                                                                                    DE_LENGTH_OF_ARRAY(khronos_mustpass_es_other_cfg);
 418                 const RunParams* runParams = isFirst ? khronos_mustpass_es_first_cfg : khronos_mustpass_es_other_cfg;
 419
 420                 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
 421                 {
 422                         if (!glu::contextSupports(glu::ContextType(type), runParams[runNdx].apiType))
 423                                 continue;
 424
 425                         const char* apiName = getApiName(runParams[runNdx].apiType);
 426
 427                         const int width   = runParams[runNdx].surfaceWidth;
 428                         const int height  = runParams[runNdx].surfaceHeight;
 429                         const int seed  = runParams[runNdx].baseSeed;
 430
 431                         TestRunParams params;
 432
 433                         params.logFilename =
 434                                 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed);
 435
 436                         getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName,
 437                                                    runParams[runNdx].screenRotation, width, height);
 438                         params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
 439
 440                         appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
 441
 442                         runs.push_back(params);
 443                 }
 444         }
 445 }
 446
 447 static void getTestRunsForNoContextGL(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
 448 {
 449 #include "glcKhronosMustpassGlNocontext.hpp"
 450         getTestRunsForNoContext(type, runs, configs, khronos_mustpass_gl_nocontext_first_cfg,
 451                                                         DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_nocontext_first_cfg), mustpassDir);
 452 }
 453
 454 static void getTestRunsForGL(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
 455 {
 456         getTestRunsForNoContextGL(type, runs, configs);
 457 #include "glcKhronosMustpassGl.hpp"
 458
 459         for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
 460         {
 461                 const bool isFirst              = cfgIter == configs.configs.begin();
 462                 const int  numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_first_cfg) :
 463                                                                                    DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_other_cfg);
 464                 const RunParams* runParams = isFirst ? khronos_mustpass_gl_first_cfg : khronos_mustpass_gl_other_cfg;
 465
 466                 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
 467                 {
 468                         if (type != runParams[runNdx].apiType)
 469                                 continue;
 470
 471                         const char* apiName = getApiName(runParams[runNdx].apiType);
 472
 473                         const int width   = runParams[runNdx].surfaceWidth;
 474                         const int height  = runParams[runNdx].surfaceHeight;
 475                         const int seed  = runParams[runNdx].baseSeed;
 476
 477                         TestRunParams params;
 478
 479                         params.logFilename =
 480                                 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed);
 481
 482                         getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName,
 483                                                    runParams[runNdx].screenRotation, width, height);
 484                         params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
 485
 486                         appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
 487
 488                         runs.push_back(params);
 489                 }
 490         }
 491 }
 492
 493 static void getTestRunParams(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
 494 {
 495         switch (type.getProfile())
 496         {
 497         case glu::PROFILE_CORE:
 498                 getTestRunsForGL(type, configs, runs);
 499                 break;
 500         case glu::PROFILE_ES:
 501                 getTestRunsForES(type, configs, runs);
 502                 break;
 503         default:
 504                 throw std::runtime_error("Unknown context type");
 505         }
 506 }
 507
 508 struct FileDeleter
 509 {
 510         void operator()(FILE* file) const
 511         {
 512                 if (file)
 513                         fclose(file);
 514         }
 515 };
 516
 517 struct XmlWriterDeleter
 518 {
 519         void operator()(qpXmlWriter* writer) const
 520         {
 521                 if (writer)
 522                         qpXmlWriter_destroy(writer);
 523         }
 524 };
 525
 526 static const char* getRunTypeName(glu::ApiType type)
 527 {
 528         if (type == glu::ApiType::es(2, 0))
 529                 return "es2";
 530         else if (type == glu::ApiType::es(3, 0))
 531                 return "es3";
 532         else if (type == glu::ApiType::es(3, 1))
 533                 return "es31";
 534         else if (type == glu::ApiType::es(3, 2))
 535                 return "es32";
 536         else if (type == glu::ApiType::core(3, 0))
 537                 return "gl30";
 538         else if (type == glu::ApiType::core(3, 1))
 539                 return "gl31";
 540         else if (type == glu::ApiType::core(3, 2))
 541                 return "gl32";
 542         else if (type == glu::ApiType::core(3, 3))
 543                 return "gl33";
 544         else if (type == glu::ApiType::core(4, 0))
 545                 return "gl40";
 546         else if (type == glu::ApiType::core(4, 1))
 547                 return "gl41";
 548         else if (type == glu::ApiType::core(4, 2))
 549                 return "gl42";
 550         else if (type == glu::ApiType::core(4, 3))
 551                 return "gl43";
 552         else if (type == glu::ApiType::core(4, 4))
 553                 return "gl44";
 554         else if (type == glu::ApiType::core(4, 5))
 555                 return "gl45";
 556         else if (type == glu::ApiType::core(4, 6))
 557                 return "gl46";
 558         else
 559                 return DE_NULL;
 560 }
 561
 562 #define XML_CHECK(X) \
 563         if (!(X))        \
 564         throw tcu::Exception("Writing XML failed")
 565
 566 static void writeRunSummary(const TestRunSummary& summary, const char* filename)
 567 {
 568         de::UniquePtr<FILE, FileDeleter> out(fopen(filename, "wb"));
 569         if (!out)
 570                 throw tcu::Exception(string("Failed to open ") + filename);
 571
 572         de::UniquePtr<qpXmlWriter, XmlWriterDeleter> writer(qpXmlWriter_createFileWriter(out.get(), DE_FALSE, DE_FALSE));
 573         if (!writer)
 574                 throw std::bad_alloc();
 575
 576         XML_CHECK(qpXmlWriter_startDocument(writer.get()));
 577
 578         {
 579                 qpXmlAttribute attribs[2];
 580
 581                 attribs[0] = qpSetStringAttrib("Type", getRunTypeName(summary.runType));
 582                 attribs[1] = qpSetBoolAttrib("Conformant", summary.isConformant ? DE_TRUE : DE_FALSE);
 583
 584                 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Summary", DE_LENGTH_OF_ARRAY(attribs), attribs));
 585         }
 586
 587         // Config run
 588         {
 589                 qpXmlAttribute attribs[1];
 590                 attribs[0] = qpSetStringAttrib("FileName", summary.configLogFilename.c_str());
 591                 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Configs", DE_LENGTH_OF_ARRAY(attribs), attribs) &&
 592                                   qpXmlWriter_endElement(writer.get(), "Configs"));
 593         }
 594
 595         // Record test run parameters (log filename & command line).
 596         for (vector<TestRunParams>::const_iterator runIter = summary.runParams.begin(); runIter != summary.runParams.end();
 597                  ++runIter)
 598         {
 599                 string             cmdLine;
 600                 qpXmlAttribute attribs[2];
 601
 602                 for (vector<string>::const_iterator argIter = runIter->args.begin(); argIter != runIter->args.end(); ++argIter)
 603                 {
 604                         if (argIter != runIter->args.begin())
 605                                 cmdLine += " ";
 606                         cmdLine += *argIter;
 607                 }
 608
 609                 attribs[0] = qpSetStringAttrib("FileName", runIter->logFilename.c_str());
 610                 attribs[1] = qpSetStringAttrib("CmdLine", cmdLine.c_str());
 611
 612                 XML_CHECK(qpXmlWriter_startElement(writer.get(), "TestRun", DE_LENGTH_OF_ARRAY(attribs), attribs) &&
 613                                   qpXmlWriter_endElement(writer.get(), "TestRun"));
 614         }
 615
 616         XML_CHECK(qpXmlWriter_endElement(writer.get(), "Summary"));
 617         XML_CHECK(qpXmlWriter_endDocument(writer.get()));
 618 }
 619
 620 #undef XML_CHECK
 621
 622 TestRunner::TestRunner(tcu::Platform& platform, tcu::Archive& archive, const char* logDirPath, glu::ApiType type,
 623                                            deUint32 flags)
 624         : m_platform(platform)
 625         , m_archive(archive)
 626         , m_logDirPath(logDirPath)
 627         , m_type(type)
 628         , m_flags(flags)
 629         , m_iterState(ITERATE_INIT)
 630         , m_curSession(DE_NULL)
 631         , m_sessionsExecuted(0)
 632         , m_sessionsPassed(0)
 633         , m_sessionsFailed(0)
 634 {
 635 }
 636
 637 TestRunner::~TestRunner(void)
 638 {
 639         delete m_curSession;
 640 }
 641
 642 bool TestRunner::iterate(void)
 643 {
 644         switch (m_iterState)
 645         {
 646         case ITERATE_INIT:
 647                 init();
 648                 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
 649                 return true;
 650
 651         case ITERATE_DEINIT:
 652                 deinit();
 653                 m_iterState = ITERATE_INIT;
 654                 return false;
 655
 656         case ITERATE_INIT_SESSION:
 657                 DE_ASSERT(m_sessionIter != m_runSessions.end());
 658                 initSession(*m_sessionIter);
 659                 if (m_flags & PRINT_SUMMARY)
 660                         m_iterState = ITERATE_DEINIT_SESSION;
 661                 else
 662                         m_iterState = ITERATE_ITERATE_SESSION;
 663                 return true;
 664
 665         case ITERATE_DEINIT_SESSION:
 666                 deinitSession();
 667                 ++m_sessionIter;
 668                 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
 669                 return true;
 670
 671         case ITERATE_ITERATE_SESSION:
 672                 if (!iterateSession())
 673                         m_iterState = ITERATE_DEINIT_SESSION;
 674                 return true;
 675
 676         default:
 677                 DE_ASSERT(false);
 678                 return false;
 679         }
 680 }
 681
 682 void TestRunner::init(void)
 683 {
 684         DE_ASSERT(m_runSessions.empty() && m_summary.runParams.empty());
 685
 686         tcu::print("Running %s conformance\n", glu::getApiTypeDescription(m_type));
 687
 688         m_summary.runType = m_type;
 689
 690         // Get list of configs to test.
 691         ConfigList configList;
 692         getDefaultConfigList(m_platform, m_type, configList);
 693
 694         tcu::print("  found %d compatible and %d excluded configs\n", (int)configList.configs.size(),
 695                            (int)configList.excludedConfigs.size());
 696
 697         // Config list run.
 698         {
 699                 const char*   configLogFilename = "configs.qpa";
 700                 TestRunParams configRun;
 701
 702                 configRun.logFilename = configLogFilename;
 703                 configRun.args.push_back("--deqp-case=CTS-Configs.*");
 704                 m_runSessions.push_back(configRun);
 705
 706                 m_summary.configLogFilename = configLogFilename;
 707         }
 708
 709         // Conformance test type specific runs
 710         getTestRunParams(m_type, configList, m_runSessions);
 711
 712         // Record run params for summary.
 713         for (std::vector<TestRunParams>::const_iterator runIter = m_runSessions.begin() + 1; runIter != m_runSessions.end();
 714                  ++runIter)
 715                 m_summary.runParams.push_back(*runIter);
 716
 717         // Session iterator
 718         m_sessionIter = m_runSessions.begin();
 719 }
 720
 721 void TestRunner::deinit(void)
 722 {
 723         // Print out totals.
 724         bool isConformant = m_sessionsExecuted == m_sessionsPassed;
 725         DE_ASSERT(m_sessionsExecuted == m_sessionsPassed + m_sessionsFailed);
 726         tcu::print("\n%d/%d sessions passed, conformance test %s\n", m_sessionsPassed, m_sessionsExecuted,
 727                            isConformant ? "PASSED" : "FAILED");
 728
 729         m_summary.isConformant = isConformant;
 730
 731         // Write out summary.
 732         writeRunSummary(m_summary, de::FilePath::join(m_logDirPath, "cts-run-summary.xml").getPath());
 733
 734         m_runSessions.clear();
 735         m_summary.clear();
 736 }
 737
 738 void TestRunner::initSession(const TestRunParams& runParams)
 739 {
 740         DE_ASSERT(!m_curSession);
 741
 742         tcu::print("\n  Test run %d / %d\n", (int)(m_sessionIter - m_runSessions.begin() + 1), (int)m_runSessions.size());
 743
 744         // Compute final args for run.
 745         vector<string> args(runParams.args);
 746         args.push_back(string("--deqp-log-filename=") + de::FilePath::join(m_logDirPath, runParams.logFilename).getPath());
 747
 748         if (!(m_flags & VERBOSE_IMAGES))
 749                 args.push_back("--deqp-log-images=disable");
 750
 751         if (!(m_flags & VERBOSE_SHADERS))
 752                 args.push_back("--deqp-log-shader-sources=disable");
 753
 754         std::ostringstream                        ostr;
 755         std::ostream_iterator<string> out_it(ostr, ", ");
 756         std::copy(args.begin(), args.end(), out_it);
 757         tcu::print("\n  Config: %s \n\n", ostr.str().c_str());
 758
 759         // Translate to argc, argv
 760         vector<const char*> argv;
 761         argv.push_back("cts-runner"); // Dummy binary name
 762         for (vector<string>::const_iterator i = args.begin(); i != args.end(); i++)
 763                 argv.push_back(i->c_str());
 764
 765         // Create session
 766         m_curSession = new RunSession(m_platform, m_archive, (int)argv.size(), &argv[0]);
 767 }
 768
 769 void TestRunner::deinitSession(void)
 770 {
 771         DE_ASSERT(m_curSession);
 772
 773         // Collect results.
 774         // \note NotSupported is treated as pass.
 775         const tcu::TestRunStatus& result = m_curSession->getResult();
 776         bool                                      isOk =
 777                 result.numExecuted == (result.numPassed + result.numNotSupported + result.numWarnings) && result.isComplete;
 778
 779         DE_ASSERT(result.numExecuted == result.numPassed + result.numFailed + result.numNotSupported + result.numWarnings);
 780
 781         m_sessionsExecuted += 1;
 782         (isOk ? m_sessionsPassed : m_sessionsFailed) += 1;
 783
 784         delete m_curSession;
 785         m_curSession = DE_NULL;
 786 }
 787
 788 inline bool TestRunner::iterateSession(void)
 789 {
 790         return m_curSession->iterate();
 791 }
 792
 793 } // glcts