framework/common/tcuTestExecutor.cpp

   1 /*-------------------------------------------------------------------------
   2  * drawElements Quality Program Tester Core
   3  * ----------------------------------------
   4  *
   5  * Copyright 2014 The Android Open Source Project
   6  *
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  *
  19  *//*!
  20  * \file
  21  * \brief Class for executing tests.
  22  *//*--------------------------------------------------------------------*/
  23
  24 #include "tcuTestExecutor.hpp"
  25 #include "tcuCommandLine.hpp"
  26 #include "tcuPlatform.hpp"
  27 #include "tcuTestLog.hpp"
  28
  29 #include "deInt32.h"
  30
  31 #include <typeinfo>
  32
  33 #include <stdio.h>
  34 #include <stdlib.h>
  35 #include <string.h>
  36
  37 using std::string;
  38 using std::vector;
  39
  40 namespace tcu
  41 {
  42
  43 TestExecutor::TestExecutor (TestContext& testCtx, const CommandLine& cmdLine)
  44         : m_testCtx                             (testCtx)
  45         , m_cmdLine                             (cmdLine)
  46         , m_rootNode                    (DE_NULL)
  47         , m_testCaseWrapper             (DE_NULL)
  48         , m_testCaseListFile    (DE_NULL)
  49         , m_testCaseListWriter  (DE_NULL)
  50 {
  51         m_abortSession  = false;
  52         m_isInTestCase  = false;
  53
  54         // Create the root node.
  55         TestPackageRegistry*                                            packageRegistry = TestPackageRegistry::getSingleton();
  56         vector<TestPackageRegistry::PackageInfo*>       packageInfos    = packageRegistry->getPackageInfos();
  57         vector<TestNode*>                                                       testPackages;
  58
  59         for (int i = 0; i < (int)packageInfos.size(); i++)
  60                 testPackages.push_back(packageInfos[i]->createFunc(testCtx));
  61
  62         m_rootNode = new TestPackageRoot(testCtx, testPackages);
  63
  64         // Init traverse stack.
  65         NodeIter iter(m_rootNode);
  66         m_sessionStack.push_back(iter);
  67 }
  68
  69 TestExecutor::~TestExecutor (void)
  70 {
  71         if (m_testCaseListWriter)
  72                 qpXmlWriter_destroy(m_testCaseListWriter);
  73
  74         if (m_testCaseListFile)
  75                 fclose(m_testCaseListFile);
  76
  77         delete m_rootNode;
  78 }
  79
  80 // Test sub-case iteration.
  81 void TestExecutor::enterTestPackage (TestPackage* testPackage, const char* packageName)
  82 {
  83         DE_ASSERT(testPackage && packageName);
  84
  85         // Open file/writer for case dumping.
  86         const RunMode runMode = m_cmdLine.getRunMode();
  87         if (runMode == RUNMODE_DUMP_XML_CASELIST || runMode == RUNMODE_DUMP_TEXT_CASELIST)
  88         {
  89                 const char* const       ext                             = (runMode == RUNMODE_DUMP_XML_CASELIST) ? "xml" : "txt";
  90                 const string            fileName                = string(packageName) + "-cases." + ext;
  91
  92                 print("Dumping all test case names in '%s' to file '%s'..\n", packageName, fileName.c_str());
  93                 TCU_CHECK(m_testCaseListFile = fopen(fileName.c_str(), "wb"));
  94
  95                 if (runMode == RUNMODE_DUMP_XML_CASELIST)
  96                 {
  97                         TCU_CHECK(m_testCaseListWriter = qpXmlWriter_createFileWriter(m_testCaseListFile, DE_FALSE));
  98
  99                         qpXmlWriter_startDocument(m_testCaseListWriter);
 100                         qpXmlWriter_startElement(m_testCaseListWriter, "TestCaseList", 0, DE_NULL);
 101                 }
 102         }
 103
 104         // Initialize package.
 105         testPackage->init();
 106
 107         // Store test case wrapper
 108         m_testCaseWrapper = &testPackage->getTestCaseWrapper();
 109         DE_ASSERT(m_testCaseWrapper);
 110
 111         // Set archive.
 112         m_testCtx.setCurrentArchive(testPackage->getArchive());
 113 }
 114
 115 void TestExecutor::leaveTestPackage (TestPackage* testPackage)
 116 {
 117         DE_ASSERT(testPackage);
 118
 119         const RunMode runMode = m_cmdLine.getRunMode();
 120         if (runMode == RUNMODE_DUMP_XML_CASELIST)
 121         {
 122                 qpXmlWriter_endElement(m_testCaseListWriter, "TestCaseList");
 123                 qpXmlWriter_endDocument(m_testCaseListWriter);
 124                 qpXmlWriter_destroy(m_testCaseListWriter);
 125                 m_testCaseListWriter = DE_NULL;
 126         }
 127
 128         if (runMode == RUNMODE_DUMP_TEXT_CASELIST || runMode == RUNMODE_DUMP_XML_CASELIST)
 129         {
 130                 fclose(m_testCaseListFile);
 131                 m_testCaseListFile = DE_NULL;
 132         }
 133
 134         DE_ASSERT(!m_testCaseListWriter && !m_testCaseListFile);
 135
 136         m_testCaseWrapper = DE_NULL;
 137         m_testCtx.setCurrentArchive(m_testCtx.getRootArchive());
 138
 139         // Deinitialize package.
 140         testPackage->deinit();
 141 }
 142
 143 void TestExecutor::enterGroupNode (TestCaseGroup* testGroup, const char* casePath)
 144 {
 145         DE_UNREF(casePath);
 146         testGroup->init();
 147 }
 148
 149 void TestExecutor::leaveGroupNode (TestCaseGroup* testGroup)
 150 {
 151         testGroup->deinit();
 152 }
 153
 154 static qpTestCaseType nodeTypeToTestCaseType (TestNodeType nodeType)
 155 {
 156         switch (nodeType)
 157         {
 158                 case NODETYPE_SELF_VALIDATE:    return QP_TEST_CASE_TYPE_SELF_VALIDATE;
 159                 case NODETYPE_PERFORMANCE:              return QP_TEST_CASE_TYPE_PERFORMANCE;
 160                 case NODETYPE_CAPABILITY:               return QP_TEST_CASE_TYPE_CAPABILITY;
 161                 case NODETYPE_ACCURACY:                 return QP_TEST_CASE_TYPE_ACCURACY;
 162                 default:
 163                         DE_ASSERT(DE_FALSE);
 164                         return QP_TEST_CASE_TYPE_LAST;
 165         }
 166 }
 167
 168 bool TestExecutor::enterTestCase (TestCase* testCase, const char* casePath)
 169 {
 170         const RunMode                   runMode         = m_cmdLine.getRunMode();
 171         const qpTestCaseType    caseType        = nodeTypeToTestCaseType(testCase->getNodeType());
 172
 173         if (runMode == RUNMODE_EXECUTE)
 174         {
 175                 print("\nTest case '%s'..\n", casePath);
 176
 177                 m_testCtx.getLog().startCase(casePath, caseType);
 178                 m_isInTestCase = true;
 179                 m_testCtx.setTestResult(QP_TEST_RESULT_LAST, "");
 180
 181                 if (!m_testCaseWrapper->initTestCase(testCase))
 182                 {
 183                         if (m_testCtx.getTestResult() == QP_TEST_RESULT_LAST)
 184                                 m_testCtx.setTestResult(QP_TEST_RESULT_INTERNAL_ERROR, "Unexpected error in subcase init");
 185                         return false;
 186                 }
 187         }
 188
 189         return true;
 190 }
 191
 192 void TestExecutor::leaveTestCase (TestCase* testCase)
 193 {
 194         const RunMode runMode = m_cmdLine.getRunMode();
 195         if (runMode == RUNMODE_EXECUTE)
 196         {
 197                 // De-init case.
 198                 const bool                      deinitOk                = m_testCaseWrapper->deinitTestCase(testCase);
 199                 const qpTestResult      testResult              = m_testCtx.getTestResult();
 200                 const char* const       testResultDesc  = m_testCtx.getTestResultDesc();
 201                 const bool                      terminateAfter  = m_testCtx.getTerminateAfter();
 202                 DE_ASSERT(testResult != QP_TEST_RESULT_LAST);
 203
 204                 m_isInTestCase = false;
 205                 m_testCtx.getLog().endCase(testResult, testResultDesc);
 206
 207                 // Update statistics.
 208                 print("  %s (%s)\n", qpGetTestResultName(testResult), testResultDesc);
 209
 210                 m_result.numExecuted += 1;
 211                 switch (testResult)
 212                 {
 213                         case QP_TEST_RESULT_PASS:                                       m_result.numPassed                      += 1;   break;
 214                         case QP_TEST_RESULT_NOT_SUPPORTED:                      m_result.numNotSupported        += 1;   break;
 215                         case QP_TEST_RESULT_QUALITY_WARNING:            m_result.numWarnings            += 1;   break;
 216                         case QP_TEST_RESULT_COMPATIBILITY_WARNING:      m_result.numWarnings            += 1;   break;
 217                         default:                                                                        m_result.numFailed                      += 1;   break;
 218                 }
 219
 220                 // terminateAfter, Resource error or any error in deinit means that execution should end
 221                 if (terminateAfter || !deinitOk || testResult == QP_TEST_RESULT_RESOURCE_ERROR)
 222                         m_abortSession = true;
 223
 224                 // \todo [2011-02-09 pyry] Disable watchdog temporarily?
 225                 if (m_testCtx.getWatchDog())
 226                         qpWatchDog_reset(m_testCtx.getWatchDog());
 227         }
 228 }
 229
 230 // Return true while session should still continue, false otherwise.
 231 bool TestExecutor::iterate (void)
 232 {
 233         try
 234         {
 235                 while (!m_sessionStack.empty())
 236                 {
 237                         // Get full path to node.
 238                         string nodePath = "";
 239                         for (int ndx = 0; ndx < (int)m_sessionStack.size(); ndx++)
 240                         {
 241                                 NodeIter& iter = m_sessionStack[ndx];
 242                                 if (ndx > 1) // ignore root package
 243                                         nodePath += ".";
 244                                 nodePath += iter.node->getName();
 245                         }
 246
 247                         // Handle the node.
 248                         NodeIter& iter = m_sessionStack[m_sessionStack.size()-1];
 249                         DE_ASSERT(iter.node != DE_NULL);
 250                         TestNode*               node    = iter.node;
 251                         bool                    isLeaf  = isTestNodeTypeExecutable(node->getNodeType());
 252
 253                         switch (iter.getState())
 254                         {
 255                                 case NodeIter::STATE_BEGIN:
 256                                 {
 257                                         // Return to parent if name doesn't match filter.
 258                                         if (!(isLeaf ? m_cmdLine.checkTestCaseName(nodePath.c_str()) : m_cmdLine.checkTestGroupName(nodePath.c_str())))
 259                                         {
 260                                                 m_sessionStack.pop_back();
 261                                                 break;
 262                                         }
 263
 264                                         // Enter node.
 265                                         bool enterOk = true;
 266                                         switch (node->getNodeType())
 267                                         {
 268                                                 case NODETYPE_ROOT:                             /* nada */                                                                                                                                      break;
 269                                                 case NODETYPE_PACKAGE:                  enterTestPackage(static_cast<TestPackage*>(node), nodePath.c_str());            break;
 270                                                 case NODETYPE_GROUP:                    enterGroupNode(static_cast<TestCaseGroup*>(node), nodePath.c_str());            break;
 271                                                 case NODETYPE_PERFORMANCE:
 272                                                 case NODETYPE_CAPABILITY:
 273                                                 case NODETYPE_ACCURACY:                 /* fall-trough */
 274                                                 case NODETYPE_SELF_VALIDATE:    enterOk = enterTestCase(static_cast<TestCase*>(node), nodePath.c_str());        break;
 275                                                 default: DE_ASSERT(false);
 276                                         }
 277
 278                                         if (m_cmdLine.getRunMode() == RUNMODE_EXECUTE)
 279                                         {
 280                                                 if (isLeaf)
 281                                                 {
 282                                                         if (enterOk)
 283                                                                 iter.setState(NodeIter::STATE_EXECUTE_TEST);
 284                                                         else
 285                                                                 iter.setState(NodeIter::STATE_FINISH);
 286                                                 }
 287                                                 else
 288                                                 {
 289                                                         iter.setState(NodeIter::STATE_TRAVERSE_CHILDREN);
 290                                                 }
 291                                         }
 292                                         else if (m_cmdLine.getRunMode() == RUNMODE_DUMP_XML_CASELIST)
 293                                         {
 294                                                 if (node->getNodeType() != NODETYPE_ROOT && node->getNodeType() != NODETYPE_PACKAGE)
 295                                                 {
 296                                                         string                  caseName        = iter.node->getName();
 297                                                         string                  description     = iter.node->getDescription();
 298                                                         qpXmlAttribute  attribs[8];
 299                                                         int                             numAttribs = 0;
 300                                                         const char*             caseType        = DE_NULL;
 301
 302                                                         switch (node->getNodeType())
 303                                                         {
 304                                                                 case NODETYPE_SELF_VALIDATE:    caseType = "SelfValidate";      break;
 305                                                                 case NODETYPE_CAPABILITY:               caseType = "Capability";        break;
 306                                                                 case NODETYPE_ACCURACY:                 caseType = "Accuracy";          break;
 307                                                                 case NODETYPE_PERFORMANCE:              caseType = "Performance";       break;
 308                                                                 default:                                                caseType = "TestGroup";         break;
 309                                                         }
 310
 311                                                         attribs[numAttribs++] = qpSetStringAttrib("Name", caseName.c_str());
 312                                                         attribs[numAttribs++] = qpSetStringAttrib("CaseType", caseType);
 313                                                         attribs[numAttribs++] = qpSetStringAttrib("Description", description.c_str());
 314                                                         qpXmlWriter_startElement(m_testCaseListWriter, "TestCase", numAttribs, attribs);
 315                                                 }
 316
 317                                                 iter.setState(isLeaf ? NodeIter::STATE_FINISH : NodeIter::STATE_TRAVERSE_CHILDREN);
 318                                         }
 319                                         else if (m_cmdLine.getRunMode() == RUNMODE_DUMP_TEXT_CASELIST)
 320                                         {
 321                                                 // \note Case list file is not open until we are in test package.
 322                                                 if (isLeaf)
 323                                                         fprintf(m_testCaseListFile, "TEST: %s\n", nodePath.c_str());
 324                                                 else if (node->getNodeType() != NODETYPE_ROOT)
 325                                                         fprintf(m_testCaseListFile, "GROUP: %s\n", nodePath.c_str());
 326                                                 iter.setState(isLeaf ? NodeIter::STATE_FINISH : NodeIter::STATE_TRAVERSE_CHILDREN);
 327                                         }
 328
 329                                         break;
 330                                 }
 331
 332                                 case NodeIter::STATE_EXECUTE_TEST:
 333                                 {
 334                                         // Touch the watchdog.
 335                                         m_testCtx.touchWatchdog();
 336
 337                                         // Iterate the sub-case.
 338                                         TestCase::IterateResult iterateResult = m_testCaseWrapper->iterateTestCase(static_cast<TestCase*>(node));
 339
 340                                         if (iterateResult == TestCase::STOP)
 341                                                 iter.setState(NodeIter::STATE_FINISH);
 342
 343                                         return true; // return after each iteration (when another iteration follows).
 344                                 }
 345
 346                                 case NodeIter::STATE_TRAVERSE_CHILDREN:
 347                                 {
 348                                         int numChildren = (int)iter.children.size();
 349                                         if (++iter.curChildNdx < numChildren)
 350                                         {
 351                                                 // Push child to stack.
 352                                                 TestNode* childNode = iter.children[iter.curChildNdx];
 353                                                 m_sessionStack.push_back(NodeIter(childNode));
 354                                         }
 355                                         else
 356                                                 iter.setState(NodeIter::STATE_FINISH);
 357
 358                                         break;
 359                                 }
 360
 361                                 case NodeIter::STATE_FINISH:
 362                                 {
 363                                         if (m_cmdLine.getRunMode() == RUNMODE_DUMP_XML_CASELIST)
 364                                         {
 365                                                 if (node->getNodeType() != NODETYPE_ROOT && node->getNodeType() != NODETYPE_PACKAGE)
 366                                                         qpXmlWriter_endElement(m_testCaseListWriter, "TestCase");
 367                                         }
 368
 369                                         // Leave node.
 370                                         switch (node->getNodeType())
 371                                         {
 372                                                 case NODETYPE_ROOT:                             /* nada */                                                                                      break;
 373                                                 case NODETYPE_PACKAGE:                  leaveTestPackage(static_cast<TestPackage*>(node));      break;
 374                                                 case NODETYPE_GROUP:                    leaveGroupNode(static_cast<TestCaseGroup*>(node));      break;
 375                                                 case NODETYPE_ACCURACY:
 376                                                 case NODETYPE_CAPABILITY:
 377                                                 case NODETYPE_PERFORMANCE:              /* fall-thru */
 378                                                 case NODETYPE_SELF_VALIDATE:    leaveTestCase(static_cast<TestCase*>(node));            break;
 379                                                 default: DE_ASSERT(false);
 380                                         }
 381
 382                                         m_sessionStack.pop_back();
 383
 384                                         // Return if execution should abort.
 385                                         if (m_abortSession)
 386                                                 return false;
 387
 388                                         // Otherwise continue iterating.
 389                                         break;
 390                                 }
 391
 392                                 default:
 393                                         DE_ASSERT(DE_FALSE);
 394                                         break;
 395                         }
 396                 }
 397         }
 398         catch (const std::exception& e)
 399         {
 400                 print("TestExecutor::iterateSession(): Caught unhandled %s: %s\n", typeid(e).name(), e.what());
 401                 throw;
 402         }
 403
 404         m_result.isComplete = true;
 405         return false;
 406 }
 407
 408 } // tcu