framework/common/tcuTestSessionExecutor.cpp

   1 /*-------------------------------------------------------------------------
   2  * drawElements Quality Program Tester Core
   3  * ----------------------------------------
   4  *
   5  * Copyright 2014 The Android Open Source Project
   6  *
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  *
  19  *//*!
  20  * \file
  21  * \brief Test executor.
  22  *//*--------------------------------------------------------------------*/
  23
  24 #include "tcuTestSessionExecutor.hpp"
  25 #include "tcuTestLog.hpp"
  26
  27 #include "deClock.h"
  28
  29 namespace tcu
  30 {
  31
  32 using std::vector;
  33
  34 static qpTestCaseType nodeTypeToTestCaseType (TestNodeType nodeType)
  35 {
  36         switch (nodeType)
  37         {
  38                 case NODETYPE_SELF_VALIDATE:    return QP_TEST_CASE_TYPE_SELF_VALIDATE;
  39                 case NODETYPE_PERFORMANCE:              return QP_TEST_CASE_TYPE_PERFORMANCE;
  40                 case NODETYPE_CAPABILITY:               return QP_TEST_CASE_TYPE_CAPABILITY;
  41                 case NODETYPE_ACCURACY:                 return QP_TEST_CASE_TYPE_ACCURACY;
  42                 default:
  43                         DE_ASSERT(false);
  44                         return QP_TEST_CASE_TYPE_LAST;
  45         }
  46 }
  47
  48 TestSessionExecutor::TestSessionExecutor (TestPackageRoot& root, TestContext& testCtx)
  49         : m_testCtx                     (testCtx)
  50         , m_inflater            (testCtx)
  51         , m_iterator            (root, m_inflater, testCtx.getCommandLine())
  52         , m_state                       (STATE_TRAVERSE_HIERARCHY)
  53         , m_abortSession        (false)
  54         , m_isInTestCase        (false)
  55         , m_testStartTime       (0)
  56 {
  57 }
  58
  59 TestSessionExecutor::~TestSessionExecutor (void)
  60 {
  61 }
  62
  63 bool TestSessionExecutor::iterate (void)
  64 {
  65         for (;;)
  66         {
  67                 switch (m_state)
  68                 {
  69                         case STATE_TRAVERSE_HIERARCHY:
  70                         {
  71                                 const TestHierarchyIterator::State      hierIterState   = m_iterator.getState();
  72
  73                                 if (hierIterState == TestHierarchyIterator::STATE_ENTER_NODE ||
  74                                         hierIterState == TestHierarchyIterator::STATE_LEAVE_NODE)
  75                                 {
  76                                         TestNode* const         curNode         = m_iterator.getNode();
  77                                         const TestNodeType      nodeType        = curNode->getNodeType();
  78                                         const bool                      isEnter         = hierIterState == TestHierarchyIterator::STATE_ENTER_NODE;
  79
  80                                         switch (nodeType)
  81                                         {
  82                                                 case NODETYPE_PACKAGE:
  83                                                 {
  84                                                         TestPackage* const testPackage = static_cast<TestPackage*>(curNode);
  85                                                         isEnter ? enterTestPackage(testPackage) : leaveTestPackage(testPackage);
  86                                                         break;
  87                                                 }
  88
  89                                                 case NODETYPE_GROUP:
  90                                                         break; // nada
  91
  92                                                 case NODETYPE_SELF_VALIDATE:
  93                                                 case NODETYPE_PERFORMANCE:
  94                                                 case NODETYPE_CAPABILITY:
  95                                                 case NODETYPE_ACCURACY:
  96                                                 {
  97                                                         TestCase* const testCase = static_cast<TestCase*>(curNode);
  98
  99                                                         if (isEnter)
 100                                                         {
 101                                                                 if (enterTestCase(testCase, m_iterator.getNodePath()))
 102                                                                         m_state = STATE_EXECUTE_TEST_CASE;
 103                                                                 // else remain in TRAVERSING_HIERARCHY => node will be exited from in the next iteration
 104                                                         }
 105                                                         else
 106                                                                 leaveTestCase(testCase);
 107
 108                                                         break;
 109                                                 }
 110
 111                                                 default:
 112                                                         DE_ASSERT(false);
 113                                                         break;
 114                                         }
 115
 116                                         m_iterator.next();
 117                                         break;
 118                                 }
 119                                 else
 120                                 {
 121                                         DE_ASSERT(hierIterState == TestHierarchyIterator::STATE_FINISHED);
 122                                         m_status.isComplete = true;
 123                                         return false;
 124                                 }
 125                         }
 126
 127                         case STATE_EXECUTE_TEST_CASE:
 128                         {
 129                                 DE_ASSERT(m_iterator.getState() == TestHierarchyIterator::STATE_LEAVE_NODE &&
 130                                                   isTestNodeTypeExecutable(m_iterator.getNode()->getNodeType()));
 131
 132                                 TestCase* const                                 testCase        = static_cast<TestCase*>(m_iterator.getNode());
 133                                 const TestCase::IterateResult   iterResult      = iterateTestCase(testCase);
 134
 135                                 if (iterResult == TestCase::STOP)
 136                                         m_state = STATE_TRAVERSE_HIERARCHY;
 137
 138                                 return true;
 139                         }
 140
 141                         default:
 142                                 DE_ASSERT(false);
 143                                 break;
 144                 }
 145         }
 146
 147         DE_ASSERT(false);
 148         return false;
 149 }
 150
 151 void TestSessionExecutor::enterTestPackage (TestPackage* testPackage)
 152 {
 153         // Create test case wrapper
 154         DE_ASSERT(!m_caseExecutor);
 155         m_caseExecutor = de::MovePtr<TestCaseExecutor>(testPackage->createExecutor());
 156 }
 157
 158 void TestSessionExecutor::leaveTestPackage (TestPackage* testPackage)
 159 {
 160         DE_UNREF(testPackage);
 161         m_caseExecutor.clear();
 162 }
 163
 164 bool TestSessionExecutor::enterTestCase (TestCase* testCase, const std::string& casePath)
 165 {
 166         TestLog&                                log                     = m_testCtx.getLog();
 167         const qpTestCaseType    caseType        = nodeTypeToTestCaseType(testCase->getNodeType());
 168         bool                                    initOk          = false;
 169
 170         print("\nTest case '%s'..\n", casePath.c_str());
 171
 172         m_testCtx.setTestResult(QP_TEST_RESULT_LAST, "");
 173         m_testCtx.setTerminateAfter(false);
 174         log.startCase(casePath.c_str(), caseType);
 175
 176         m_isInTestCase  = true;
 177         m_testStartTime = deGetMicroseconds();
 178
 179         try
 180         {
 181                 m_caseExecutor->init(testCase, casePath);
 182                 initOk = true;
 183         }
 184         catch (const std::bad_alloc&)
 185         {
 186                 DE_ASSERT(!initOk);
 187                 m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory in test case init");
 188                 m_testCtx.setTerminateAfter(true);
 189         }
 190         catch (const tcu::TestException& e)
 191         {
 192                 DE_ASSERT(!initOk);
 193                 DE_ASSERT(e.getTestResult() != QP_TEST_RESULT_LAST);
 194                 m_testCtx.setTestResult(e.getTestResult(), e.getMessage());
 195                 m_testCtx.setTerminateAfter(e.isFatal());
 196                 log << e;
 197         }
 198         catch (const tcu::Exception& e)
 199         {
 200                 DE_ASSERT(!initOk);
 201                 m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage());
 202                 log << e;
 203         }
 204
 205         DE_ASSERT(initOk || m_testCtx.getTestResult() != QP_TEST_RESULT_LAST);
 206
 207         return initOk;
 208 }
 209
 210 void TestSessionExecutor::leaveTestCase (TestCase* testCase)
 211 {
 212         TestLog&        log             = m_testCtx.getLog();
 213
 214         // De-init case.
 215         try
 216         {
 217                 m_caseExecutor->deinit(testCase);
 218         }
 219         catch (const tcu::Exception& e)
 220         {
 221                 log << e << TestLog::Message << "Error in test case deinit, test program will terminate." << TestLog::EndMessage;
 222                 m_testCtx.setTerminateAfter(true);
 223         }
 224
 225         {
 226                 const deInt64 duration = deGetMicroseconds()-m_testStartTime;
 227                 m_testStartTime = 0;
 228                 m_testCtx.getLog() << TestLog::Integer("TestDuration", "Test case duration in microseconds", "us", QP_KEY_TAG_TIME, duration);
 229         }
 230
 231         {
 232                 const qpTestResult      testResult              = m_testCtx.getTestResult();
 233                 const char* const       testResultDesc  = m_testCtx.getTestResultDesc();
 234                 const bool                      terminateAfter  = m_testCtx.getTerminateAfter();
 235                 DE_ASSERT(testResult != QP_TEST_RESULT_LAST);
 236
 237                 m_isInTestCase = false;
 238                 m_testCtx.getLog().endCase(testResult, testResultDesc);
 239
 240                 // Update statistics.
 241                 print("  %s (%s)\n", qpGetTestResultName(testResult), testResultDesc);
 242
 243                 m_status.numExecuted += 1;
 244                 switch (testResult)
 245                 {
 246                         case QP_TEST_RESULT_PASS:                                       m_status.numPassed                      += 1;   break;
 247                         case QP_TEST_RESULT_NOT_SUPPORTED:                      m_status.numNotSupported        += 1;   break;
 248                         case QP_TEST_RESULT_QUALITY_WARNING:            m_status.numWarnings            += 1;   break;
 249                         case QP_TEST_RESULT_COMPATIBILITY_WARNING:      m_status.numWarnings            += 1;   break;
 250                         default:                                                                        m_status.numFailed                      += 1;   break;
 251                 }
 252
 253                 // terminateAfter, Resource error or any error in deinit means that execution should end
 254                 if (terminateAfter || testResult == QP_TEST_RESULT_RESOURCE_ERROR)
 255                         m_abortSession = true;
 256         }
 257
 258         if (m_testCtx.getWatchDog())
 259                 qpWatchDog_reset(m_testCtx.getWatchDog());
 260 }
 261
 262 TestCase::IterateResult TestSessionExecutor::iterateTestCase (TestCase* testCase)
 263 {
 264         TestLog&                                log                             = m_testCtx.getLog();
 265         TestCase::IterateResult iterateResult   = TestCase::STOP;
 266
 267         m_testCtx.touchWatchdog();
 268
 269         try
 270         {
 271                 iterateResult = m_caseExecutor->iterate(testCase);
 272         }
 273         catch (const std::bad_alloc&)
 274         {
 275                 m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory during test execution");
 276                 m_testCtx.setTerminateAfter(true);
 277         }
 278         catch (const tcu::TestException& e)
 279         {
 280                 log << e;
 281                 m_testCtx.setTestResult(e.getTestResult(), e.getMessage());
 282                 m_testCtx.setTerminateAfter(e.isFatal());
 283         }
 284         catch (const tcu::Exception& e)
 285         {
 286                 log << e;
 287                 m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage());
 288         }
 289
 290         return iterateResult;
 291 }
 292
 293 } // tcu