framework/common/tcuTestSessionExecutor.cpp

   1 /*-------------------------------------------------------------------------
   2  * drawElements Quality Program Tester Core
   3  * ----------------------------------------
   4  *
   5  * Copyright 2014 The Android Open Source Project
   6  *
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  *
  19  *//*!
  20  * \file
  21  * \brief Test executor.
  22  *//*--------------------------------------------------------------------*/
  23
  24 #include "tcuTestSessionExecutor.hpp"
  25 #include "tcuTestLog.hpp"
  26
  27 #include "deClock.h"
  28
  29 namespace tcu
  30 {
  31
  32 using std::vector;
  33
  34 static qpTestCaseType nodeTypeToTestCaseType (TestNodeType nodeType)
  35 {
  36         switch (nodeType)
  37         {
  38                 case NODETYPE_SELF_VALIDATE:    return QP_TEST_CASE_TYPE_SELF_VALIDATE;
  39                 case NODETYPE_PERFORMANCE:              return QP_TEST_CASE_TYPE_PERFORMANCE;
  40                 case NODETYPE_CAPABILITY:               return QP_TEST_CASE_TYPE_CAPABILITY;
  41                 case NODETYPE_ACCURACY:                 return QP_TEST_CASE_TYPE_ACCURACY;
  42                 default:
  43                         DE_ASSERT(false);
  44                         return QP_TEST_CASE_TYPE_LAST;
  45         }
  46 }
  47
  48 TestSessionExecutor::TestSessionExecutor (TestPackageRoot& root, TestContext& testCtx)
  49         : m_testCtx                     (testCtx)
  50         , m_inflater            (testCtx)
  51         , m_iterator            (root, m_inflater, testCtx.getCommandLine())
  52         , m_state                       (STATE_TRAVERSE_HIERARCHY)
  53         , m_abortSession        (false)
  54         , m_isInTestCase        (false)
  55         , m_testStartTime       (0)
  56 {
  57 }
  58
  59 TestSessionExecutor::~TestSessionExecutor (void)
  60 {
  61 }
  62
  63 bool TestSessionExecutor::iterate (void)
  64 {
  65         while (!m_abortSession)
  66         {
  67                 switch (m_state)
  68                 {
  69                         case STATE_TRAVERSE_HIERARCHY:
  70                         {
  71                                 const TestHierarchyIterator::State      hierIterState   = m_iterator.getState();
  72
  73                                 if (hierIterState == TestHierarchyIterator::STATE_ENTER_NODE ||
  74                                         hierIterState == TestHierarchyIterator::STATE_LEAVE_NODE)
  75                                 {
  76                                         TestNode* const         curNode         = m_iterator.getNode();
  77                                         const TestNodeType      nodeType        = curNode->getNodeType();
  78                                         const bool                      isEnter         = hierIterState == TestHierarchyIterator::STATE_ENTER_NODE;
  79
  80                                         switch (nodeType)
  81                                         {
  82                                                 case NODETYPE_PACKAGE:
  83                                                 {
  84                                                         TestPackage* const testPackage = static_cast<TestPackage*>(curNode);
  85                                                         isEnter ? enterTestPackage(testPackage) : leaveTestPackage(testPackage);
  86                                                         break;
  87                                                 }
  88
  89                                                 case NODETYPE_GROUP:
  90                                                         break; // nada
  91
  92                                                 case NODETYPE_SELF_VALIDATE:
  93                                                 case NODETYPE_PERFORMANCE:
  94                                                 case NODETYPE_CAPABILITY:
  95                                                 case NODETYPE_ACCURACY:
  96                                                 {
  97                                                         TestCase* const testCase = static_cast<TestCase*>(curNode);
  98
  99                                                         if (isEnter)
 100                                                         {
 101                                                                 if (enterTestCase(testCase, m_iterator.getNodePath()))
 102                                                                         m_state = STATE_EXECUTE_TEST_CASE;
 103                                                                 // else remain in TRAVERSING_HIERARCHY => node will be exited from in the next iteration
 104                                                         }
 105                                                         else
 106                                                                 leaveTestCase(testCase);
 107
 108                                                         break;
 109                                                 }
 110
 111                                                 default:
 112                                                         DE_ASSERT(false);
 113                                                         break;
 114                                         }
 115
 116                                         m_iterator.next();
 117                                         break;
 118                                 }
 119                                 else
 120                                 {
 121                                         DE_ASSERT(hierIterState == TestHierarchyIterator::STATE_FINISHED);
 122                                         m_status.isComplete = true;
 123                                         return false;
 124                                 }
 125                         }
 126
 127                         case STATE_EXECUTE_TEST_CASE:
 128                         {
 129                                 DE_ASSERT(m_iterator.getState() == TestHierarchyIterator::STATE_LEAVE_NODE &&
 130                                                   isTestNodeTypeExecutable(m_iterator.getNode()->getNodeType()));
 131
 132                                 TestCase* const                                 testCase        = static_cast<TestCase*>(m_iterator.getNode());
 133                                 const TestCase::IterateResult   iterResult      = iterateTestCase(testCase);
 134
 135                                 if (iterResult == TestCase::STOP)
 136                                         m_state = STATE_TRAVERSE_HIERARCHY;
 137
 138                                 return true;
 139                         }
 140
 141                         default:
 142                                 DE_ASSERT(false);
 143                                 break;
 144                 }
 145         }
 146
 147         return false;
 148 }
 149
 150 void TestSessionExecutor::enterTestPackage (TestPackage* testPackage)
 151 {
 152         // Create test case wrapper
 153         DE_ASSERT(!m_caseExecutor);
 154         m_caseExecutor = de::MovePtr<TestCaseExecutor>(testPackage->createExecutor());
 155 }
 156
 157 void TestSessionExecutor::leaveTestPackage (TestPackage* testPackage)
 158 {
 159         DE_UNREF(testPackage);
 160         m_caseExecutor.clear();
 161 }
 162
 163 bool TestSessionExecutor::enterTestCase (TestCase* testCase, const std::string& casePath)
 164 {
 165         TestLog&                                log                     = m_testCtx.getLog();
 166         const qpTestCaseType    caseType        = nodeTypeToTestCaseType(testCase->getNodeType());
 167         bool                                    initOk          = false;
 168
 169         print("\nTest case '%s'..\n", casePath.c_str());
 170
 171         m_testCtx.setTestResult(QP_TEST_RESULT_LAST, "");
 172         m_testCtx.setTerminateAfter(false);
 173         log.startCase(casePath.c_str(), caseType);
 174
 175         m_isInTestCase  = true;
 176         m_testStartTime = deGetMicroseconds();
 177
 178         try
 179         {
 180                 m_caseExecutor->init(testCase, casePath);
 181                 initOk = true;
 182         }
 183         catch (const std::bad_alloc&)
 184         {
 185                 DE_ASSERT(!initOk);
 186                 m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory in test case init");
 187                 m_testCtx.setTerminateAfter(true);
 188         }
 189         catch (const tcu::TestException& e)
 190         {
 191                 DE_ASSERT(!initOk);
 192                 DE_ASSERT(e.getTestResult() != QP_TEST_RESULT_LAST);
 193                 m_testCtx.setTestResult(e.getTestResult(), e.getMessage());
 194                 m_testCtx.setTerminateAfter(e.isFatal());
 195                 log << e;
 196         }
 197         catch (const tcu::Exception& e)
 198         {
 199                 DE_ASSERT(!initOk);
 200                 m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage());
 201                 log << e;
 202         }
 203
 204         DE_ASSERT(initOk || m_testCtx.getTestResult() != QP_TEST_RESULT_LAST);
 205
 206         return initOk;
 207 }
 208
 209 void TestSessionExecutor::leaveTestCase (TestCase* testCase)
 210 {
 211         TestLog&        log             = m_testCtx.getLog();
 212
 213         // De-init case.
 214         try
 215         {
 216                 m_caseExecutor->deinit(testCase);
 217         }
 218         catch (const tcu::Exception& e)
 219         {
 220                 log << e << TestLog::Message << "Error in test case deinit, test program will terminate." << TestLog::EndMessage;
 221                 m_testCtx.setTerminateAfter(true);
 222         }
 223
 224         {
 225                 const deInt64 duration = deGetMicroseconds()-m_testStartTime;
 226                 m_testStartTime = 0;
 227                 m_testCtx.getLog() << TestLog::Integer("TestDuration", "Test case duration in microseconds", "us", QP_KEY_TAG_TIME, duration);
 228         }
 229
 230         {
 231                 const qpTestResult      testResult              = m_testCtx.getTestResult();
 232                 const char* const       testResultDesc  = m_testCtx.getTestResultDesc();
 233                 const bool                      terminateAfter  = m_testCtx.getTerminateAfter();
 234                 DE_ASSERT(testResult != QP_TEST_RESULT_LAST);
 235
 236                 m_isInTestCase = false;
 237                 m_testCtx.getLog().endCase(testResult, testResultDesc);
 238
 239                 // Update statistics.
 240                 print("  %s (%s)\n", qpGetTestResultName(testResult), testResultDesc);
 241
 242                 m_status.numExecuted += 1;
 243                 switch (testResult)
 244                 {
 245                         case QP_TEST_RESULT_PASS:                                       m_status.numPassed                      += 1;   break;
 246                         case QP_TEST_RESULT_NOT_SUPPORTED:                      m_status.numNotSupported        += 1;   break;
 247                         case QP_TEST_RESULT_QUALITY_WARNING:            m_status.numWarnings            += 1;   break;
 248                         case QP_TEST_RESULT_COMPATIBILITY_WARNING:      m_status.numWarnings            += 1;   break;
 249                         default:                                                                        m_status.numFailed                      += 1;   break;
 250                 }
 251
 252                 // terminateAfter, Resource error or any error in deinit means that execution should end
 253                 if (terminateAfter || testResult == QP_TEST_RESULT_RESOURCE_ERROR)
 254                         m_abortSession = true;
 255         }
 256
 257         if (m_testCtx.getWatchDog())
 258                 qpWatchDog_reset(m_testCtx.getWatchDog());
 259 }
 260
 261 TestCase::IterateResult TestSessionExecutor::iterateTestCase (TestCase* testCase)
 262 {
 263         TestLog&                                log                             = m_testCtx.getLog();
 264         TestCase::IterateResult iterateResult   = TestCase::STOP;
 265
 266         m_testCtx.touchWatchdog();
 267
 268         try
 269         {
 270                 iterateResult = m_caseExecutor->iterate(testCase);
 271         }
 272         catch (const std::bad_alloc&)
 273         {
 274                 m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory during test execution");
 275                 m_testCtx.setTerminateAfter(true);
 276         }
 277         catch (const tcu::TestException& e)
 278         {
 279                 log << e;
 280                 m_testCtx.setTestResult(e.getTestResult(), e.getMessage());
 281                 m_testCtx.setTerminateAfter(e.isFatal());
 282         }
 283         catch (const tcu::Exception& e)
 284         {
 285                 log << e;
 286                 m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage());
 287         }
 288
 289         return iterateResult;
 290 }
 291
 292 } // tcu