framework/common/tcuTestSessionExecutor.cpp

   1 /*-------------------------------------------------------------------------
   2  * drawElements Quality Program Tester Core
   3  * ----------------------------------------
   4  *
   5  * Copyright 2014 The Android Open Source Project
   6  *
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  *
  19  *//*!
  20  * \file
  21  * \brief Test executor.
  22  *//*--------------------------------------------------------------------*/
  23
  24 #include "tcuTestSessionExecutor.hpp"
  25 #include "tcuCommandLine.hpp"
  26 #include "tcuTestLog.hpp"
  27
  28 #include "deClock.h"
  29
  30 namespace tcu
  31 {
  32
  33 using std::vector;
  34
  35 static qpTestCaseType nodeTypeToTestCaseType (TestNodeType nodeType)
  36 {
  37         switch (nodeType)
  38         {
  39                 case NODETYPE_SELF_VALIDATE:    return QP_TEST_CASE_TYPE_SELF_VALIDATE;
  40                 case NODETYPE_PERFORMANCE:              return QP_TEST_CASE_TYPE_PERFORMANCE;
  41                 case NODETYPE_CAPABILITY:               return QP_TEST_CASE_TYPE_CAPABILITY;
  42                 case NODETYPE_ACCURACY:                 return QP_TEST_CASE_TYPE_ACCURACY;
  43                 default:
  44                         DE_ASSERT(false);
  45                         return QP_TEST_CASE_TYPE_LAST;
  46         }
  47 }
  48
  49 TestSessionExecutor::TestSessionExecutor (TestPackageRoot& root, TestContext& testCtx)
  50         : m_testCtx                     (testCtx)
  51         , m_inflater            (testCtx)
  52         , m_caseListFilter      (testCtx.getCommandLine().createCaseListFilter(testCtx.getArchive()))
  53         , m_iterator            (root, m_inflater, *m_caseListFilter)
  54         , m_state                       (STATE_TRAVERSE_HIERARCHY)
  55         , m_abortSession        (false)
  56         , m_isInTestCase        (false)
  57         , m_testStartTime       (0)
  58 {
  59 }
  60
  61 TestSessionExecutor::~TestSessionExecutor (void)
  62 {
  63 }
  64
  65 bool TestSessionExecutor::iterate (void)
  66 {
  67         while (!m_abortSession)
  68         {
  69                 switch (m_state)
  70                 {
  71                         case STATE_TRAVERSE_HIERARCHY:
  72                         {
  73                                 const TestHierarchyIterator::State      hierIterState   = m_iterator.getState();
  74
  75                                 if (hierIterState == TestHierarchyIterator::STATE_ENTER_NODE ||
  76                                         hierIterState == TestHierarchyIterator::STATE_LEAVE_NODE)
  77                                 {
  78                                         TestNode* const         curNode         = m_iterator.getNode();
  79                                         const TestNodeType      nodeType        = curNode->getNodeType();
  80                                         const bool                      isEnter         = hierIterState == TestHierarchyIterator::STATE_ENTER_NODE;
  81
  82                                         switch (nodeType)
  83                                         {
  84                                                 case NODETYPE_PACKAGE:
  85                                                 {
  86                                                         TestPackage* const testPackage = static_cast<TestPackage*>(curNode);
  87                                                         isEnter ? enterTestPackage(testPackage) : leaveTestPackage(testPackage);
  88                                                         break;
  89                                                 }
  90
  91                                                 case NODETYPE_GROUP:
  92                                                         break; // nada
  93
  94                                                 case NODETYPE_SELF_VALIDATE:
  95                                                 case NODETYPE_PERFORMANCE:
  96                                                 case NODETYPE_CAPABILITY:
  97                                                 case NODETYPE_ACCURACY:
  98                                                 {
  99                                                         TestCase* const testCase = static_cast<TestCase*>(curNode);
 100
 101                                                         if (isEnter)
 102                                                         {
 103                                                                 if (enterTestCase(testCase, m_iterator.getNodePath()))
 104                                                                         m_state = STATE_EXECUTE_TEST_CASE;
 105                                                                 // else remain in TRAVERSING_HIERARCHY => node will be exited from in the next iteration
 106                                                         }
 107                                                         else
 108                                                                 leaveTestCase(testCase);
 109
 110                                                         break;
 111                                                 }
 112
 113                                                 default:
 114                                                         DE_ASSERT(false);
 115                                                         break;
 116                                         }
 117
 118                                         m_iterator.next();
 119                                         break;
 120                                 }
 121                                 else
 122                                 {
 123                                         DE_ASSERT(hierIterState == TestHierarchyIterator::STATE_FINISHED);
 124                                         m_status.isComplete = true;
 125                                         return false;
 126                                 }
 127                         }
 128
 129                         case STATE_EXECUTE_TEST_CASE:
 130                         {
 131                                 DE_ASSERT(m_iterator.getState() == TestHierarchyIterator::STATE_LEAVE_NODE &&
 132                                                   isTestNodeTypeExecutable(m_iterator.getNode()->getNodeType()));
 133
 134                                 TestCase* const                                 testCase        = static_cast<TestCase*>(m_iterator.getNode());
 135                                 const TestCase::IterateResult   iterResult      = iterateTestCase(testCase);
 136
 137                                 if (iterResult == TestCase::STOP)
 138                                         m_state = STATE_TRAVERSE_HIERARCHY;
 139
 140                                 return true;
 141                         }
 142
 143                         default:
 144                                 DE_ASSERT(false);
 145                                 break;
 146                 }
 147         }
 148
 149         return false;
 150 }
 151
 152 void TestSessionExecutor::enterTestPackage (TestPackage* testPackage)
 153 {
 154         // Create test case wrapper
 155         DE_ASSERT(!m_caseExecutor);
 156         m_caseExecutor = de::MovePtr<TestCaseExecutor>(testPackage->createExecutor());
 157 }
 158
 159 void TestSessionExecutor::leaveTestPackage (TestPackage* testPackage)
 160 {
 161         DE_UNREF(testPackage);
 162         m_caseExecutor.clear();
 163 }
 164
 165 bool TestSessionExecutor::enterTestCase (TestCase* testCase, const std::string& casePath)
 166 {
 167         TestLog&                                log                     = m_testCtx.getLog();
 168         const qpTestCaseType    caseType        = nodeTypeToTestCaseType(testCase->getNodeType());
 169         bool                                    initOk          = false;
 170
 171         print("\nTest case '%s'..\n", casePath.c_str());
 172
 173         m_testCtx.setTestResult(QP_TEST_RESULT_LAST, "");
 174         m_testCtx.setTerminateAfter(false);
 175         log.startCase(casePath.c_str(), caseType);
 176
 177         m_isInTestCase  = true;
 178         m_testStartTime = deGetMicroseconds();
 179
 180         try
 181         {
 182                 m_caseExecutor->init(testCase, casePath);
 183                 initOk = true;
 184         }
 185         catch (const std::bad_alloc&)
 186         {
 187                 DE_ASSERT(!initOk);
 188                 m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory in test case init");
 189                 m_testCtx.setTerminateAfter(true);
 190         }
 191         catch (const tcu::TestException& e)
 192         {
 193                 DE_ASSERT(!initOk);
 194                 DE_ASSERT(e.getTestResult() != QP_TEST_RESULT_LAST);
 195                 m_testCtx.setTestResult(e.getTestResult(), e.getMessage());
 196                 m_testCtx.setTerminateAfter(e.isFatal());
 197                 log << e;
 198         }
 199         catch (const tcu::Exception& e)
 200         {
 201                 DE_ASSERT(!initOk);
 202                 m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage());
 203                 log << e;
 204         }
 205
 206         DE_ASSERT(initOk || m_testCtx.getTestResult() != QP_TEST_RESULT_LAST);
 207
 208         return initOk;
 209 }
 210
 211 void TestSessionExecutor::leaveTestCase (TestCase* testCase)
 212 {
 213         TestLog&        log             = m_testCtx.getLog();
 214
 215         // De-init case.
 216         try
 217         {
 218                 m_caseExecutor->deinit(testCase);
 219         }
 220         catch (const tcu::Exception& e)
 221         {
 222                 log << e << TestLog::Message << "Error in test case deinit, test program will terminate." << TestLog::EndMessage;
 223                 m_testCtx.setTerminateAfter(true);
 224         }
 225
 226         {
 227                 const deInt64 duration = deGetMicroseconds()-m_testStartTime;
 228                 m_testStartTime = 0;
 229                 m_testCtx.getLog() << TestLog::Integer("TestDuration", "Test case duration in microseconds", "us", QP_KEY_TAG_TIME, duration);
 230         }
 231
 232         {
 233                 const qpTestResult      testResult              = m_testCtx.getTestResult();
 234                 const char* const       testResultDesc  = m_testCtx.getTestResultDesc();
 235                 const bool                      terminateAfter  = m_testCtx.getTerminateAfter();
 236                 DE_ASSERT(testResult != QP_TEST_RESULT_LAST);
 237
 238                 m_isInTestCase = false;
 239                 m_testCtx.getLog().endCase(testResult, testResultDesc);
 240
 241                 // Update statistics.
 242                 print("  %s (%s)\n", qpGetTestResultName(testResult), testResultDesc);
 243
 244                 m_status.numExecuted += 1;
 245                 switch (testResult)
 246                 {
 247                         case QP_TEST_RESULT_PASS:                                       m_status.numPassed                      += 1;   break;
 248                         case QP_TEST_RESULT_NOT_SUPPORTED:                      m_status.numNotSupported        += 1;   break;
 249                         case QP_TEST_RESULT_QUALITY_WARNING:            m_status.numWarnings            += 1;   break;
 250                         case QP_TEST_RESULT_COMPATIBILITY_WARNING:      m_status.numWarnings            += 1;   break;
 251                         default:                                                                        m_status.numFailed                      += 1;   break;
 252                 }
 253
 254                 // terminateAfter, Resource error or any error in deinit means that execution should end
 255                 if (terminateAfter || testResult == QP_TEST_RESULT_RESOURCE_ERROR)
 256                         m_abortSession = true;
 257         }
 258
 259         if (m_testCtx.getWatchDog())
 260                 qpWatchDog_reset(m_testCtx.getWatchDog());
 261 }
 262
 263 TestCase::IterateResult TestSessionExecutor::iterateTestCase (TestCase* testCase)
 264 {
 265         TestLog&                                log                             = m_testCtx.getLog();
 266         TestCase::IterateResult iterateResult   = TestCase::STOP;
 267
 268         m_testCtx.touchWatchdog();
 269
 270         try
 271         {
 272                 iterateResult = m_caseExecutor->iterate(testCase);
 273         }
 274         catch (const std::bad_alloc&)
 275         {
 276                 m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory during test execution");
 277                 m_testCtx.setTerminateAfter(true);
 278         }
 279         catch (const tcu::TestException& e)
 280         {
 281                 log << e;
 282                 m_testCtx.setTestResult(e.getTestResult(), e.getMessage());
 283                 m_testCtx.setTerminateAfter(e.isFatal());
 284         }
 285         catch (const tcu::Exception& e)
 286         {
 287                 log << e;
 288                 m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage());
 289         }
 290
 291         return iterateResult;
 292 }
 293
 294 } // tcu