framework/common/tcuApp.cpp

   1 /*-------------------------------------------------------------------------
   2  * drawElements Quality Program Tester Core
   3  * ----------------------------------------
   4  *
   5  * Copyright 2014 The Android Open Source Project
   6  *
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  *
  19  *//*!
  20  * \file
  21  * \brief Render target info.
  22  *//*--------------------------------------------------------------------*/
  23
  24 #include "tcuApp.hpp"
  25 #include "tcuPlatform.hpp"
  26 #include "tcuTestContext.hpp"
  27 #include "tcuTestSessionExecutor.hpp"
  28 #include "tcuTestHierarchyUtil.hpp"
  29 #include "tcuCommandLine.hpp"
  30 #include "tcuTestLog.hpp"
  31
  32 #include "qpInfo.h"
  33 #include "qpDebugOut.h"
  34
  35 #include "deMath.h"
  36
  37 #include <iostream>
  38
  39 namespace tcu
  40 {
  41
  42 using std::string;
  43
  44 /*--------------------------------------------------------------------*//*!
  45  *  Writes all packages found stdout without any
  46  *  separations. Recommended to be used with a single package
  47  *  only. It's possible to use test selectors for limiting the export
  48  *  to one package in a multipackage binary.
  49  *//*--------------------------------------------------------------------*/
  50 static void writeCaselistsToStdout (TestPackageRoot& root, TestContext& testCtx)
  51 {
  52         DefaultHierarchyInflater                        inflater                (testCtx);
  53         de::MovePtr<const CaseListFilter>       caseListFilter  (testCtx.getCommandLine().createCaseListFilter(testCtx.getArchive()));
  54         TestHierarchyIterator                           iter                    (root, inflater, *caseListFilter);
  55
  56         while (iter.getState() != TestHierarchyIterator::STATE_FINISHED)
  57         {
  58                 iter.next();
  59
  60                 while (iter.getNode()->getNodeType() != NODETYPE_PACKAGE)
  61                 {
  62                         if (iter.getState() == TestHierarchyIterator::STATE_ENTER_NODE)
  63                                 std::cout << (isTestNodeTypeExecutable(iter.getNode()->getNodeType()) ? "TEST" : "GROUP") << ": " << iter.getNodePath() << "\n";
  64                         iter.next();
  65                 }
  66
  67                 DE_ASSERT(iter.getState() == TestHierarchyIterator::STATE_LEAVE_NODE &&
  68                                   iter.getNode()->getNodeType() == NODETYPE_PACKAGE);
  69                 iter.next();
  70         }
  71 }
  72
  73
  74 /*--------------------------------------------------------------------*//*!
  75  * Verifies that amber capability requirements in the .amber files
  76  * match with capabilities defined on the CTS C code.
  77  *//*--------------------------------------------------------------------*/
  78 static void verifyAmberCapabilityCoherency (TestPackageRoot& root, TestContext& testCtx)
  79 {
  80         DefaultHierarchyInflater                        inflater(testCtx);
  81         de::MovePtr<const CaseListFilter>       caseListFilter(testCtx.getCommandLine().createCaseListFilter(testCtx.getArchive()));
  82         TestHierarchyIterator                           iter(root, inflater, *caseListFilter);
  83         int                                                                     count = 0;
  84         int                                                                     errorCount = 0;
  85
  86         bool ok = true;
  87
  88         while (iter.getState() != TestHierarchyIterator::STATE_FINISHED)
  89         {
  90                 iter.next();
  91
  92                 while (iter.getNode()->getNodeType() != NODETYPE_PACKAGE)
  93                 {
  94                         if (iter.getState() == TestHierarchyIterator::STATE_ENTER_NODE &&
  95                                 isTestNodeTypeExecutable(iter.getNode()->getNodeType()))
  96                         {
  97                                 std::cout << iter.getNodePath() << "\n";
  98                                 testCtx.getLog() << tcu::TestLog::Message << iter.getNodePath() << tcu::TestLog::EndMessage;
  99                                 if (!iter.getNode()->validateRequirements())
 100                                 {
 101                                         ok = false;
 102                                         errorCount++;
 103                                 }
 104                                 count++;
 105                         }
 106                         iter.next();
 107                 }
 108
 109                 DE_ASSERT(iter.getState() == TestHierarchyIterator::STATE_LEAVE_NODE &&
 110                         iter.getNode()->getNodeType() == NODETYPE_PACKAGE);
 111                 iter.next();
 112         }
 113         std::cout << count << " amber tests, " << errorCount << " errors.\n";
 114         if (!ok)
 115                 TCU_THROW(InternalError, "One or more CTS and Amber test requirements do not match; check log for details");
 116
 117 }
 118
 119 /*--------------------------------------------------------------------*//*!
 120  * \brief Construct test application
 121  *
 122  * If a fatal error occurs during initialization constructor will call
 123  * die() with debug information.
 124  *
 125  * \param platform Reference to platform implementation.
 126  *//*--------------------------------------------------------------------*/
 127 App::App (Platform& platform, Archive& archive, TestLog& log, const CommandLine& cmdLine)
 128         : m_platform            (platform)
 129         , m_watchDog            (DE_NULL)
 130         , m_crashHandler        (DE_NULL)
 131         , m_crashed                     (false)
 132         , m_testCtx                     (DE_NULL)
 133         , m_testRoot            (DE_NULL)
 134         , m_testExecutor        (DE_NULL)
 135 {
 136         if (!cmdLine.isSubProcess())
 137         {
 138                 print("dEQP Core %s (0x%08x) starting..\n", qpGetReleaseName(), qpGetReleaseId());
 139                 print("  target implementation = '%s'\n", qpGetTargetName());
 140         }
 141
 142         if (!deSetRoundingMode(DE_ROUNDINGMODE_TO_NEAREST_EVEN))
 143                 qpPrintf("WARNING: Failed to set floating-point rounding mode!\n");
 144
 145         try
 146         {
 147                 const RunMode   runMode = cmdLine.getRunMode();
 148
 149                 // Initialize watchdog
 150                 if (cmdLine.isWatchDogEnabled())
 151                         TCU_CHECK_INTERNAL(m_watchDog = qpWatchDog_create(onWatchdogTimeout, this, WATCHDOG_TOTAL_TIME_LIMIT_SECS, WATCHDOG_INTERVAL_TIME_LIMIT_SECS));
 152
 153                 // Initialize crash handler.
 154                 if (cmdLine.isCrashHandlingEnabled())
 155                         TCU_CHECK_INTERNAL(m_crashHandler = qpCrashHandler_create(onCrash, this));
 156
 157                 // Create test context
 158                 m_testCtx = new TestContext(m_platform, archive, log, cmdLine, m_watchDog);
 159
 160                 // Create root from registry
 161                 m_testRoot = new TestPackageRoot(*m_testCtx, TestPackageRegistry::getSingleton());
 162
 163                 // \note No executor is created if runmode is not EXECUTE
 164                 if (runMode == RUNMODE_EXECUTE)
 165                         m_testExecutor = new TestSessionExecutor(*m_testRoot, *m_testCtx);
 166                 else if (runMode == RUNMODE_DUMP_STDOUT_CASELIST)
 167                         writeCaselistsToStdout(*m_testRoot, *m_testCtx);
 168                 else if (runMode == RUNMODE_DUMP_XML_CASELIST)
 169                         writeXmlCaselistsToFiles(*m_testRoot, *m_testCtx, cmdLine);
 170                 else if (runMode == RUNMODE_DUMP_TEXT_CASELIST)
 171                         writeTxtCaselistsToFiles(*m_testRoot, *m_testCtx, cmdLine);
 172                 else if (runMode == RUNMODE_VERIFY_AMBER_COHERENCY)
 173                         verifyAmberCapabilityCoherency(*m_testRoot, *m_testCtx);
 174                 else
 175                         DE_ASSERT(false);
 176         }
 177         catch (const std::exception& e)
 178         {
 179                 cleanup();
 180                 die("Failed to initialize dEQP: %s", e.what());
 181         }
 182 }
 183
 184 App::~App (void)
 185 {
 186         cleanup();
 187 }
 188
 189 void App::cleanup (void)
 190 {
 191         delete m_testExecutor;
 192         delete m_testRoot;
 193         delete m_testCtx;
 194
 195         if (m_crashHandler)
 196                 qpCrashHandler_destroy(m_crashHandler);
 197
 198         if (m_watchDog)
 199                 qpWatchDog_destroy(m_watchDog);
 200 }
 201
 202 /*--------------------------------------------------------------------*//*!
 203  * \brief Step forward test execution
 204  * \return true if application should call iterate() again and false
 205  *         if test execution session is complete.
 206  *//*--------------------------------------------------------------------*/
 207 bool App::iterate (void)
 208 {
 209         if (!m_testExecutor)
 210         {
 211                 DE_ASSERT(m_testCtx->getCommandLine().getRunMode() != RUNMODE_EXECUTE);
 212                 return false;
 213         }
 214
 215         // Poll platform events
 216         const bool platformOk = m_platform.processEvents();
 217
 218         // Iterate a step.
 219         bool testExecOk = false;
 220         if (platformOk)
 221         {
 222                 try
 223                 {
 224                         testExecOk = m_testExecutor->iterate();
 225                 }
 226                 catch (const std::exception& e)
 227                 {
 228                         die("%s", e.what());
 229                 }
 230         }
 231
 232         if ((!platformOk || !testExecOk) )
 233         {
 234                 if (!m_testCtx->getCommandLine().isSubProcess())
 235                 {
 236                         if (!platformOk)
 237                                 print("\nABORTED!\n");
 238                         else
 239                                 print("\nDONE!\n");
 240                 }
 241
 242                 const RunMode runMode = m_testCtx->getCommandLine().getRunMode();
 243                 if (runMode == RUNMODE_EXECUTE)
 244                 {
 245                         const TestRunStatus& result = m_testExecutor->getStatus();
 246                         if(!m_testCtx->getCommandLine().isSubProcess())
 247                         {
 248                                 // Report statistics.
 249                                 print("\nTest run totals:\n");
 250                                 print("  Passed:        %d/%d (%.1f%%)\n", result.numPassed,            result.numExecuted, (result.numExecuted > 0 ? (100.0f * (float)result.numPassed                 / (float)result.numExecuted) : 0.0f));
 251                                 print("  Failed:        %d/%d (%.1f%%)\n", result.numFailed,            result.numExecuted, (result.numExecuted > 0 ? (100.0f * (float)result.numFailed                 / (float)result.numExecuted) : 0.0f));
 252                                 print("  Not supported: %d/%d (%.1f%%)\n", result.numNotSupported,      result.numExecuted, (result.numExecuted > 0 ? (100.0f * (float)result.numNotSupported   / (float)result.numExecuted) : 0.0f));
 253                                 print("  Warnings:      %d/%d (%.1f%%)\n", result.numWarnings,          result.numExecuted, (result.numExecuted > 0 ? (100.0f * (float)result.numWarnings               / (float)result.numExecuted) : 0.0f));
 254                                 print("  Waived:        %d/%d (%.1f%%)\n", result.numWaived,            result.numExecuted, (result.numExecuted > 0 ? (100.0f * (float)result.numWaived                 / (float)result.numExecuted) : 0.0f));
 255                                 if (!result.isComplete)
 256                                         print("Test run was ABORTED!\n");
 257                         }
 258                         else
 259                         {
 260                                 // subprocess sends test statisticts through qpa file, so that main process may read it
 261                                 // and add to global statistics ( search for #SubProcessStatus to see how it's done )
 262                                 std::ostringstream str;
 263                                 str << "\n#SubProcessStatus " <<
 264                                         result.numExecuted              << " " <<
 265                                         result.numPassed                << " " <<
 266                                         result.numFailed                << " " <<
 267                                         result.numNotSupported  << " " <<
 268                                         result.numWarnings              << " " <<
 269                                         result.numWaived                << "\n";
 270                                 m_testCtx->getLog().writeRaw(str.str().c_str());
 271                         }
 272                 }
 273         }
 274
 275         return platformOk && testExecOk;
 276 }
 277
 278 const TestRunStatus& App::getResult (void) const
 279 {
 280         return m_testExecutor->getStatus();
 281 }
 282
 283 void App::onWatchdogTimeout (qpWatchDog* watchDog, void* userPtr, qpTimeoutReason reason)
 284 {
 285         DE_UNREF(watchDog);
 286         static_cast<App*>(userPtr)->onWatchdogTimeout(reason);
 287 }
 288
 289 void App::onCrash (qpCrashHandler* crashHandler, void* userPtr)
 290 {
 291         DE_UNREF(crashHandler);
 292         static_cast<App*>(userPtr)->onCrash();
 293 }
 294
 295 void App::onWatchdogTimeout (qpTimeoutReason reason)
 296 {
 297         if (!m_crashLock.tryLock() || m_crashed)
 298                 return; // In crash handler already.
 299
 300         m_crashed = true;
 301
 302         m_testCtx->getLog().terminateCase(QP_TEST_RESULT_TIMEOUT);
 303         die("Watchdog timer timeout for %s", (reason == QP_TIMEOUT_REASON_INTERVAL_LIMIT ? "touch interval" : "total time"));
 304 }
 305
 306 static void writeCrashToLog (void* userPtr, const char* infoString)
 307 {
 308         // \note THIS IS CALLED BY SIGNAL HANDLER! CALLING MALLOC/FREE IS NOT ALLOWED!
 309         TestLog* log = static_cast<TestLog*>(userPtr);
 310         log->writeMessage(infoString);
 311 }
 312
 313 static void writeCrashToConsole (void* userPtr, const char* infoString)
 314 {
 315         // \note THIS IS CALLED BY SIGNAL HANDLER! CALLING MALLOC/FREE IS NOT ALLOWED!
 316         DE_UNREF(userPtr);
 317         qpPrint(infoString);
 318 }
 319
 320 void App::onCrash (void)
 321 {
 322         // \note THIS IS CALLED BY SIGNAL HANDLER! CALLING MALLOC/FREE IS NOT ALLOWED!
 323
 324         if (!m_crashLock.tryLock() || m_crashed)
 325                 return; // In crash handler already.
 326
 327         m_crashed = true;
 328
 329         bool isInCase = m_testExecutor ? m_testExecutor->isInTestCase() : false;
 330
 331         if (isInCase)
 332         {
 333                 qpCrashHandler_writeCrashInfo(m_crashHandler, writeCrashToLog, &m_testCtx->getLog());
 334                 m_testCtx->getLog().terminateCase(QP_TEST_RESULT_CRASH);
 335         }
 336         else
 337                 qpCrashHandler_writeCrashInfo(m_crashHandler, writeCrashToConsole, DE_NULL);
 338
 339         die("Test program crashed");
 340 }
 341
 342 } // tcu