src/framework/include/dpl/test/test_runner.h

   1 /*
   2  * Copyright (c) 2011-2015 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.h
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @version     1.0
  21  * @brief       This file is the header file of test runner
  22  */
  23
  24 #ifndef DPL_TEST_RUNNER_H
  25 #define DPL_TEST_RUNNER_H
  26
  27 #include <chrono>
  28 #include <cstdlib>
  29 #include <cstring>
  30 #include <exception>
  31 #include <iostream>
  32 #include <list>
  33 #include <map>
  34 #include <queue>
  35 #include <set>
  36 #include <sstream>
  37 #include <string>
  38 #include <vector>
  39
  40 #include <dpl/atomic.h>
  41 #include <dpl/availability.h>
  42 #include <dpl/colors.h>
  43 #include <dpl/gdbbacktrace.h>
  44 #include <dpl/singleton.h>
  45 #include <dpl/test/test_exception.h>
  46 #include <dpl/test/test_failed.h>
  47 #include <dpl/test/test_ignored.h>
  48 #include <dpl/test/test_results_collector.h>
  49
  50 namespace DPL {
  51 namespace Test {
  52 class TestRunner
  53 {
  54     typedef std::map<std::string, TestResultsCollectorBasePtr>
  55     TestResultsCollectors;
  56     TestResultsCollectors m_collectors;
  57
  58     std::string m_startTestId;
  59     bool m_runIgnored;
  60
  61     std::queue<std::string> m_failReason;
  62
  63   public:
  64     TestRunner() :
  65         m_currentTestCase(nullptr)
  66       , m_terminate(false)
  67       , m_allowChildLogs(false)
  68       , m_deferDeepness(0U)
  69       , m_firstDeferredExceptionType(DeferredExceptionType::DEFERRED_FAILED)
  70     {}
  71
  72     void beginPerformanceTestTime(std::chrono::system_clock::duration maxTimeInMicroseconds);
  73     void endPerformanceTestTime();
  74     void getCurrentTestCasePerformanceResult(bool& isPerformanceTest,
  75                                              std::chrono::system_clock::duration& result,
  76                                              std::chrono::system_clock::duration& resultMax);
  77     void setCurrentTestCasePerformanceResult(bool isPerformanceTest,
  78                                              std::chrono::system_clock::duration result,
  79                                              std::chrono::system_clock::duration resultMax);
  80
  81     void addFailReason(const std::string &reason);
  82
  83     typedef void (*TestCase)();
  84
  85   private:
  86     struct TestCaseStruct
  87     {
  88         std::string name;
  89         TestCase proc;
  90
  91         bool m_isPerformanceTest;
  92         std::chrono::system_clock::time_point m_performanceTestStartTime;
  93         std::chrono::system_clock::duration m_performanceTestDurationTime;
  94         std::chrono::system_clock::duration m_performanceMaxTime;
  95
  96         bool operator <(const TestCaseStruct &other) const
  97         {
  98             return name < other.name;
  99         }
 100
 101         bool operator ==(const TestCaseStruct &other) const
 102         {
 103             return name == other.name;
 104         }
 105
 106         TestCaseStruct(const std::string &n, TestCase p) :
 107             name(n),
 108             proc(p),
 109             m_isPerformanceTest(false)
 110         {}
 111     };
 112
 113     typedef std::list<TestCaseStruct> TestCaseStructList;
 114     typedef std::map<std::string, TestCaseStructList> TestCaseGroupMap;
 115     TestCaseGroupMap m_testGroups;
 116
 117     TestCaseStruct * m_currentTestCase;
 118
 119     typedef std::set<std::string> SelectedTestNameSet;
 120     SelectedTestNameSet m_selectedTestNamesSet;
 121     typedef std::set<std::string> SelectedTestGroupSet;
 122     SelectedTestGroupSet m_selectedTestGroupSet;
 123     std::string m_currentGroup;
 124
 125     DPL::Atomic m_totalAssertions;
 126
 127     // Terminate without any logs.
 128     // Some test requires to call fork function.
 129     // Child process must not produce any logs and should die quietly.
 130     bool m_terminate;
 131     bool m_allowChildLogs;
 132
 133     void Banner();
 134     void InvalidArgs(const std::string& message = "Invalid arguments!");
 135     void Usage();
 136
 137     bool filterGroupsByXmls(const std::vector<std::string> & files);
 138     bool filterByXML(std::map<std::string, bool> & casesMap);
 139     void normalizeXMLTag(std::string& str, const std::string& testcase);
 140
 141     enum Status { FAILED, IGNORED, PASS };
 142
 143     Status RunTestCase(const TestCaseStruct& testCase);
 144
 145     void setCurrentTestCase(TestCaseStruct* testCase);
 146     TestCaseStruct *getCurrentTestCase();
 147
 148     void RunTests();
 149
 150     std::string getConcatedFailReason(const std::string &reason);
 151
 152     void CollectResult(const std::string& id,
 153                        const TestResultsCollectorBase::FailStatus status
 154                            = TestResultsCollectorBase::FailStatus::NONE,
 155                        const std::string& reason = std::string(),
 156                        const bool& isPerformanceTest = false,
 157                        const std::chrono::system_clock::duration& performanceTime = std::chrono::microseconds::zero(),
 158                        const std::chrono::system_clock::duration& performanceMaxTime = std::chrono::microseconds::zero());
 159
 160   public:
 161     void MarkAssertion();
 162
 163     void RegisterTest(const char *testName, TestCase proc);
 164     void InitGroup(const char* name);
 165
 166     int ExecTestRunner(int argc, char *argv[]);
 167     typedef std::vector<std::string> ArgsList;
 168     int ExecTestRunner(ArgsList args);
 169     bool getRunIgnored() const;
 170     // The runner will terminate as soon as possible (after current test).
 171     void Terminate();
 172     bool GetAllowChildLogs();
 173
 174     void deferFailedException(const DPL::Test::TestFailed &ex);
 175     void deferIgnoredException(const DPL::Test::TestIgnored &ex);
 176     void deferBegin();
 177     void deferEnd();
 178
 179 private:
 180     std::vector<std::string> m_deferredExceptionsMessages;
 181     std::size_t m_deferDeepness;
 182     enum DeferredExceptionType {
 183         DEFERRED_FAILED,
 184         DEFERRED_IGNORED,
 185     } m_firstDeferredExceptionType;
 186     DPL::Test::TestFailed m_firstDeferredFail;
 187     DPL::Test::TestIgnored m_firstDeferredIgnore;
 188 };
 189
 190 typedef DPL::Singleton<TestRunner> TestRunnerSingleton;
 191 }
 192 } // namespace DPL
 193
 194 #define RUNNER_TEST_GROUP_INIT(GroupName)                                 \
 195     static int Static##GroupName##Init()                                  \
 196     {                                                                     \
 197         DPL::Test::TestRunnerSingleton::Instance().InitGroup(#GroupName); \
 198         return 0;                                                         \
 199     }                                                                     \
 200     const int DPL_UNUSED Static##GroupName##InitVar =                     \
 201         Static##GroupName##Init();
 202
 203 #define RUNNER_TEST(Proc)                                                      \
 204     void Proc();                                                               \
 205     static int Static##Proc##Init()                                            \
 206     {                                                                          \
 207         DPL::Test::TestRunnerSingleton::Instance().RegisterTest(#Proc, &Proc); \
 208         return 0;                                                              \
 209     }                                                                          \
 210     const int DPL_UNUSED Static##Proc##InitVar = Static##Proc##Init();         \
 211     void Proc()
 212
 213
 214 /**
 215  * ASSERT MACROS
 216  *
 217  * Use them to create assertions in test cases. To do that put them inside test
 218  * body. Failing assertion indicates failing test.
 219  */
 220
 221 #define RUNNER_ASSERT_MSG(test, message)                                              \
 222     do                                                                                \
 223     {                                                                                 \
 224         DPL::Test::TestRunnerSingleton::Instance().MarkAssertion();                   \
 225                                                                                       \
 226         if (!(test))                                                                  \
 227         {                                                                             \
 228             std::ostringstream assertMsg;                                             \
 229             assertMsg << message << DPL::gdbbacktrace();                              \
 230             DPL::Test::TestFailed e(#test,                                            \
 231                                     __FILE__,                                         \
 232                                     __LINE__,                                         \
 233                                     assertMsg.str());                                 \
 234             if (!std::uncaught_exception())                                           \
 235                 throw e;                                                              \
 236             DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
 237         }                                                                             \
 238     } while (0)
 239
 240 #define RUNNER_ASSERT_ERRNO_MSG(test, message)                                        \
 241     do                                                                                \
 242     {                                                                                 \
 243         DPL::Test::TestRunnerSingleton::Instance().MarkAssertion();                   \
 244                                                                                       \
 245         if (!(test))                                                                  \
 246         {                                                                             \
 247             const char *err = strerror(errno);                                        \
 248             std::ostringstream assertMsg;                                             \
 249             assertMsg << message;                                                     \
 250             if (!assertMsg.str().empty())                                             \
 251                 assertMsg << ". ";                                                    \
 252             assertMsg << err << DPL::gdbbacktrace();                                  \
 253             DPL::Test::TestFailed e(#test,                                            \
 254                                     __FILE__,                                         \
 255                                     __LINE__,                                         \
 256                                     assertMsg.str());                                 \
 257             if (!std::uncaught_exception())                                           \
 258                 throw e;                                                              \
 259             DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
 260         }                                                                             \
 261     } while (0)
 262
 263 #define RUNNER_ASSERT_ERRNO(test) \
 264     RUNNER_ASSERT_ERRNO_MSG(test, "")
 265
 266 #define RUNNER_FAIL_MSG(message) \
 267     RUNNER_ASSERT_MSG(false, message)
 268
 269 #define RUNNER_ASSERT(test) \
 270     RUNNER_ASSERT_MSG(test, "")
 271
 272 /**
 273  * IGNORE MACRO
 274  *
 275  * When test reaches this macro call, its furhter code will be ignored.
 276  * To ignore whole test, put this macro call at the beginning of this tests
 277  * body.
 278  */
 279
 280 #define RUNNER_IGNORED_MSG(message)                    \
 281     do                                                 \
 282     {                                                  \
 283         std::ostringstream assertMsg;                  \
 284         assertMsg << message;                          \
 285         throw DPL::Test::TestIgnored(assertMsg.str()); \
 286     } while (0)
 287
 288 /**
 289  * PERF MACROS
 290  *
 291  * Use these macros to do the time measurement. The first macro will start time measurement,
 292  * the second will gather the result. These macros can be used only once per test-case.
 293  * The result of time measurement will be displayed only if the test will pass.
 294  * Notice that these macros will work only if will be used in parent process. If these
 295  * macros will be used in child process then there will be no time measure results printed.
 296  * This macro in multiprocess tests has effect only if used in parent process. This macro
 297  * used in child process in multiprocess test has no effect.
 298  * The precision of measurement is 1 microsecond - the smallest time value that can be
 299  * measured is 0.000001s.
 300  * The time measure results will be printed only specific output format:
 301  *     - text
 302  *     - html
 303  *     - xml
 304  */
 305
 306 #define RUNNER_PERF_TEST_BEGIN(maxTime)                                                \
 307     do {                                                                               \
 308         DPL::Test::TestRunnerSingleton::Instance().beginPerformanceTestTime(           \
 309             std::chrono::microseconds{static_cast<long long int>(maxTime*1000000.0)}); \
 310     } while (0)
 311
 312 #define RUNNER_PERF_TEST_END()                                                \
 313     do {                                                                      \
 314         DPL::Test::TestRunnerSingleton::Instance().endPerformanceTestTime();  \
 315     } while (0)
 316
 317 /**
 318  * MSG MACROS
 319  *
 320  * Use these macros to print error messages during test run time
 321  */
 322
 323 #define RUNNER_ERROR_MSG(message)                             \
 324     do {                                                      \
 325         std::cerr << DPL::Colors::Text::RED_BEGIN << message  \
 326                   << DPL::Colors::Text::RED_END << std::endl; \
 327     } while (0)
 328
 329 /**
 330  * DEFER MACROS
 331  *
 332  * Use them to defer fails and ignores in test cases.
 333  * Some code constructions disallow to throw. Such places can be surrounded
 334  * with RUNNER_DEFER_SCOPE macro. RUNNER_DEFER_TRYCATCH macro can be used to catch possibly thrown
 335  * exceptions within such scope. Possibly catched exceptions will be rethrown
 336  * when leaving RUNNER_DEFER_SCOPE scope.
 337  * Macros can be safely nested.
 338  */
 339
 340
 341 #define RUNNER_DEFER_TRYCATCH(scope)                                              \
 342     do {                                                                          \
 343         try                                                                       \
 344         {                                                                         \
 345             scope                                                                 \
 346         }                                                                         \
 347         catch (const DPL::Test::TestFailed &ex)                                   \
 348         {                                                                         \
 349             DPL::Test::TestRunnerSingleton::Instance().deferFailedException(ex);  \
 350         }                                                                         \
 351         catch (const DPL::Test::TestIgnored &ex)                                  \
 352         {                                                                         \
 353             DPL::Test::TestRunnerSingleton::Instance().deferIgnoredException(ex); \
 354         }                                                                         \
 355     } while (0)                                                                   \
 356
 357 #define RUNNER_DEFER_SCOPE(scope)                                \
 358     do {                                                         \
 359         DPL::Test::TestRunnerSingleton::Instance().deferBegin(); \
 360         scope                                                    \
 361         DPL::Test::TestRunnerSingleton::Instance().deferEnd();   \
 362     } while (0)                                                  \
 363
 364 #endif // DPL_TEST_RUNNER_H