src/framework/include/dpl/test/test_runner.h

   1 /*
   2  * Copyright (c) 2011-2017 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.h
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @author      Marcin Niesluchowski (m.niesluchow@samsung.com)
  21  * @version     1.0
  22  * @brief       This file is the header file of test runner
  23  */
  24
  25 #ifndef DPL_TEST_RUNNER_H
  26 #define DPL_TEST_RUNNER_H
  27
  28 #include <cerrno>
  29 #include <chrono>
  30 #include <cstdlib>
  31 #include <cstring>
  32 #include <exception>
  33 #include <functional>
  34 #include <iostream>
  35 #include <map>
  36 #include <queue>
  37 #include <set>
  38 #include <sstream>
  39 #include <string>
  40 #include <vector>
  41 #include <memory>
  42
  43 #include <dpl/atomic.h>
  44 #include <dpl/availability.h>
  45 #include <dpl/colors.h>
  46 #include <dpl/gdbbacktrace.h>
  47 #include <dpl/singleton.h>
  48 #include <dpl/test/performance_result.h>
  49 #include <dpl/test/test_group.h>
  50 #include <dpl/test/test_case_extended.h>
  51 #include <dpl/test/test_exception.h>
  52 #include <dpl/test/test_failed.h>
  53 #include <dpl/test/test_ignored.h>
  54 #include <dpl/test/test_result.h>
  55 #include <dpl/test/test_results_collector.h>
  56
  57 namespace DPL {
  58 namespace Test {
  59 class TestRunner
  60 {
  61     typedef std::map<std::string, TestResultsCollectorBasePtr>
  62     TestResultsCollectors;
  63     TestResultsCollectors m_collectors;
  64
  65     std::string m_startTestId;
  66     bool m_runIgnored;
  67
  68     std::queue<std::string> m_failReason;
  69
  70   public:
  71     TestRunner() :
  72         m_currentTestCase(nullptr)
  73       , m_terminate(false)
  74       , m_allowChildLogs(false)
  75       , m_deferDeepness(0U)
  76       , m_firstDeferredExceptionType(DeferredExceptionType::DEFERRED_FAILED)
  77     {}
  78
  79     void beginPerformance(std::chrono::system_clock::duration maxDurationInMicroseconds);
  80     void endPerformance();
  81     void setCurrentTestCasePerformanceResult(const PerformanceResultPtr &performance);
  82     ConstPerformanceResultPtr getCurrentTestCasePerformanceResult();
  83
  84     void addFailReason(const std::string &reason);
  85
  86     ~TestRunner();
  87
  88   private:
  89     typedef std::map<std::string, TestGroup*> TestCaseGroupMap;
  90
  91     TestCaseGroupMap m_testGroups;
  92
  93     TestCasePtr m_currentTestCase;
  94
  95     TestGroupPtr m_currentGroup;
  96
  97     // Terminate without any logs.
  98     // Some test requires to call fork function.
  99     // Child process must not produce any logs and should die quietly.
 100     bool m_terminate;
 101     bool m_allowChildLogs;
 102
 103     void Banner();
 104     void InvalidArgs(const std::string& message = "Invalid arguments!");
 105     void Usage();
 106
 107     bool filterGroupsByXmls(const std::vector<std::string> & files);
 108     bool filterByXML(std::map<std::string, bool> & casesMap);
 109     void normalizeXMLTag(std::string& str, const std::string& testcase);
 110
 111     void RunTestCase(TestCasePtr testCase);
 112
 113     void setCurrentTestCase(TestCasePtr testCase);
 114     TestCasePtr getCurrentTestCase();
 115
 116     void RunTests();
 117
 118     std::string getConcatedFailReason(const std::string &reason);
 119
 120     void CollectResult(const std::string& id, const TestResult &result);
 121
 122   public:
 123     void InitGroup(const char* name, TestGroup* group = nullptr);
 124     void RegisterTest(TestCase* testCase);
 125     int ExecTestRunner(int argc, char *argv[]);
 126     typedef std::vector<std::string> ArgsList;
 127     int ExecTestRunner(ArgsList args);
 128     // The runner will terminate as soon as possible (after current test).
 129     void Terminate();
 130     bool GetAllowChildLogs();
 131     bool GetRunIgnored() const;
 132
 133     void deferFailedException(const DPL::Test::TestFailed &ex);
 134     void deferIgnoredException(const DPL::Test::TestIgnored &ex);
 135     void deferBegin();
 136     void deferEnd();
 137
 138 private:
 139     std::vector<std::string> m_deferredExceptionsMessages;
 140     std::size_t m_deferDeepness;
 141     enum DeferredExceptionType {
 142         DEFERRED_FAILED,
 143         DEFERRED_IGNORED,
 144     } m_firstDeferredExceptionType;
 145     DPL::Test::TestFailed m_firstDeferredFail;
 146     DPL::Test::TestIgnored m_firstDeferredIgnore;
 147 };
 148
 149 typedef DPL::Singleton<TestRunner> TestRunnerSingleton;
 150
 151 TestResult::FailStatus TryCatch(const std::function<void(void)> &func, std::string &reason);
 152
 153 /*
 154  * Test execution in multiple environments.
 155  *
 156  * To execute the same test in different environments provide an environment classes CLASS1, CLASS2,.... Classes have to
 157  * provide init() & finish() methods to be called before and after execution in each environment.
 158  * They should also provide suffix() method returning test case name suffix for given environment.
 159  *
 160  * Define the test as follows
 161  *
 162  * RUNNER_TEST_MULTIPLE(my_test, CLASS1, CLASS2,...)
 163  * {
 164  *     // do the test stuff
 165  * }
 166  *
 167  * For each class C the order of operation will be as follows:
 168  * - construct C (default constructor)
 169  * - call init() on C object
 170  * - call test case function
 171  * - call finish() on C object
 172  * - destroy the C object
 173  */
 174
 175 template <typename ...Args>
 176 struct StaticProc;
 177
 178 template <typename First>
 179 struct StaticProc<First> {
 180 protected:
 181     static void Internal(const std::string& name, const TestFnType& proc) {
 182         TestRunnerSingleton::Instance().RegisterTest(
 183                 new TestCaseExt2<First>(name + First::suffix(), proc));
 184     }
 185 };
 186
 187 template <typename First, typename ...Args>
 188 struct StaticProc<First, Args...> : public StaticProc<First>, public StaticProc<Args...> {
 189     static int Init(const std::string& name, const TestFnType& proc) {
 190         StaticProc<First, Args...>::Internal(name, proc);
 191         return 0;
 192     }
 193 protected:
 194     static void Internal(const std::string& name, const TestFnType& proc) {
 195         StaticProc<First>::Internal(name, proc);
 196         StaticProc<Args...>::Internal(name, proc);
 197     }
 198 };
 199
 200 } // Test
 201 } // namespace DPL
 202
 203 #define RUNNER_TEST_GROUP_INIT_ENV(GroupName, GroupEnv)                                 \
 204     static int Static##GroupName##Init()                                                \
 205     {                                                                                   \
 206         DPL::Test::TestRunnerSingleton::Instance().InitGroup(#GroupName, new GroupEnv); \
 207         return 0;                                                                       \
 208     }                                                                                   \
 209     const int DPL_UNUSED Static##GroupName##InitVar =                                   \
 210         Static##GroupName##Init();
 211
 212 #define RUNNER_TEST_GROUP_INIT(GroupName)                                 \
 213     static int Static##GroupName##Init()                                  \
 214     {                                                                     \
 215         DPL::Test::TestRunnerSingleton::Instance().InitGroup(#GroupName); \
 216         return 0;                                                         \
 217     }                                                                     \
 218     const int DPL_UNUSED Static##GroupName##InitVar =                     \
 219         Static##GroupName##Init();
 220
 221 #define RUNNER_TEST(Proc, ...)                                                          \
 222     void Proc(std::tuple<__VA_ARGS__> &optionalArgsTuple);                              \
 223     static int Static##Proc##Init()                                                     \
 224     {                                                                                   \
 225         DPL::Test::TestRunnerSingleton::Instance().RegisterTest(                        \
 226             new DPL::Test::TestCaseExtended<__VA_ARGS__>(#Proc, &Proc));                \
 227         return 0;                                                                       \
 228     }                                                                                   \
 229     const int DPL_UNUSED Static##Proc##InitVar = Static##Proc##Init();                  \
 230     void Proc(std::tuple<__VA_ARGS__> &optionalArgsTuple DPL_UNUSED)
 231
 232
 233 #define RUNNER_TEST_MULTIPLE(Proc, ...)                                                                  \
 234     void Proc();                                                                                         \
 235     const int DPL_UNUSED Static##Proc##InitVar = DPL::Test::StaticProc<__VA_ARGS__>::Init(#Proc, &Proc); \
 236     void Proc()
 237
 238 /**
 239  * ASSERT MACROS
 240  *
 241  * Use them to create assertions in test cases. To do that put them inside test
 242  * body. Failing assertion indicates failing test.
 243  */
 244
 245 #define RUNNER_ASSERT_MSG(test, message)                                              \
 246     do                                                                                \
 247     {                                                                                 \
 248         if (!(test))                                                                  \
 249         {                                                                             \
 250             std::ostringstream assertMsg;                                             \
 251             assertMsg << message << DPL::gdbbacktrace();                              \
 252             DPL::Test::TestFailed e(#test,                                            \
 253                                     __FILE__,                                         \
 254                                     __LINE__,                                         \
 255                                     assertMsg.str());                                 \
 256             throw e;                                                                  \
 257         }                                                                             \
 258     } while (0)
 259
 260 #define RUNNER_ASSERT_ERRNO_MSG(test, message)                                        \
 261     do                                                                                \
 262     {                                                                                 \
 263         if (!(test))                                                                  \
 264         {                                                                             \
 265             const char *err = strerror(errno);                                        \
 266             std::ostringstream assertMsg;                                             \
 267             assertMsg << message;                                                     \
 268             if (!assertMsg.str().empty())                                             \
 269                 assertMsg << ". ";                                                    \
 270             assertMsg << err << DPL::gdbbacktrace();                                  \
 271             DPL::Test::TestFailed e(#test,                                            \
 272                                     __FILE__,                                         \
 273                                     __LINE__,                                         \
 274                                     assertMsg.str());                                 \
 275             throw e;                                                                  \
 276         }                                                                             \
 277     } while (0)
 278
 279 #define RUNNER_ASSERT_ERRNO(test) \
 280     RUNNER_ASSERT_ERRNO_MSG(test, "")
 281
 282 #define RUNNER_FAIL_MSG(message) \
 283     RUNNER_ASSERT_MSG(false, message)
 284
 285 #define RUNNER_ASSERT(test) \
 286     RUNNER_ASSERT_MSG(test, "")
 287
 288 /**
 289  * IGNORE MACRO
 290  *
 291  * When test reaches this macro call, its further code will be ignored.
 292  * To ignore whole test, put this macro call at the beginning of this tests
 293  * body.
 294  */
 295
 296 #define RUNNER_IGNORED_MSG(message)                                         \
 297     do                                                                      \
 298     {                                                                       \
 299         if (DPL::Test::TestRunnerSingleton::Instance().GetRunIgnored()) {   \
 300             break;                                                          \
 301         }                                                                   \
 302         std::ostringstream assertMsg;                                       \
 303         assertMsg << message;                                               \
 304         throw DPL::Test::TestIgnored(assertMsg.str());                      \
 305     } while (0)
 306
 307 /**
 308  * PERF MACROS
 309  *
 310  * Use these macros to do the time measurement. The first macro will start time measurement,
 311  * the second will gather the result. These macros can be used only once per test-case.
 312  * The result of time measurement will be displayed only if the test will pass.
 313  * Notice that these macros will work only if will be used in parent process. If these
 314  * macros will be used in child process then there will be no time measure results printed.
 315  * This macro in multiprocess tests has effect only if used in parent process. This macro
 316  * used in child process in multiprocess test has no effect.
 317  * The precision of measurement is 1 microsecond - the smallest time value that can be
 318  * measured is 0.000001s.
 319  * The time measure results will be printed only specific output format:
 320  *     - text
 321  *     - html
 322  *     - xml
 323  */
 324
 325 #define RUNNER_PERF_TEST_BEGIN(maxTime)                                                \
 326     do {                                                                               \
 327         DPL::Test::TestRunnerSingleton::Instance().beginPerformance(                   \
 328             std::chrono::microseconds{static_cast<long long int>(maxTime*1000000.0)}); \
 329     } while (0)
 330
 331 #define RUNNER_PERF_TEST_END()                                       \
 332     do {                                                             \
 333         DPL::Test::TestRunnerSingleton::Instance().endPerformance(); \
 334     } while (0)
 335
 336 /**
 337  * MSG MACROS
 338  *
 339  * Use these macros to print error messages during test run time
 340  */
 341
 342 #define RUNNER_ERROR_MSG(message)                             \
 343     do {                                                      \
 344         std::cerr << DPL::Colors::Text::RED_BEGIN << message  \
 345                   << DPL::Colors::Text::RED_END << std::endl; \
 346     } while (0)
 347
 348 /**
 349  * DEFER MACROS
 350  *
 351  * Use them to defer fails and ignores in test cases.
 352  * Some code constructions disallow to throw. Such places can be surrounded
 353  * with RUNNER_DEFER_SCOPE macro. RUNNER_DEFER_TRYCATCH macro can be used to catch possibly thrown
 354  * exceptions within such scope. Possibly catched exceptions will be rethrown
 355  * when leaving RUNNER_DEFER_SCOPE scope.
 356  * Macros can be safely nested.
 357  */
 358
 359
 360 #define RUNNER_DEFER_TRYCATCH(scope)                                              \
 361     do {                                                                          \
 362         try                                                                       \
 363         {                                                                         \
 364             scope                                                                 \
 365         }                                                                         \
 366         catch (const DPL::Test::TestFailed &ex)                                   \
 367         {                                                                         \
 368             DPL::Test::TestRunnerSingleton::Instance().deferFailedException(ex);  \
 369         }                                                                         \
 370         catch (const DPL::Test::TestIgnored &ex)                                  \
 371         {                                                                         \
 372             DPL::Test::TestRunnerSingleton::Instance().deferIgnoredException(ex); \
 373         }                                                                         \
 374     } while (0)                                                                   \
 375
 376 #define RUNNER_DEFER_SCOPE(scope)                                \
 377     do {                                                         \
 378         DPL::Test::TestRunnerSingleton::Instance().deferBegin(); \
 379         scope                                                    \
 380         DPL::Test::TestRunnerSingleton::Instance().deferEnd();   \
 381     } while (0)                                                  \
 382
 383 #endif // DPL_TEST_RUNNER_H