src/framework/include/dpl/test/test_runner.h

   1 /*
   2  * Copyright (c) 2011-2015 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.h
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @author      Marcin Niesluchowski (m.niesluchow@samsung.com)
  21  * @version     1.0
  22  * @brief       This file is the header file of test runner
  23  */
  24
  25 #ifndef DPL_TEST_RUNNER_H
  26 #define DPL_TEST_RUNNER_H
  27
  28 #include <chrono>
  29 #include <cstdlib>
  30 #include <cstring>
  31 #include <exception>
  32 #include <functional>
  33 #include <iostream>
  34 #include <list>
  35 #include <map>
  36 #include <queue>
  37 #include <set>
  38 #include <sstream>
  39 #include <string>
  40 #include <vector>
  41
  42 #include <dpl/atomic.h>
  43 #include <dpl/availability.h>
  44 #include <dpl/colors.h>
  45 #include <dpl/gdbbacktrace.h>
  46 #include <dpl/singleton.h>
  47 #include <dpl/test/performance_result.h>
  48 #include <dpl/test/test_case.h>
  49 #include <dpl/test/test_case_extended.h>
  50 #include <dpl/test/test_exception.h>
  51 #include <dpl/test/test_failed.h>
  52 #include <dpl/test/test_ignored.h>
  53 #include <dpl/test/test_result.h>
  54 #include <dpl/test/test_results_collector.h>
  55
  56 namespace DPL {
  57 namespace Test {
  58 class TestRunner
  59 {
  60     typedef std::map<std::string, TestResultsCollectorBasePtr>
  61     TestResultsCollectors;
  62     TestResultsCollectors m_collectors;
  63
  64     std::string m_startTestId;
  65     bool m_runIgnored;
  66
  67     std::queue<std::string> m_failReason;
  68
  69   public:
  70     TestRunner() :
  71         m_currentTestCase(nullptr)
  72       , m_terminate(false)
  73       , m_allowChildLogs(false)
  74       , m_deferDeepness(0U)
  75       , m_firstDeferredExceptionType(DeferredExceptionType::DEFERRED_FAILED)
  76     {}
  77
  78     void beginPerformance(std::chrono::system_clock::duration maxDurationInMicroseconds);
  79     void endPerformance();
  80     void setCurrentTestCasePerformanceResult(const PerformanceResultPtr &performance);
  81     ConstPerformanceResultPtr getCurrentTestCasePerformanceResult();
  82
  83     void addFailReason(const std::string &reason);
  84
  85     ~TestRunner();
  86
  87   private:
  88     typedef std::list<TestCasePtr> TestCaseList;
  89     typedef std::map<std::string, TestCaseList> TestCaseGroupMap;
  90     typedef std::set<TestCasePtr> TestCaseSet;
  91
  92     TestCaseGroupMap m_testGroups;
  93     TestCaseSet m_testCaseSet;
  94
  95     TestCasePtr m_currentTestCase;
  96
  97
  98     std::string m_currentGroup;
  99
 100     // Terminate without any logs.
 101     // Some test requires to call fork function.
 102     // Child process must not produce any logs and should die quietly.
 103     bool m_terminate;
 104     bool m_allowChildLogs;
 105
 106     void Banner();
 107     void InvalidArgs(const std::string& message = "Invalid arguments!");
 108     void Usage();
 109
 110     bool filterGroupsByXmls(const std::vector<std::string> & files);
 111     bool filterByXML(std::map<std::string, bool> & casesMap);
 112     void normalizeXMLTag(std::string& str, const std::string& testcase);
 113
 114     void RunTestCase(TestCasePtr testCase);
 115
 116     void setCurrentTestCase(TestCasePtr testCase);
 117     TestCasePtr getCurrentTestCase();
 118
 119     void RunTests();
 120
 121     std::string getConcatedFailReason(const std::string &reason);
 122
 123     void CollectResult(const std::string& id, const TestResult &result);
 124
 125   public:
 126     void InitGroup(const char* name);
 127     void RegisterTest(TestCasePtr testCase);
 128     int ExecTestRunner(int argc, char *argv[]);
 129     typedef std::vector<std::string> ArgsList;
 130     int ExecTestRunner(ArgsList args);
 131     // The runner will terminate as soon as possible (after current test).
 132     void Terminate();
 133     bool GetAllowChildLogs();
 134
 135     void deferFailedException(const DPL::Test::TestFailed &ex);
 136     void deferIgnoredException(const DPL::Test::TestIgnored &ex);
 137     void deferBegin();
 138     void deferEnd();
 139
 140 private:
 141     std::vector<std::string> m_deferredExceptionsMessages;
 142     std::size_t m_deferDeepness;
 143     enum DeferredExceptionType {
 144         DEFERRED_FAILED,
 145         DEFERRED_IGNORED,
 146     } m_firstDeferredExceptionType;
 147     DPL::Test::TestFailed m_firstDeferredFail;
 148     DPL::Test::TestIgnored m_firstDeferredIgnore;
 149 };
 150
 151 typedef DPL::Singleton<TestRunner> TestRunnerSingleton;
 152
 153 TestResult::FailStatus TryCatch(const std::function<void(void)> &func, std::string &reason);
 154
 155 }
 156 } // namespace DPL
 157
 158 #define RUNNER_TEST_GROUP_INIT(GroupName)                                 \
 159     static int Static##GroupName##Init()                                  \
 160     {                                                                     \
 161         DPL::Test::TestRunnerSingleton::Instance().InitGroup(#GroupName); \
 162         return 0;                                                         \
 163     }                                                                     \
 164     const int DPL_UNUSED Static##GroupName##InitVar =                     \
 165         Static##GroupName##Init();
 166
 167 #define RUNNER_TEST(Proc, ...)                                                          \
 168     void Proc(std::tuple<__VA_ARGS__> &optionalArgsTuple);                              \
 169     static int Static##Proc##Init()                                                     \
 170     {                                                                                   \
 171         DPL::Test::TestRunnerSingleton::Instance().RegisterTest(                        \
 172             new DPL::Test::TestCaseExtended<__VA_ARGS__>(#Proc, &Proc));                \
 173         return 0;                                                                       \
 174     }                                                                                   \
 175     const int DPL_UNUSED Static##Proc##InitVar = Static##Proc##Init();                  \
 176     void Proc(std::tuple<__VA_ARGS__> &optionalArgsTuple DPL_UNUSED)
 177
 178 /**
 179  * ASSERT MACROS
 180  *
 181  * Use them to create assertions in test cases. To do that put them inside test
 182  * body. Failing assertion indicates failing test.
 183  */
 184
 185 #define RUNNER_ASSERT_MSG(test, message)                                              \
 186     do                                                                                \
 187     {                                                                                 \
 188         if (!(test))                                                                  \
 189         {                                                                             \
 190             std::ostringstream assertMsg;                                             \
 191             assertMsg << message << DPL::gdbbacktrace();                              \
 192             DPL::Test::TestFailed e(#test,                                            \
 193                                     __FILE__,                                         \
 194                                     __LINE__,                                         \
 195                                     assertMsg.str());                                 \
 196             if (!std::uncaught_exception())                                           \
 197                 throw e;                                                              \
 198             DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
 199         }                                                                             \
 200     } while (0)
 201
 202 #define RUNNER_ASSERT_ERRNO_MSG(test, message)                                        \
 203     do                                                                                \
 204     {                                                                                 \
 205         if (!(test))                                                                  \
 206         {                                                                             \
 207             const char *err = strerror(errno);                                        \
 208             std::ostringstream assertMsg;                                             \
 209             assertMsg << message;                                                     \
 210             if (!assertMsg.str().empty())                                             \
 211                 assertMsg << ". ";                                                    \
 212             assertMsg << err << DPL::gdbbacktrace();                                  \
 213             DPL::Test::TestFailed e(#test,                                            \
 214                                     __FILE__,                                         \
 215                                     __LINE__,                                         \
 216                                     assertMsg.str());                                 \
 217             if (!std::uncaught_exception())                                           \
 218                 throw e;                                                              \
 219             DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
 220         }                                                                             \
 221     } while (0)
 222
 223 #define RUNNER_ASSERT_ERRNO(test) \
 224     RUNNER_ASSERT_ERRNO_MSG(test, "")
 225
 226 #define RUNNER_FAIL_MSG(message) \
 227     RUNNER_ASSERT_MSG(false, message)
 228
 229 #define RUNNER_ASSERT(test) \
 230     RUNNER_ASSERT_MSG(test, "")
 231
 232 /**
 233  * IGNORE MACRO
 234  *
 235  * When test reaches this macro call, its furhter code will be ignored.
 236  * To ignore whole test, put this macro call at the beginning of this tests
 237  * body.
 238  */
 239
 240 #define RUNNER_IGNORED_MSG(message)                    \
 241     do                                                 \
 242     {                                                  \
 243         std::ostringstream assertMsg;                  \
 244         assertMsg << message;                          \
 245         throw DPL::Test::TestIgnored(assertMsg.str()); \
 246     } while (0)
 247
 248 /**
 249  * PERF MACROS
 250  *
 251  * Use these macros to do the time measurement. The first macro will start time measurement,
 252  * the second will gather the result. These macros can be used only once per test-case.
 253  * The result of time measurement will be displayed only if the test will pass.
 254  * Notice that these macros will work only if will be used in parent process. If these
 255  * macros will be used in child process then there will be no time measure results printed.
 256  * This macro in multiprocess tests has effect only if used in parent process. This macro
 257  * used in child process in multiprocess test has no effect.
 258  * The precision of measurement is 1 microsecond - the smallest time value that can be
 259  * measured is 0.000001s.
 260  * The time measure results will be printed only specific output format:
 261  *     - text
 262  *     - html
 263  *     - xml
 264  */
 265
 266 #define RUNNER_PERF_TEST_BEGIN(maxTime)                                                \
 267     do {                                                                               \
 268         DPL::Test::TestRunnerSingleton::Instance().beginPerformance(                   \
 269             std::chrono::microseconds{static_cast<long long int>(maxTime*1000000.0)}); \
 270     } while (0)
 271
 272 #define RUNNER_PERF_TEST_END()                                       \
 273     do {                                                             \
 274         DPL::Test::TestRunnerSingleton::Instance().endPerformance(); \
 275     } while (0)
 276
 277 /**
 278  * MSG MACROS
 279  *
 280  * Use these macros to print error messages during test run time
 281  */
 282
 283 #define RUNNER_ERROR_MSG(message)                             \
 284     do {                                                      \
 285         std::cerr << DPL::Colors::Text::RED_BEGIN << message  \
 286                   << DPL::Colors::Text::RED_END << std::endl; \
 287     } while (0)
 288
 289 /**
 290  * DEFER MACROS
 291  *
 292  * Use them to defer fails and ignores in test cases.
 293  * Some code constructions disallow to throw. Such places can be surrounded
 294  * with RUNNER_DEFER_SCOPE macro. RUNNER_DEFER_TRYCATCH macro can be used to catch possibly thrown
 295  * exceptions within such scope. Possibly catched exceptions will be rethrown
 296  * when leaving RUNNER_DEFER_SCOPE scope.
 297  * Macros can be safely nested.
 298  */
 299
 300
 301 #define RUNNER_DEFER_TRYCATCH(scope)                                              \
 302     do {                                                                          \
 303         try                                                                       \
 304         {                                                                         \
 305             scope                                                                 \
 306         }                                                                         \
 307         catch (const DPL::Test::TestFailed &ex)                                   \
 308         {                                                                         \
 309             DPL::Test::TestRunnerSingleton::Instance().deferFailedException(ex);  \
 310         }                                                                         \
 311         catch (const DPL::Test::TestIgnored &ex)                                  \
 312         {                                                                         \
 313             DPL::Test::TestRunnerSingleton::Instance().deferIgnoredException(ex); \
 314         }                                                                         \
 315     } while (0)                                                                   \
 316
 317 #define RUNNER_DEFER_SCOPE(scope)                                \
 318     do {                                                         \
 319         DPL::Test::TestRunnerSingleton::Instance().deferBegin(); \
 320         scope                                                    \
 321         DPL::Test::TestRunnerSingleton::Instance().deferEnd();   \
 322     } while (0)                                                  \
 323
 324 #endif // DPL_TEST_RUNNER_H