src/framework/include/dpl/test/test_runner.h

   1 /*
   2  * Copyright (c) 2011-2015 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.h
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @version     1.0
  21  * @brief       This file is the header file of test runner
  22  */
  23
  24 #ifndef DPL_TEST_RUNNER_H
  25 #define DPL_TEST_RUNNER_H
  26
  27 #include <chrono>
  28 #include <cstring>
  29 #include <exception>
  30 #include <iostream>
  31 #include <list>
  32 #include <map>
  33 #include <queue>
  34 #include <set>
  35 #include <sstream>
  36 #include <string>
  37 #include <vector>
  38
  39 #include <dpl/atomic.h>
  40 #include <dpl/availability.h>
  41 #include <dpl/colors.h>
  42 #include <dpl/gdbbacktrace.h>
  43 #include <dpl/singleton.h>
  44 #include <dpl/test/performance_result.h>
  45 #include <dpl/test/test_results_collector.h>
  46
  47 namespace DPL {
  48 namespace Test {
  49 class TestRunner
  50 {
  51     typedef std::map<std::string, TestResultsCollectorBasePtr>
  52     TestResultsCollectors;
  53     TestResultsCollectors m_collectors;
  54
  55     std::string m_startTestId;
  56     bool m_runIgnored;
  57
  58     std::queue<std::string> m_failReason;
  59
  60   public:
  61     TestRunner() :
  62         m_currentTestCase(nullptr)
  63       , m_terminate(false)
  64       , m_allowChildLogs(false)
  65     {}
  66
  67     void beginPerformance(std::chrono::system_clock::duration maxDurationInMicroseconds);
  68     void endPerformance();
  69     void setCurrentTestCasePerformanceResult(const PerformanceResultPtr &performance);
  70     ConstPerformanceResultPtr getCurrentTestCasePerformanceResult();
  71
  72     void addFailReason(const std::string &reason);
  73
  74     typedef void (*TestCase)();
  75
  76   private:
  77     struct TestCaseStruct
  78     {
  79         std::string name;
  80         TestCase proc;
  81         PerformanceResultPtr performance;
  82
  83         bool operator <(const TestCaseStruct &other) const
  84         {
  85             return name < other.name;
  86         }
  87
  88         bool operator ==(const TestCaseStruct &other) const
  89         {
  90             return name == other.name;
  91         }
  92
  93         TestCaseStruct(const std::string &n, TestCase p) :
  94             name(n),
  95             proc(p)
  96         {}
  97     };
  98
  99     typedef std::list<TestCaseStruct> TestCaseStructList;
 100     typedef std::map<std::string, TestCaseStructList> TestCaseGroupMap;
 101     TestCaseGroupMap m_testGroups;
 102
 103     TestCaseStruct * m_currentTestCase;
 104
 105     typedef std::set<std::string> SelectedTestNameSet;
 106     SelectedTestNameSet m_selectedTestNamesSet;
 107     typedef std::set<std::string> SelectedTestGroupSet;
 108     SelectedTestGroupSet m_selectedTestGroupSet;
 109     std::string m_currentGroup;
 110
 111     DPL::Atomic m_totalAssertions;
 112
 113     // Terminate without any logs.
 114     // Some test requires to call fork function.
 115     // Child process must not produce any logs and should die quietly.
 116     bool m_terminate;
 117     bool m_allowChildLogs;
 118
 119     void Banner();
 120     void InvalidArgs(const std::string& message = "Invalid arguments!");
 121     void Usage();
 122
 123     bool filterGroupsByXmls(const std::vector<std::string> & files);
 124     bool filterByXML(std::map<std::string, bool> & casesMap);
 125     void normalizeXMLTag(std::string& str, const std::string& testcase);
 126
 127     enum Status { FAILED, IGNORED, PASS };
 128
 129     Status RunTestCase(const TestCaseStruct& testCase);
 130
 131     void setCurrentTestCase(TestCaseStruct* testCase);
 132     TestCaseStruct *getCurrentTestCase();
 133
 134     void RunTests();
 135
 136     std::string getConcatedFailReason(const std::string &reason);
 137
 138     void CollectResult(const std::string& id,
 139                        const TestResultsCollectorBase::FailStatus status
 140                            = TestResultsCollectorBase::FailStatus::NONE,
 141                        const std::string& reason = std::string(),
 142                        const ConstPerformanceResultPtr &performance = nullptr);
 143   public:
 144     class TestFailed
 145     {
 146       private:
 147         std::string m_message;
 148
 149       public:
 150         TestFailed()
 151         {}
 152
 153         //! \brief Failed test message creator
 154         //!
 155         //! \param[in] aTest string for tested expression
 156         //! \param[in] aFile source file name
 157         //! \param[in] aLine source file line
 158         //! \param[in] aMessage error message
 159         TestFailed(const char* aTest,
 160                    const char* aFile,
 161                    int aLine,
 162                    const std::string &aMessage);
 163
 164         TestFailed(const std::string &message);
 165
 166         std::string GetMessage() const
 167         {
 168             return m_message;
 169         }
 170     };
 171
 172     class Ignored
 173     {
 174       private:
 175         std::string m_message;
 176
 177       public:
 178         Ignored()
 179         {}
 180
 181         Ignored(const std::string &message) :
 182             m_message(message)
 183         {}
 184
 185         std::string GetMessage() const
 186         {
 187             return m_message;
 188         }
 189     };
 190
 191     void MarkAssertion();
 192
 193     void RegisterTest(const char *testName, TestCase proc);
 194     void InitGroup(const char* name);
 195
 196     int ExecTestRunner(int argc, char *argv[]);
 197     typedef std::vector<std::string> ArgsList;
 198     int ExecTestRunner(ArgsList args);
 199     bool getRunIgnored() const;
 200     // The runner will terminate as soon as possible (after current test).
 201     void Terminate();
 202     bool GetAllowChildLogs();
 203 };
 204
 205 typedef DPL::Singleton<TestRunner> TestRunnerSingleton;
 206 }
 207 } // namespace DPL
 208
 209 #define RUNNER_TEST_GROUP_INIT(GroupName)                                 \
 210     static int Static##GroupName##Init()                                  \
 211     {                                                                     \
 212         DPL::Test::TestRunnerSingleton::Instance().InitGroup(#GroupName); \
 213         return 0;                                                         \
 214     }                                                                     \
 215     const int DPL_UNUSED Static##GroupName##InitVar =                     \
 216         Static##GroupName##Init();
 217
 218 #define RUNNER_TEST(Proc)                                                      \
 219     void Proc();                                                               \
 220     static int Static##Proc##Init()                                            \
 221     {                                                                          \
 222         DPL::Test::TestRunnerSingleton::Instance().RegisterTest(#Proc, &Proc); \
 223         return 0;                                                              \
 224     }                                                                          \
 225     const int DPL_UNUSED Static##Proc##InitVar = Static##Proc##Init();         \
 226     void Proc()
 227
 228
 229 /**
 230  * ASSERT MACROS
 231  *
 232  * Use them to create assertions in test cases. To do that put them inside test
 233  * body. Failing assertion indicates failing test.
 234  */
 235
 236 #define RUNNER_ASSERT_MSG(test, message)                                              \
 237     do                                                                                \
 238     {                                                                                 \
 239         DPL::Test::TestRunnerSingleton::Instance().MarkAssertion();                   \
 240                                                                                       \
 241         if (!(test))                                                                  \
 242         {                                                                             \
 243             std::ostringstream assertMsg;                                             \
 244             assertMsg << message << DPL::gdbbacktrace();                              \
 245             DPL::Test::TestRunner::TestFailed e(#test,                                \
 246                                                 __FILE__,                             \
 247                                                 __LINE__,                             \
 248                                                 assertMsg.str());                     \
 249             if (!std::uncaught_exception())                                           \
 250                 throw e;                                                              \
 251             DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
 252         }                                                                             \
 253     } while (0)
 254
 255 #define RUNNER_ASSERT_ERRNO_MSG(test, message)                                        \
 256     do                                                                                \
 257     {                                                                                 \
 258         DPL::Test::TestRunnerSingleton::Instance().MarkAssertion();                   \
 259                                                                                       \
 260         if (!(test))                                                                  \
 261         {                                                                             \
 262             const char *err = strerror(errno);                                        \
 263             std::ostringstream assertMsg;                                             \
 264             assertMsg << message;                                                     \
 265             if (!assertMsg.str().empty())                                             \
 266                 assertMsg << ". ";                                                    \
 267             assertMsg << err << DPL::gdbbacktrace();                                  \
 268             DPL::Test::TestRunner::TestFailed e(#test,                                \
 269                                                 __FILE__,                             \
 270                                                 __LINE__,                             \
 271                                                 assertMsg.str());                     \
 272             if (!std::uncaught_exception())                                           \
 273                 throw e;                                                              \
 274             DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
 275         }                                                                             \
 276     } while (0)
 277
 278 #define RUNNER_ASSERT_ERRNO(test) \
 279     RUNNER_ASSERT_ERRNO_MSG(test, "")
 280
 281 #define RUNNER_FAIL_MSG(message) \
 282     RUNNER_ASSERT_MSG(false, message)
 283
 284 #define RUNNER_ASSERT(test) \
 285     RUNNER_ASSERT_MSG(test, "")
 286
 287 /**
 288  * IGNORE MACRO
 289  *
 290  * When test reaches this macro call, its furhter code will be ignored.
 291  * To ignore whole test, put this macro call at the beginning of this tests
 292  * body.
 293  */
 294
 295 #define RUNNER_IGNORED_MSG(message)                            \
 296     do                                                         \
 297     {                                                          \
 298         std::ostringstream assertMsg;                          \
 299         assertMsg << message;                                  \
 300         throw DPL::Test::TestRunner::Ignored(assertMsg.str()); \
 301     } while (0)
 302
 303 /**
 304  * PERF MACROS
 305  *
 306  * Use these macros to do the time measurement. The first macro will start time measurement,
 307  * the second will gather the result. These macros can be used only once per test-case.
 308  * The result of time measurement will be displayed only if the test will pass.
 309  * Notice that these macros will work only if will be used in parent process. If these
 310  * macros will be used in child process then there will be no time measure results printed.
 311  * This macro in multiprocess tests has effect only if used in parent process. This macro
 312  * used in child process in multiprocess test has no effect.
 313  * The precision of measurement is 1 microsecond - the smallest time value that can be
 314  * measured is 0.000001s.
 315  * The time measure results will be printed only specific output format:
 316  *     - text
 317  *     - html
 318  *     - xml
 319  */
 320
 321 #define RUNNER_PERF_TEST_BEGIN(maxTime)                                                \
 322     do {                                                                               \
 323         DPL::Test::TestRunnerSingleton::Instance().beginPerformance(                   \
 324             std::chrono::microseconds{static_cast<long long int>(maxTime*1000000.0)}); \
 325     } while (0)
 326
 327 #define RUNNER_PERF_TEST_END()                                       \
 328     do {                                                             \
 329         DPL::Test::TestRunnerSingleton::Instance().endPerformance(); \
 330     } while (0)
 331
 332 /**
 333  * MSG MACROS
 334  *
 335  * Use these macros to print error messages during test run time
 336  */
 337
 338 #define RUNNER_ERROR_MSG(message)                             \
 339     do {                                                      \
 340         std::cerr << DPL::Colors::Text::RED_BEGIN << message  \
 341                   << DPL::Colors::Text::RED_END << std::endl; \
 342     } while (0)
 343
 344 #endif // DPL_TEST_RUNNER_H