src/framework/include/dpl/test/test_runner.h

   1 /*
   2  * Copyright (c) 2011-2015 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.h
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @version     1.0
  21  * @brief       This file is the header file of test runner
  22  */
  23
  24 #ifndef DPL_TEST_RUNNER_H
  25 #define DPL_TEST_RUNNER_H
  26
  27 #include <chrono>
  28 #include <cstring>
  29 #include <exception>
  30 #include <iostream>
  31 #include <list>
  32 #include <map>
  33 #include <queue>
  34 #include <set>
  35 #include <sstream>
  36 #include <string>
  37 #include <vector>
  38
  39 #include <dpl/atomic.h>
  40 #include <dpl/availability.h>
  41 #include <dpl/colors.h>
  42 #include <dpl/gdbbacktrace.h>
  43 #include <dpl/singleton.h>
  44 #include <dpl/test/test_results_collector.h>
  45
  46 namespace DPL {
  47 namespace Test {
  48 class TestRunner
  49 {
  50     typedef std::map<std::string, TestResultsCollectorBasePtr>
  51     TestResultsCollectors;
  52     TestResultsCollectors m_collectors;
  53
  54     std::string m_startTestId;
  55     bool m_runIgnored;
  56
  57     std::queue<std::string> m_failReason;
  58
  59   public:
  60     TestRunner() :
  61         m_currentTestCase(nullptr)
  62       , m_terminate(false)
  63       , m_allowChildLogs(false)
  64     {}
  65
  66     void beginPerformanceTestTime(std::chrono::system_clock::duration maxTimeInMicroseconds);
  67     void endPerformanceTestTime();
  68     void getCurrentTestCasePerformanceResult(bool& isPerformanceTest,
  69                                              std::chrono::system_clock::duration& result,
  70                                              std::chrono::system_clock::duration& resultMax);
  71     void setCurrentTestCasePerformanceResult(bool isPerformanceTest,
  72                                              std::chrono::system_clock::duration result,
  73                                              std::chrono::system_clock::duration resultMax);
  74
  75     void addFailReason(const std::string &reason);
  76
  77     typedef void (*TestCase)();
  78
  79   private:
  80     struct TestCaseStruct
  81     {
  82         std::string name;
  83         TestCase proc;
  84
  85         bool m_isPerformanceTest;
  86         std::chrono::system_clock::time_point m_performanceTestStartTime;
  87         std::chrono::system_clock::duration m_performanceTestDurationTime;
  88         std::chrono::system_clock::duration m_performanceMaxTime;
  89
  90         bool operator <(const TestCaseStruct &other) const
  91         {
  92             return name < other.name;
  93         }
  94
  95         bool operator ==(const TestCaseStruct &other) const
  96         {
  97             return name == other.name;
  98         }
  99
 100         TestCaseStruct(const std::string &n, TestCase p) :
 101             name(n),
 102             proc(p),
 103             m_isPerformanceTest(false)
 104         {}
 105     };
 106
 107     typedef std::list<TestCaseStruct> TestCaseStructList;
 108     typedef std::map<std::string, TestCaseStructList> TestCaseGroupMap;
 109     TestCaseGroupMap m_testGroups;
 110
 111     TestCaseStruct * m_currentTestCase;
 112
 113     typedef std::set<std::string> SelectedTestNameSet;
 114     SelectedTestNameSet m_selectedTestNamesSet;
 115     typedef std::set<std::string> SelectedTestGroupSet;
 116     SelectedTestGroupSet m_selectedTestGroupSet;
 117     std::string m_currentGroup;
 118
 119     DPL::Atomic m_totalAssertions;
 120
 121     // Terminate without any logs.
 122     // Some test requires to call fork function.
 123     // Child process must not produce any logs and should die quietly.
 124     bool m_terminate;
 125     bool m_allowChildLogs;
 126
 127     void Banner();
 128     void InvalidArgs(const std::string& message = "Invalid arguments!");
 129     void Usage();
 130
 131     bool filterGroupsByXmls(const std::vector<std::string> & files);
 132     bool filterByXML(std::map<std::string, bool> & casesMap);
 133     void normalizeXMLTag(std::string& str, const std::string& testcase);
 134
 135     enum Status { FAILED, IGNORED, PASS };
 136
 137     Status RunTestCase(const TestCaseStruct& testCase);
 138
 139     void setCurrentTestCase(TestCaseStruct* testCase);
 140     TestCaseStruct *getCurrentTestCase();
 141
 142     void RunTests();
 143
 144     std::string getConcatedFailReason(const std::string &reason);
 145
 146     void CollectResult(const std::string& id,
 147                        const TestResultsCollectorBase::FailStatus status
 148                            = TestResultsCollectorBase::FailStatus::NONE,
 149                        const std::string& reason = std::string(),
 150                        const bool& isPerformanceTest = false,
 151                        const std::chrono::system_clock::duration& performanceTime = std::chrono::microseconds::zero(),
 152                        const std::chrono::system_clock::duration& performanceMaxTime = std::chrono::microseconds::zero());
 153
 154   public:
 155     class TestFailed
 156     {
 157       private:
 158         std::string m_message;
 159
 160       public:
 161         TestFailed()
 162         {}
 163
 164         //! \brief Failed test message creator
 165         //!
 166         //! \param[in] aTest string for tested expression
 167         //! \param[in] aFile source file name
 168         //! \param[in] aLine source file line
 169         //! \param[in] aMessage error message
 170         TestFailed(const char* aTest,
 171                    const char* aFile,
 172                    int aLine,
 173                    const std::string &aMessage);
 174
 175         TestFailed(const std::string &message);
 176
 177         std::string GetMessage() const
 178         {
 179             return m_message;
 180         }
 181     };
 182
 183     class Ignored
 184     {
 185       private:
 186         std::string m_message;
 187
 188       public:
 189         Ignored()
 190         {}
 191
 192         Ignored(const std::string &message) :
 193             m_message(message)
 194         {}
 195
 196         std::string GetMessage() const
 197         {
 198             return m_message;
 199         }
 200     };
 201
 202     void MarkAssertion();
 203
 204     void RegisterTest(const char *testName, TestCase proc);
 205     void InitGroup(const char* name);
 206
 207     int ExecTestRunner(int argc, char *argv[]);
 208     typedef std::vector<std::string> ArgsList;
 209     int ExecTestRunner(ArgsList args);
 210     bool getRunIgnored() const;
 211     // The runner will terminate as soon as possible (after current test).
 212     void Terminate();
 213     bool GetAllowChildLogs();
 214 };
 215
 216 typedef DPL::Singleton<TestRunner> TestRunnerSingleton;
 217 }
 218 } // namespace DPL
 219
 220 #define RUNNER_TEST_GROUP_INIT(GroupName)                                 \
 221     static int Static##GroupName##Init()                                  \
 222     {                                                                     \
 223         DPL::Test::TestRunnerSingleton::Instance().InitGroup(#GroupName); \
 224         return 0;                                                         \
 225     }                                                                     \
 226     const int DPL_UNUSED Static##GroupName##InitVar =                     \
 227         Static##GroupName##Init();
 228
 229 #define RUNNER_TEST(Proc)                                                      \
 230     void Proc();                                                               \
 231     static int Static##Proc##Init()                                            \
 232     {                                                                          \
 233         DPL::Test::TestRunnerSingleton::Instance().RegisterTest(#Proc, &Proc); \
 234         return 0;                                                              \
 235     }                                                                          \
 236     const int DPL_UNUSED Static##Proc##InitVar = Static##Proc##Init();         \
 237     void Proc()
 238
 239
 240 /**
 241  * ASSERT MACROS
 242  *
 243  * Use them to create assertions in test cases. To do that put them inside test
 244  * body. Failing assertion indicates failing test.
 245  */
 246
 247 #define RUNNER_ASSERT_MSG(test, message)                                              \
 248     do                                                                                \
 249     {                                                                                 \
 250         DPL::Test::TestRunnerSingleton::Instance().MarkAssertion();                   \
 251                                                                                       \
 252         if (!(test))                                                                  \
 253         {                                                                             \
 254             std::ostringstream assertMsg;                                             \
 255             assertMsg << message << DPL::gdbbacktrace();                              \
 256             DPL::Test::TestRunner::TestFailed e(#test,                                \
 257                                                 __FILE__,                             \
 258                                                 __LINE__,                             \
 259                                                 assertMsg.str());                     \
 260             if (!std::uncaught_exception())                                           \
 261                 throw e;                                                              \
 262             DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
 263         }                                                                             \
 264     } while (0)
 265
 266 #define RUNNER_ASSERT_ERRNO_MSG(test, message)                                        \
 267     do                                                                                \
 268     {                                                                                 \
 269         DPL::Test::TestRunnerSingleton::Instance().MarkAssertion();                   \
 270                                                                                       \
 271         if (!(test))                                                                  \
 272         {                                                                             \
 273             const char *err = strerror(errno);                                        \
 274             std::ostringstream assertMsg;                                             \
 275             assertMsg << message;                                                     \
 276             if (!assertMsg.str().empty())                                             \
 277                 assertMsg << ". ";                                                    \
 278             assertMsg << err << DPL::gdbbacktrace();                                  \
 279             DPL::Test::TestRunner::TestFailed e(#test,                                \
 280                                                 __FILE__,                             \
 281                                                 __LINE__,                             \
 282                                                 assertMsg.str());                     \
 283             if (!std::uncaught_exception())                                           \
 284                 throw e;                                                              \
 285             DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
 286         }                                                                             \
 287     } while (0)
 288
 289 #define RUNNER_ASSERT_ERRNO(test) \
 290     RUNNER_ASSERT_ERRNO_MSG(test, "")
 291
 292 #define RUNNER_FAIL_MSG(message) \
 293     RUNNER_ASSERT_MSG(false, message)
 294
 295 #define RUNNER_ASSERT(test) \
 296     RUNNER_ASSERT_MSG(test, "")
 297
 298 /**
 299  * IGNORE MACRO
 300  *
 301  * When test reaches this macro call, its furhter code will be ignored.
 302  * To ignore whole test, put this macro call at the beginning of this tests
 303  * body.
 304  */
 305
 306 #define RUNNER_IGNORED_MSG(message)                            \
 307     do                                                         \
 308     {                                                          \
 309         std::ostringstream assertMsg;                          \
 310         assertMsg << message;                                  \
 311         throw DPL::Test::TestRunner::Ignored(assertMsg.str()); \
 312     } while (0)
 313
 314 /**
 315  * PERF MACROS
 316  *
 317  * Use these macros to do the time measurement. The first macro will start time measurement,
 318  * the second will gather the result. These macros can be used only once per test-case.
 319  * The result of time measurement will be displayed only if the test will pass.
 320  * Notice that these macros will work only if will be used in parent process. If these
 321  * macros will be used in child process then there will be no time measure results printed.
 322  * This macro in multiprocess tests has effect only if used in parent process. This macro
 323  * used in child process in multiprocess test has no effect.
 324  * The precision of measurement is 1 microsecond - the smallest time value that can be
 325  * measured is 0.000001s.
 326  * The time measure results will be printed only specific output format:
 327  *     - text
 328  *     - html
 329  *     - xml
 330  */
 331
 332 #define RUNNER_PERF_TEST_BEGIN(maxTime)                                                \
 333     do {                                                                               \
 334         DPL::Test::TestRunnerSingleton::Instance().beginPerformanceTestTime(           \
 335             std::chrono::microseconds{static_cast<long long int>(maxTime*1000000.0)}); \
 336     } while (0)
 337
 338 #define RUNNER_PERF_TEST_END()                                                \
 339     do {                                                                      \
 340         DPL::Test::TestRunnerSingleton::Instance().endPerformanceTestTime();  \
 341     } while (0)
 342
 343 /**
 344  * MSG MACROS
 345  *
 346  * Use these macros to print error messages during test run time
 347  */
 348
 349 #define RUNNER_ERROR_MSG(message)                             \
 350     do {                                                      \
 351         std::cerr << DPL::Colors::Text::RED_BEGIN << message  \
 352                   << DPL::Colors::Text::RED_END << std::endl; \
 353     } while (0)
 354
 355 #endif // DPL_TEST_RUNNER_H