tests/framework/include/dpl/test/test_runner.h

   1 /*
   2  * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.h
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @version     1.0
  21  * @brief       This file is the header file of test runner
  22  */
  23 #ifndef DPL_TEST_RUNNER_H
  24 #define DPL_TEST_RUNNER_H
  25
  26 #include <dpl/singleton.h>
  27 #include <dpl/availability.h>
  28 #include <dpl/atomic.h>
  29 #include <dpl/test/test_results_collector.h>
  30 #include <sstream>
  31 #include <string>
  32 #include <vector>
  33 #include <chrono>
  34 #include <list>
  35 #include <set>
  36 #include <map>
  37
  38 namespace DPL {
  39 namespace Test {
  40 class TestRunner
  41 {
  42     typedef std::map<std::string, TestResultsCollectorBasePtr>
  43     TestResultsCollectors;
  44     TestResultsCollectors m_collectors;
  45
  46     std::string m_startTestId;
  47     bool m_runIgnored;
  48
  49   public:
  50     TestRunner() :
  51         m_currentTestCase(NULL)
  52       , m_terminate(false)
  53       , m_allowChildLogs(false)
  54     {}
  55
  56     void beginPerformanceTestTime(std::chrono::system_clock::duration maxTimeInMicroseconds);
  57     void endPerformanceTestTime();
  58     void getCurrentTestCasePerformanceResult(bool& isPerformanceTest,
  59                                              std::chrono::system_clock::duration& result,
  60                                              std::chrono::system_clock::duration& resultMax);
  61     void setCurrentTestCasePerformanceResult(bool isPerformanceTest,
  62                                              std::chrono::system_clock::duration result,
  63                                              std::chrono::system_clock::duration resultMax);
  64
  65     typedef void (*TestCase)();
  66
  67   private:
  68     struct TestCaseStruct
  69     {
  70         std::string name;
  71         TestCase proc;
  72
  73         bool m_isPerformanceTest;
  74         std::chrono::system_clock::time_point m_performanceTestStartTime;
  75         std::chrono::system_clock::duration m_performanceTestDurationTime;
  76         std::chrono::system_clock::duration m_performanceMaxTime;
  77
  78         bool operator <(const TestCaseStruct &other) const
  79         {
  80             return name < other.name;
  81         }
  82
  83         bool operator ==(const TestCaseStruct &other) const
  84         {
  85             return name == other.name;
  86         }
  87
  88         TestCaseStruct(const std::string &n, TestCase p) :
  89             name(n),
  90             proc(p),
  91             m_isPerformanceTest(false)
  92         {}
  93     };
  94
  95     typedef std::list<TestCaseStruct> TestCaseStructList;
  96     typedef std::map<std::string, TestCaseStructList> TestCaseGroupMap;
  97     TestCaseGroupMap m_testGroups;
  98
  99     TestCaseStruct * m_currentTestCase;
 100
 101     typedef std::set<std::string> SelectedTestNameSet;
 102     SelectedTestNameSet m_selectedTestNamesSet;
 103     typedef std::set<std::string> SelectedTestGroupSet;
 104     SelectedTestGroupSet m_selectedTestGroupSet;
 105     std::string m_currentGroup;
 106
 107     DPL::Atomic m_totalAssertions;
 108
 109     // Terminate without any logs.
 110     // Some test requires to call fork function.
 111     // Child process must not produce any logs and should die quietly.
 112     bool m_terminate;
 113     bool m_allowChildLogs;
 114
 115     void Banner();
 116     void InvalidArgs(const std::string& message = "Invalid arguments!");
 117     void Usage();
 118
 119     bool filterGroupsByXmls(const std::vector<std::string> & files);
 120     bool filterByXML(std::map<std::string, bool> & casesMap);
 121     void normalizeXMLTag(std::string& str, const std::string& testcase);
 122
 123     enum Status { FAILED, IGNORED, PASS };
 124
 125     Status RunTestCase(const TestCaseStruct& testCase);
 126
 127     void setCurrentTestCase(TestCaseStruct* testCase);
 128     TestCaseStruct *getCurrentTestCase();
 129
 130     void RunTests();
 131
 132     void CollectResult(const std::string& id,
 133                        const std::string& description,
 134                        const TestResultsCollectorBase::FailStatus::Type status
 135                            = TestResultsCollectorBase::FailStatus::NONE,
 136                        const std::string& reason = std::string(),
 137                        const bool& isPerformanceTest = false,
 138                        const std::chrono::system_clock::duration& performanceTime = std::chrono::microseconds::zero(),
 139                        const std::chrono::system_clock::duration& performanceMaxTime = std::chrono::microseconds::zero());
 140
 141   public:
 142     class TestFailed
 143     {
 144       private:
 145         std::string m_message;
 146
 147       public:
 148         TestFailed()
 149         {}
 150
 151         //! \brief Failed test message creator
 152         //!
 153         //! \param[in] aTest string for tested expression
 154         //! \param[in] aFile source file name
 155         //! \param[in] aLine source file line
 156         //! \param[in] aMessage error message
 157         TestFailed(const char* aTest,
 158                    const char* aFile,
 159                    int aLine,
 160                    const std::string &aMessage);
 161
 162         TestFailed(const std::string &message);
 163
 164         std::string GetMessage() const
 165         {
 166             return m_message;
 167         }
 168     };
 169
 170     class Ignored
 171     {
 172       private:
 173         std::string m_message;
 174
 175       public:
 176         Ignored()
 177         {}
 178
 179         Ignored(const std::string &message) :
 180             m_message(message)
 181         {}
 182
 183         std::string GetMessage() const
 184         {
 185             return m_message;
 186         }
 187     };
 188
 189     void MarkAssertion();
 190
 191     void RegisterTest(const char *testName, TestCase proc);
 192     void InitGroup(const char* name);
 193
 194     int ExecTestRunner(int argc, char *argv[]);
 195     typedef std::vector<std::string> ArgsList;
 196     int ExecTestRunner(ArgsList args);
 197     bool getRunIgnored() const;
 198     // The runner will terminate as soon as possible (after current test).
 199     void Terminate();
 200     bool GetAllowChildLogs();
 201 };
 202
 203 typedef DPL::Singleton<TestRunner> TestRunnerSingleton;
 204 }
 205 } // namespace DPL
 206
 207 #define RUNNER_TEST_GROUP_INIT(GroupName)                                \
 208     static int Static##GroupName##Init()                                 \
 209     {                                                                    \
 210         DPL::Test::TestRunnerSingleton::Instance().InitGroup(#GroupName); \
 211         return 0;                                                        \
 212     }                                                                    \
 213     const int DPL_UNUSED Static##GroupName##InitVar =                   \
 214         Static##GroupName##Init();
 215
 216 #define RUNNER_TEST(Proc)                                                \
 217     void Proc();                                                         \
 218     static int Static##Proc##Init()                                      \
 219     {                                                                    \
 220         DPL::Test::TestRunnerSingleton::Instance().RegisterTest(#Proc, &Proc); \
 221         return 0;                                                        \
 222     }                                                                    \
 223     const int DPL_UNUSED Static##Proc##InitVar = Static##Proc##Init();  \
 224     void Proc()
 225
 226 //! \brief Returns base name for path
 227
 228 #define RUNNER_ASSERT_MSG(test, message)                                               \
 229     do                                                                                     \
 230     {                                                                                      \
 231         DPL::Test::TestRunnerSingleton::Instance().MarkAssertion();                        \
 232                                                                                        \
 233         if (!(test))                                                                       \
 234         {                                                                                  \
 235             std::ostringstream assertMsg;                                                  \
 236             assertMsg << message;                                                          \
 237             throw DPL::Test::TestRunner::TestFailed(#test, \
 238                                                     __FILE__, \
 239                                                     __LINE__, \
 240                                                     assertMsg.str()); \
 241         }                                                                                  \
 242     } while (0)
 243
 244 #define RUNNER_ASSERT(test) RUNNER_ASSERT_MSG(test, "")
 245
 246 #define RUNNER_FAIL RUNNER_ASSERT(false)
 247
 248 #define RUNNER_IGNORED_MSG(message) do { std::ostringstream assertMsg; \
 249                                          assertMsg << message; \
 250                                          throw DPL::Test::TestRunner::Ignored( \
 251                                                    assertMsg.str()); \
 252 } while (0)
 253
 254 /*
 255  * Use these macros to do the time measurement. The first macro will start time measurement,
 256  * the second will gather the result. These macros can be used only once per test-case.
 257  * The result of time measurement will be displayed only if the test will pass.
 258  * Notice that these macros will work only if will be used in parent process. If these
 259  * macros will be used in child process then there will be no time measure results printed.
 260  * This macro in multiprocess tests has effect only if used in parent process. This macro
 261  * used in child process in multiprocess test has no effect.
 262  * The precision of measurement is 1 microsecond - the smallest time value that can be
 263  * measured is 0.000001s.
 264  * The time measure results will be printed only specific output format:
 265  *     - text
 266  *     - html
 267  *     - xml
 268  *     - csv
 269  * In TAP format performance result will not be displayed.
 270  */
 271 #define RUNNER_PERF_TEST_BEGIN(maxTime)                                        \
 272     do {                                                                       \
 273         DPL::Test::TestRunnerSingleton::Instance().beginPerformanceTestTime(   \
 274                 std::chrono::microseconds{static_cast<long long int>(maxTime*1000000.0)}); \
 275     } while (0)
 276
 277 #define RUNNER_PERF_TEST_END()                                                \
 278     do {                                                                      \
 279         DPL::Test::TestRunnerSingleton::Instance().endPerformanceTestTime();  \
 280     } while (0)
 281
 282 #endif // DPL_TEST_RUNNER_H