src/framework/include/dpl/test/test_runner.h

   1 /*
   2  * Copyright (c) 2011-2015 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.h
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @version     1.0
  21  * @brief       This file is the header file of test runner
  22  */
  23
  24 #ifndef DPL_TEST_RUNNER_H
  25 #define DPL_TEST_RUNNER_H
  26
  27 #include <chrono>
  28 #include <cstring>
  29 #include <exception>
  30 #include <iostream>
  31 #include <list>
  32 #include <map>
  33 #include <queue>
  34 #include <set>
  35 #include <sstream>
  36 #include <string>
  37 #include <vector>
  38
  39 #include <dpl/atomic.h>
  40 #include <dpl/availability.h>
  41 #include <dpl/colors.h>
  42 #include <dpl/gdbbacktrace.h>
  43 #include <dpl/singleton.h>
  44 #include <dpl/test/test_failed.h>
  45 #include <dpl/test/test_ignored.h>
  46 #include <dpl/test/test_results_collector.h>
  47
  48 namespace DPL {
  49 namespace Test {
  50 class TestRunner
  51 {
  52     typedef std::map<std::string, TestResultsCollectorBasePtr>
  53     TestResultsCollectors;
  54     TestResultsCollectors m_collectors;
  55
  56     std::string m_startTestId;
  57     bool m_runIgnored;
  58
  59     std::queue<std::string> m_failReason;
  60
  61   public:
  62     TestRunner() :
  63         m_currentTestCase(nullptr)
  64       , m_terminate(false)
  65       , m_allowChildLogs(false)
  66     {}
  67
  68     void beginPerformanceTestTime(std::chrono::system_clock::duration maxTimeInMicroseconds);
  69     void endPerformanceTestTime();
  70     void getCurrentTestCasePerformanceResult(bool& isPerformanceTest,
  71                                              std::chrono::system_clock::duration& result,
  72                                              std::chrono::system_clock::duration& resultMax);
  73     void setCurrentTestCasePerformanceResult(bool isPerformanceTest,
  74                                              std::chrono::system_clock::duration result,
  75                                              std::chrono::system_clock::duration resultMax);
  76
  77     void addFailReason(const std::string &reason);
  78
  79     typedef void (*TestCase)();
  80
  81   private:
  82     struct TestCaseStruct
  83     {
  84         std::string name;
  85         TestCase proc;
  86
  87         bool m_isPerformanceTest;
  88         std::chrono::system_clock::time_point m_performanceTestStartTime;
  89         std::chrono::system_clock::duration m_performanceTestDurationTime;
  90         std::chrono::system_clock::duration m_performanceMaxTime;
  91
  92         bool operator <(const TestCaseStruct &other) const
  93         {
  94             return name < other.name;
  95         }
  96
  97         bool operator ==(const TestCaseStruct &other) const
  98         {
  99             return name == other.name;
 100         }
 101
 102         TestCaseStruct(const std::string &n, TestCase p) :
 103             name(n),
 104             proc(p),
 105             m_isPerformanceTest(false)
 106         {}
 107     };
 108
 109     typedef std::list<TestCaseStruct> TestCaseStructList;
 110     typedef std::map<std::string, TestCaseStructList> TestCaseGroupMap;
 111     TestCaseGroupMap m_testGroups;
 112
 113     TestCaseStruct * m_currentTestCase;
 114
 115     typedef std::set<std::string> SelectedTestNameSet;
 116     SelectedTestNameSet m_selectedTestNamesSet;
 117     typedef std::set<std::string> SelectedTestGroupSet;
 118     SelectedTestGroupSet m_selectedTestGroupSet;
 119     std::string m_currentGroup;
 120
 121     DPL::Atomic m_totalAssertions;
 122
 123     // Terminate without any logs.
 124     // Some test requires to call fork function.
 125     // Child process must not produce any logs and should die quietly.
 126     bool m_terminate;
 127     bool m_allowChildLogs;
 128
 129     void Banner();
 130     void InvalidArgs(const std::string& message = "Invalid arguments!");
 131     void Usage();
 132
 133     bool filterGroupsByXmls(const std::vector<std::string> & files);
 134     bool filterByXML(std::map<std::string, bool> & casesMap);
 135     void normalizeXMLTag(std::string& str, const std::string& testcase);
 136
 137     enum Status { FAILED, IGNORED, PASS };
 138
 139     Status RunTestCase(const TestCaseStruct& testCase);
 140
 141     void setCurrentTestCase(TestCaseStruct* testCase);
 142     TestCaseStruct *getCurrentTestCase();
 143
 144     void RunTests();
 145
 146     std::string getConcatedFailReason(const std::string &reason);
 147
 148     void CollectResult(const std::string& id,
 149                        const TestResultsCollectorBase::FailStatus status
 150                            = TestResultsCollectorBase::FailStatus::NONE,
 151                        const std::string& reason = std::string(),
 152                        const bool& isPerformanceTest = false,
 153                        const std::chrono::system_clock::duration& performanceTime = std::chrono::microseconds::zero(),
 154                        const std::chrono::system_clock::duration& performanceMaxTime = std::chrono::microseconds::zero());
 155
 156   public:
 157     void MarkAssertion();
 158
 159     void RegisterTest(const char *testName, TestCase proc);
 160     void InitGroup(const char* name);
 161
 162     int ExecTestRunner(int argc, char *argv[]);
 163     typedef std::vector<std::string> ArgsList;
 164     int ExecTestRunner(ArgsList args);
 165     bool getRunIgnored() const;
 166     // The runner will terminate as soon as possible (after current test).
 167     void Terminate();
 168     bool GetAllowChildLogs();
 169 };
 170
 171 typedef DPL::Singleton<TestRunner> TestRunnerSingleton;
 172 }
 173 } // namespace DPL
 174
 175 #define RUNNER_TEST_GROUP_INIT(GroupName)                                 \
 176     static int Static##GroupName##Init()                                  \
 177     {                                                                     \
 178         DPL::Test::TestRunnerSingleton::Instance().InitGroup(#GroupName); \
 179         return 0;                                                         \
 180     }                                                                     \
 181     const int DPL_UNUSED Static##GroupName##InitVar =                     \
 182         Static##GroupName##Init();
 183
 184 #define RUNNER_TEST(Proc)                                                      \
 185     void Proc();                                                               \
 186     static int Static##Proc##Init()                                            \
 187     {                                                                          \
 188         DPL::Test::TestRunnerSingleton::Instance().RegisterTest(#Proc, &Proc); \
 189         return 0;                                                              \
 190     }                                                                          \
 191     const int DPL_UNUSED Static##Proc##InitVar = Static##Proc##Init();         \
 192     void Proc()
 193
 194
 195 /**
 196  * ASSERT MACROS
 197  *
 198  * Use them to create assertions in test cases. To do that put them inside test
 199  * body. Failing assertion indicates failing test.
 200  */
 201
 202 #define RUNNER_ASSERT_MSG(test, message)                                              \
 203     do                                                                                \
 204     {                                                                                 \
 205         DPL::Test::TestRunnerSingleton::Instance().MarkAssertion();                   \
 206                                                                                       \
 207         if (!(test))                                                                  \
 208         {                                                                             \
 209             std::ostringstream assertMsg;                                             \
 210             assertMsg << message << DPL::gdbbacktrace();                              \
 211             DPL::Test::TestFailed e(#test,                                            \
 212                                     __FILE__,                                         \
 213                                     __LINE__,                                         \
 214                                     assertMsg.str());                                 \
 215             if (!std::uncaught_exception())                                           \
 216                 throw e;                                                              \
 217             DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
 218         }                                                                             \
 219     } while (0)
 220
 221 #define RUNNER_ASSERT_ERRNO_MSG(test, message)                                        \
 222     do                                                                                \
 223     {                                                                                 \
 224         DPL::Test::TestRunnerSingleton::Instance().MarkAssertion();                   \
 225                                                                                       \
 226         if (!(test))                                                                  \
 227         {                                                                             \
 228             const char *err = strerror(errno);                                        \
 229             std::ostringstream assertMsg;                                             \
 230             assertMsg << message;                                                     \
 231             if (!assertMsg.str().empty())                                             \
 232                 assertMsg << ". ";                                                    \
 233             assertMsg << err << DPL::gdbbacktrace();                                  \
 234             DPL::Test::TestFailed e(#test,                                            \
 235                                     __FILE__,                                         \
 236                                     __LINE__,                                         \
 237                                     assertMsg.str());                                 \
 238             if (!std::uncaught_exception())                                           \
 239                 throw e;                                                              \
 240             DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
 241         }                                                                             \
 242     } while (0)
 243
 244 #define RUNNER_ASSERT_ERRNO(test) \
 245     RUNNER_ASSERT_ERRNO_MSG(test, "")
 246
 247 #define RUNNER_FAIL_MSG(message) \
 248     RUNNER_ASSERT_MSG(false, message)
 249
 250 #define RUNNER_ASSERT(test) \
 251     RUNNER_ASSERT_MSG(test, "")
 252
 253 /**
 254  * IGNORE MACRO
 255  *
 256  * When test reaches this macro call, its furhter code will be ignored.
 257  * To ignore whole test, put this macro call at the beginning of this tests
 258  * body.
 259  */
 260
 261 #define RUNNER_IGNORED_MSG(message)                    \
 262     do                                                 \
 263     {                                                  \
 264         std::ostringstream assertMsg;                  \
 265         assertMsg << message;                          \
 266         throw DPL::Test::TestIgnored(assertMsg.str()); \
 267     } while (0)
 268
 269 /**
 270  * PERF MACROS
 271  *
 272  * Use these macros to do the time measurement. The first macro will start time measurement,
 273  * the second will gather the result. These macros can be used only once per test-case.
 274  * The result of time measurement will be displayed only if the test will pass.
 275  * Notice that these macros will work only if will be used in parent process. If these
 276  * macros will be used in child process then there will be no time measure results printed.
 277  * This macro in multiprocess tests has effect only if used in parent process. This macro
 278  * used in child process in multiprocess test has no effect.
 279  * The precision of measurement is 1 microsecond - the smallest time value that can be
 280  * measured is 0.000001s.
 281  * The time measure results will be printed only specific output format:
 282  *     - text
 283  *     - html
 284  *     - xml
 285  */
 286
 287 #define RUNNER_PERF_TEST_BEGIN(maxTime)                                                \
 288     do {                                                                               \
 289         DPL::Test::TestRunnerSingleton::Instance().beginPerformanceTestTime(           \
 290             std::chrono::microseconds{static_cast<long long int>(maxTime*1000000.0)}); \
 291     } while (0)
 292
 293 #define RUNNER_PERF_TEST_END()                                                \
 294     do {                                                                      \
 295         DPL::Test::TestRunnerSingleton::Instance().endPerformanceTestTime();  \
 296     } while (0)
 297
 298 /**
 299  * MSG MACROS
 300  *
 301  * Use these macros to print error messages during test run time
 302  */
 303
 304 #define RUNNER_ERROR_MSG(message)                             \
 305     do {                                                      \
 306         std::cerr << DPL::Colors::Text::RED_BEGIN << message  \
 307                   << DPL::Colors::Text::RED_END << std::endl; \
 308     } while (0)
 309
 310 #endif // DPL_TEST_RUNNER_H