src/framework/include/dpl/test/test_runner.h

   1 /*
   2  * Copyright (c) 2011-2015 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.h
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @version     1.0
  21  * @brief       This file is the header file of test runner
  22  */
  23
  24 #ifndef DPL_TEST_RUNNER_H
  25 #define DPL_TEST_RUNNER_H
  26
  27 #include <chrono>
  28 #include <cstring>
  29 #include <exception>
  30 #include <iostream>
  31 #include <list>
  32 #include <map>
  33 #include <queue>
  34 #include <set>
  35 #include <sstream>
  36 #include <string>
  37 #include <vector>
  38
  39 #include <dpl/atomic.h>
  40 #include <dpl/availability.h>
  41 #include <dpl/colors.h>
  42 #include <dpl/gdbbacktrace.h>
  43 #include <dpl/singleton.h>
  44 #include <dpl/test/performance_result.h>
  45 #include <dpl/test/test_result.h>
  46 #include <dpl/test/test_results_collector.h>
  47
  48 namespace DPL {
  49 namespace Test {
  50 class TestRunner
  51 {
  52     typedef std::map<std::string, TestResultsCollectorBasePtr>
  53     TestResultsCollectors;
  54     TestResultsCollectors m_collectors;
  55
  56     std::string m_startTestId;
  57     bool m_runIgnored;
  58
  59     std::queue<std::string> m_failReason;
  60
  61   public:
  62     TestRunner() :
  63         m_currentTestCase(nullptr)
  64       , m_terminate(false)
  65       , m_allowChildLogs(false)
  66     {}
  67
  68     void beginPerformance(std::chrono::system_clock::duration maxDurationInMicroseconds);
  69     void endPerformance();
  70     void setCurrentTestCasePerformanceResult(const PerformanceResultPtr &performance);
  71     ConstPerformanceResultPtr getCurrentTestCasePerformanceResult();
  72
  73     void addFailReason(const std::string &reason);
  74
  75     typedef void (*TestCase)();
  76
  77   private:
  78     struct TestCaseStruct
  79     {
  80         std::string name;
  81         TestCase proc;
  82         PerformanceResultPtr performance;
  83
  84         bool operator <(const TestCaseStruct &other) const
  85         {
  86             return name < other.name;
  87         }
  88
  89         bool operator ==(const TestCaseStruct &other) const
  90         {
  91             return name == other.name;
  92         }
  93
  94         TestCaseStruct(const std::string &n, TestCase p) :
  95             name(n),
  96             proc(p)
  97         {}
  98     };
  99
 100     typedef std::list<TestCaseStruct> TestCaseStructList;
 101     typedef std::map<std::string, TestCaseStructList> TestCaseGroupMap;
 102     TestCaseGroupMap m_testGroups;
 103
 104     TestCaseStruct * m_currentTestCase;
 105
 106     typedef std::set<std::string> SelectedTestNameSet;
 107     SelectedTestNameSet m_selectedTestNamesSet;
 108     typedef std::set<std::string> SelectedTestGroupSet;
 109     SelectedTestGroupSet m_selectedTestGroupSet;
 110     std::string m_currentGroup;
 111
 112     DPL::Atomic m_totalAssertions;
 113
 114     // Terminate without any logs.
 115     // Some test requires to call fork function.
 116     // Child process must not produce any logs and should die quietly.
 117     bool m_terminate;
 118     bool m_allowChildLogs;
 119
 120     void Banner();
 121     void InvalidArgs(const std::string& message = "Invalid arguments!");
 122     void Usage();
 123
 124     bool filterGroupsByXmls(const std::vector<std::string> & files);
 125     bool filterByXML(std::map<std::string, bool> & casesMap);
 126     void normalizeXMLTag(std::string& str, const std::string& testcase);
 127
 128     void RunTestCase(const TestCaseStruct& testCase);
 129
 130     void setCurrentTestCase(TestCaseStruct* testCase);
 131     TestCaseStruct *getCurrentTestCase();
 132
 133     void RunTests();
 134
 135     std::string getConcatedFailReason(const std::string &reason);
 136
 137     void CollectResult(const std::string& id, const TestResult &result);
 138
 139   public:
 140     class TestFailed
 141     {
 142       private:
 143         std::string m_message;
 144
 145       public:
 146         TestFailed()
 147         {}
 148
 149         //! \brief Failed test message creator
 150         //!
 151         //! \param[in] aTest string for tested expression
 152         //! \param[in] aFile source file name
 153         //! \param[in] aLine source file line
 154         //! \param[in] aMessage error message
 155         TestFailed(const char* aTest,
 156                    const char* aFile,
 157                    int aLine,
 158                    const std::string &aMessage);
 159
 160         TestFailed(const std::string &message);
 161
 162         std::string GetMessage() const
 163         {
 164             return m_message;
 165         }
 166     };
 167
 168     class Ignored
 169     {
 170       private:
 171         std::string m_message;
 172
 173       public:
 174         Ignored()
 175         {}
 176
 177         Ignored(const std::string &message) :
 178             m_message(message)
 179         {}
 180
 181         std::string GetMessage() const
 182         {
 183             return m_message;
 184         }
 185     };
 186
 187     void MarkAssertion();
 188
 189     void RegisterTest(const char *testName, TestCase proc);
 190     void InitGroup(const char* name);
 191
 192     int ExecTestRunner(int argc, char *argv[]);
 193     typedef std::vector<std::string> ArgsList;
 194     int ExecTestRunner(ArgsList args);
 195     bool getRunIgnored() const;
 196     // The runner will terminate as soon as possible (after current test).
 197     void Terminate();
 198     bool GetAllowChildLogs();
 199 };
 200
 201 typedef DPL::Singleton<TestRunner> TestRunnerSingleton;
 202 }
 203 } // namespace DPL
 204
 205 #define RUNNER_TEST_GROUP_INIT(GroupName)                                 \
 206     static int Static##GroupName##Init()                                  \
 207     {                                                                     \
 208         DPL::Test::TestRunnerSingleton::Instance().InitGroup(#GroupName); \
 209         return 0;                                                         \
 210     }                                                                     \
 211     const int DPL_UNUSED Static##GroupName##InitVar =                     \
 212         Static##GroupName##Init();
 213
 214 #define RUNNER_TEST(Proc)                                                      \
 215     void Proc();                                                               \
 216     static int Static##Proc##Init()                                            \
 217     {                                                                          \
 218         DPL::Test::TestRunnerSingleton::Instance().RegisterTest(#Proc, &Proc); \
 219         return 0;                                                              \
 220     }                                                                          \
 221     const int DPL_UNUSED Static##Proc##InitVar = Static##Proc##Init();         \
 222     void Proc()
 223
 224
 225 /**
 226  * ASSERT MACROS
 227  *
 228  * Use them to create assertions in test cases. To do that put them inside test
 229  * body. Failing assertion indicates failing test.
 230  */
 231
 232 #define RUNNER_ASSERT_MSG(test, message)                                              \
 233     do                                                                                \
 234     {                                                                                 \
 235         DPL::Test::TestRunnerSingleton::Instance().MarkAssertion();                   \
 236                                                                                       \
 237         if (!(test))                                                                  \
 238         {                                                                             \
 239             std::ostringstream assertMsg;                                             \
 240             assertMsg << message << DPL::gdbbacktrace();                              \
 241             DPL::Test::TestRunner::TestFailed e(#test,                                \
 242                                                 __FILE__,                             \
 243                                                 __LINE__,                             \
 244                                                 assertMsg.str());                     \
 245             if (!std::uncaught_exception())                                           \
 246                 throw e;                                                              \
 247             DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
 248         }                                                                             \
 249     } while (0)
 250
 251 #define RUNNER_ASSERT_ERRNO_MSG(test, message)                                        \
 252     do                                                                                \
 253     {                                                                                 \
 254         DPL::Test::TestRunnerSingleton::Instance().MarkAssertion();                   \
 255                                                                                       \
 256         if (!(test))                                                                  \
 257         {                                                                             \
 258             const char *err = strerror(errno);                                        \
 259             std::ostringstream assertMsg;                                             \
 260             assertMsg << message;                                                     \
 261             if (!assertMsg.str().empty())                                             \
 262                 assertMsg << ". ";                                                    \
 263             assertMsg << err << DPL::gdbbacktrace();                                  \
 264             DPL::Test::TestRunner::TestFailed e(#test,                                \
 265                                                 __FILE__,                             \
 266                                                 __LINE__,                             \
 267                                                 assertMsg.str());                     \
 268             if (!std::uncaught_exception())                                           \
 269                 throw e;                                                              \
 270             DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
 271         }                                                                             \
 272     } while (0)
 273
 274 #define RUNNER_ASSERT_ERRNO(test) \
 275     RUNNER_ASSERT_ERRNO_MSG(test, "")
 276
 277 #define RUNNER_FAIL_MSG(message) \
 278     RUNNER_ASSERT_MSG(false, message)
 279
 280 #define RUNNER_ASSERT(test) \
 281     RUNNER_ASSERT_MSG(test, "")
 282
 283 /**
 284  * IGNORE MACRO
 285  *
 286  * When test reaches this macro call, its furhter code will be ignored.
 287  * To ignore whole test, put this macro call at the beginning of this tests
 288  * body.
 289  */
 290
 291 #define RUNNER_IGNORED_MSG(message)                            \
 292     do                                                         \
 293     {                                                          \
 294         std::ostringstream assertMsg;                          \
 295         assertMsg << message;                                  \
 296         throw DPL::Test::TestRunner::Ignored(assertMsg.str()); \
 297     } while (0)
 298
 299 /**
 300  * PERF MACROS
 301  *
 302  * Use these macros to do the time measurement. The first macro will start time measurement,
 303  * the second will gather the result. These macros can be used only once per test-case.
 304  * The result of time measurement will be displayed only if the test will pass.
 305  * Notice that these macros will work only if will be used in parent process. If these
 306  * macros will be used in child process then there will be no time measure results printed.
 307  * This macro in multiprocess tests has effect only if used in parent process. This macro
 308  * used in child process in multiprocess test has no effect.
 309  * The precision of measurement is 1 microsecond - the smallest time value that can be
 310  * measured is 0.000001s.
 311  * The time measure results will be printed only specific output format:
 312  *     - text
 313  *     - html
 314  *     - xml
 315  */
 316
 317 #define RUNNER_PERF_TEST_BEGIN(maxTime)                                                \
 318     do {                                                                               \
 319         DPL::Test::TestRunnerSingleton::Instance().beginPerformance(                   \
 320             std::chrono::microseconds{static_cast<long long int>(maxTime*1000000.0)}); \
 321     } while (0)
 322
 323 #define RUNNER_PERF_TEST_END()                                       \
 324     do {                                                             \
 325         DPL::Test::TestRunnerSingleton::Instance().endPerformance(); \
 326     } while (0)
 327
 328 /**
 329  * MSG MACROS
 330  *
 331  * Use these macros to print error messages during test run time
 332  */
 333
 334 #define RUNNER_ERROR_MSG(message)                             \
 335     do {                                                      \
 336         std::cerr << DPL::Colors::Text::RED_BEGIN << message  \
 337                   << DPL::Colors::Text::RED_END << std::endl; \
 338     } while (0)
 339
 340 #endif // DPL_TEST_RUNNER_H