src/framework/include/dpl/test/test_runner.h

   1 /*
   2  * Copyright (c) 2011-2015 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.h
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @version     1.0
  21  * @brief       This file is the header file of test runner
  22  */
  23
  24 #ifndef DPL_TEST_RUNNER_H
  25 #define DPL_TEST_RUNNER_H
  26
  27 #include <chrono>
  28 #include <cstdlib>
  29 #include <cstring>
  30 #include <exception>
  31 #include <iostream>
  32 #include <list>
  33 #include <map>
  34 #include <queue>
  35 #include <set>
  36 #include <sstream>
  37 #include <string>
  38 #include <vector>
  39
  40 #include <dpl/atomic.h>
  41 #include <dpl/availability.h>
  42 #include <dpl/colors.h>
  43 #include <dpl/gdbbacktrace.h>
  44 #include <dpl/singleton.h>
  45 #include <dpl/test/performance_result.h>
  46 #include <dpl/test/test_exception.h>
  47 #include <dpl/test/test_failed.h>
  48 #include <dpl/test/test_ignored.h>
  49 #include <dpl/test/test_result.h>
  50 #include <dpl/test/test_results_collector.h>
  51
  52 namespace DPL {
  53 namespace Test {
  54 class TestRunner
  55 {
  56     typedef std::map<std::string, TestResultsCollectorBasePtr>
  57     TestResultsCollectors;
  58     TestResultsCollectors m_collectors;
  59
  60     std::string m_startTestId;
  61     bool m_runIgnored;
  62
  63     std::queue<std::string> m_failReason;
  64
  65   public:
  66     TestRunner() :
  67         m_currentTestCase(nullptr)
  68       , m_terminate(false)
  69       , m_allowChildLogs(false)
  70       , m_deferDeepness(0U)
  71       , m_firstDeferredExceptionType(DeferredExceptionType::DEFERRED_FAILED)
  72     {}
  73
  74     void beginPerformance(std::chrono::system_clock::duration maxDurationInMicroseconds);
  75     void endPerformance();
  76     void setCurrentTestCasePerformanceResult(const PerformanceResultPtr &performance);
  77     ConstPerformanceResultPtr getCurrentTestCasePerformanceResult();
  78
  79     void addFailReason(const std::string &reason);
  80
  81     typedef void (*TestCase)();
  82
  83   private:
  84     struct TestCaseStruct
  85     {
  86         std::string name;
  87         TestCase proc;
  88         PerformanceResultPtr performance;
  89
  90         bool operator <(const TestCaseStruct &other) const
  91         {
  92             return name < other.name;
  93         }
  94
  95         bool operator ==(const TestCaseStruct &other) const
  96         {
  97             return name == other.name;
  98         }
  99
 100         TestCaseStruct(const std::string &n, TestCase p) :
 101             name(n),
 102             proc(p)
 103         {}
 104     };
 105
 106     typedef std::list<TestCaseStruct> TestCaseStructList;
 107     typedef std::map<std::string, TestCaseStructList> TestCaseGroupMap;
 108     TestCaseGroupMap m_testGroups;
 109
 110     TestCaseStruct * m_currentTestCase;
 111
 112     typedef std::set<std::string> SelectedTestNameSet;
 113     SelectedTestNameSet m_selectedTestNamesSet;
 114     typedef std::set<std::string> SelectedTestGroupSet;
 115     SelectedTestGroupSet m_selectedTestGroupSet;
 116     std::string m_currentGroup;
 117
 118     DPL::Atomic m_totalAssertions;
 119
 120     // Terminate without any logs.
 121     // Some test requires to call fork function.
 122     // Child process must not produce any logs and should die quietly.
 123     bool m_terminate;
 124     bool m_allowChildLogs;
 125
 126     void Banner();
 127     void InvalidArgs(const std::string& message = "Invalid arguments!");
 128     void Usage();
 129
 130     bool filterGroupsByXmls(const std::vector<std::string> & files);
 131     bool filterByXML(std::map<std::string, bool> & casesMap);
 132     void normalizeXMLTag(std::string& str, const std::string& testcase);
 133
 134     void RunTestCase(const TestCaseStruct& testCase);
 135
 136     void setCurrentTestCase(TestCaseStruct* testCase);
 137     TestCaseStruct *getCurrentTestCase();
 138
 139     void RunTests();
 140
 141     std::string getConcatedFailReason(const std::string &reason);
 142
 143     void CollectResult(const std::string& id, const TestResult &result);
 144
 145   public:
 146     void MarkAssertion();
 147
 148     void RegisterTest(const char *testName, TestCase proc);
 149     void InitGroup(const char* name);
 150
 151     int ExecTestRunner(int argc, char *argv[]);
 152     typedef std::vector<std::string> ArgsList;
 153     int ExecTestRunner(ArgsList args);
 154     bool getRunIgnored() const;
 155     // The runner will terminate as soon as possible (after current test).
 156     void Terminate();
 157     bool GetAllowChildLogs();
 158
 159     void deferFailedException(const DPL::Test::TestFailed &ex);
 160     void deferIgnoredException(const DPL::Test::TestIgnored &ex);
 161     void deferBegin();
 162     void deferEnd();
 163
 164 private:
 165     std::vector<std::string> m_deferredExceptionsMessages;
 166     std::size_t m_deferDeepness;
 167     enum DeferredExceptionType {
 168         DEFERRED_FAILED,
 169         DEFERRED_IGNORED,
 170     } m_firstDeferredExceptionType;
 171     DPL::Test::TestFailed m_firstDeferredFail;
 172     DPL::Test::TestIgnored m_firstDeferredIgnore;
 173 };
 174
 175 typedef DPL::Singleton<TestRunner> TestRunnerSingleton;
 176 }
 177 } // namespace DPL
 178
 179 #define RUNNER_TEST_GROUP_INIT(GroupName)                                 \
 180     static int Static##GroupName##Init()                                  \
 181     {                                                                     \
 182         DPL::Test::TestRunnerSingleton::Instance().InitGroup(#GroupName); \
 183         return 0;                                                         \
 184     }                                                                     \
 185     const int DPL_UNUSED Static##GroupName##InitVar =                     \
 186         Static##GroupName##Init();
 187
 188 #define RUNNER_TEST(Proc)                                                      \
 189     void Proc();                                                               \
 190     static int Static##Proc##Init()                                            \
 191     {                                                                          \
 192         DPL::Test::TestRunnerSingleton::Instance().RegisterTest(#Proc, &Proc); \
 193         return 0;                                                              \
 194     }                                                                          \
 195     const int DPL_UNUSED Static##Proc##InitVar = Static##Proc##Init();         \
 196     void Proc()
 197
 198
 199 /**
 200  * ASSERT MACROS
 201  *
 202  * Use them to create assertions in test cases. To do that put them inside test
 203  * body. Failing assertion indicates failing test.
 204  */
 205
 206 #define RUNNER_ASSERT_MSG(test, message)                                              \
 207     do                                                                                \
 208     {                                                                                 \
 209         DPL::Test::TestRunnerSingleton::Instance().MarkAssertion();                   \
 210                                                                                       \
 211         if (!(test))                                                                  \
 212         {                                                                             \
 213             std::ostringstream assertMsg;                                             \
 214             assertMsg << message << DPL::gdbbacktrace();                              \
 215             DPL::Test::TestFailed e(#test,                                            \
 216                                     __FILE__,                                         \
 217                                     __LINE__,                                         \
 218                                     assertMsg.str());                                 \
 219             if (!std::uncaught_exception())                                           \
 220                 throw e;                                                              \
 221             DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
 222         }                                                                             \
 223     } while (0)
 224
 225 #define RUNNER_ASSERT_ERRNO_MSG(test, message)                                        \
 226     do                                                                                \
 227     {                                                                                 \
 228         DPL::Test::TestRunnerSingleton::Instance().MarkAssertion();                   \
 229                                                                                       \
 230         if (!(test))                                                                  \
 231         {                                                                             \
 232             const char *err = strerror(errno);                                        \
 233             std::ostringstream assertMsg;                                             \
 234             assertMsg << message;                                                     \
 235             if (!assertMsg.str().empty())                                             \
 236                 assertMsg << ". ";                                                    \
 237             assertMsg << err << DPL::gdbbacktrace();                                  \
 238             DPL::Test::TestFailed e(#test,                                            \
 239                                     __FILE__,                                         \
 240                                     __LINE__,                                         \
 241                                     assertMsg.str());                                 \
 242             if (!std::uncaught_exception())                                           \
 243                 throw e;                                                              \
 244             DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
 245         }                                                                             \
 246     } while (0)
 247
 248 #define RUNNER_ASSERT_ERRNO(test) \
 249     RUNNER_ASSERT_ERRNO_MSG(test, "")
 250
 251 #define RUNNER_FAIL_MSG(message) \
 252     RUNNER_ASSERT_MSG(false, message)
 253
 254 #define RUNNER_ASSERT(test) \
 255     RUNNER_ASSERT_MSG(test, "")
 256
 257 /**
 258  * IGNORE MACRO
 259  *
 260  * When test reaches this macro call, its furhter code will be ignored.
 261  * To ignore whole test, put this macro call at the beginning of this tests
 262  * body.
 263  */
 264
 265 #define RUNNER_IGNORED_MSG(message)                    \
 266     do                                                 \
 267     {                                                  \
 268         std::ostringstream assertMsg;                  \
 269         assertMsg << message;                          \
 270         throw DPL::Test::TestIgnored(assertMsg.str()); \
 271     } while (0)
 272
 273 /**
 274  * PERF MACROS
 275  *
 276  * Use these macros to do the time measurement. The first macro will start time measurement,
 277  * the second will gather the result. These macros can be used only once per test-case.
 278  * The result of time measurement will be displayed only if the test will pass.
 279  * Notice that these macros will work only if will be used in parent process. If these
 280  * macros will be used in child process then there will be no time measure results printed.
 281  * This macro in multiprocess tests has effect only if used in parent process. This macro
 282  * used in child process in multiprocess test has no effect.
 283  * The precision of measurement is 1 microsecond - the smallest time value that can be
 284  * measured is 0.000001s.
 285  * The time measure results will be printed only specific output format:
 286  *     - text
 287  *     - html
 288  *     - xml
 289  */
 290
 291 #define RUNNER_PERF_TEST_BEGIN(maxTime)                                                \
 292     do {                                                                               \
 293         DPL::Test::TestRunnerSingleton::Instance().beginPerformance(                   \
 294             std::chrono::microseconds{static_cast<long long int>(maxTime*1000000.0)}); \
 295     } while (0)
 296
 297 #define RUNNER_PERF_TEST_END()                                       \
 298     do {                                                             \
 299         DPL::Test::TestRunnerSingleton::Instance().endPerformance(); \
 300     } while (0)
 301
 302 /**
 303  * MSG MACROS
 304  *
 305  * Use these macros to print error messages during test run time
 306  */
 307
 308 #define RUNNER_ERROR_MSG(message)                             \
 309     do {                                                      \
 310         std::cerr << DPL::Colors::Text::RED_BEGIN << message  \
 311                   << DPL::Colors::Text::RED_END << std::endl; \
 312     } while (0)
 313
 314 /**
 315  * DEFER MACROS
 316  *
 317  * Use them to defer fails and ignores in test cases.
 318  * Some code constructions disallow to throw. Such places can be surrounded
 319  * with RUNNER_DEFER_SCOPE macro. RUNNER_DEFER_TRYCATCH macro can be used to catch possibly thrown
 320  * exceptions within such scope. Possibly catched exceptions will be rethrown
 321  * when leaving RUNNER_DEFER_SCOPE scope.
 322  * Macros can be safely nested.
 323  */
 324
 325
 326 #define RUNNER_DEFER_TRYCATCH(scope)                                              \
 327     do {                                                                          \
 328         try                                                                       \
 329         {                                                                         \
 330             scope                                                                 \
 331         }                                                                         \
 332         catch (const DPL::Test::TestFailed &ex)                                   \
 333         {                                                                         \
 334             DPL::Test::TestRunnerSingleton::Instance().deferFailedException(ex);  \
 335         }                                                                         \
 336         catch (const DPL::Test::TestIgnored &ex)                                  \
 337         {                                                                         \
 338             DPL::Test::TestRunnerSingleton::Instance().deferIgnoredException(ex); \
 339         }                                                                         \
 340     } while (0)                                                                   \
 341
 342 #define RUNNER_DEFER_SCOPE(scope)                                \
 343     do {                                                         \
 344         DPL::Test::TestRunnerSingleton::Instance().deferBegin(); \
 345         scope                                                    \
 346         DPL::Test::TestRunnerSingleton::Instance().deferEnd();   \
 347     } while (0)                                                  \
 348
 349 #endif // DPL_TEST_RUNNER_H