2 * Copyright (c) 2011-2015 Samsung Electronics Co., Ltd All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
18 * @author Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
19 * @author Lukasz Wrzosek (l.wrzosek@samsung.com)
21 * @brief This file is the header file of test runner
24 #ifndef DPL_TEST_RUNNER_H
25 #define DPL_TEST_RUNNER_H
39 #include <dpl/atomic.h>
40 #include <dpl/availability.h>
41 #include <dpl/colors.h>
42 #include <dpl/gdbbacktrace.h>
43 #include <dpl/singleton.h>
44 #include <dpl/test/test_results_collector.h>
50 typedef std::map<std::string, TestResultsCollectorBasePtr>
51 TestResultsCollectors;
52 TestResultsCollectors m_collectors;
54 std::string m_startTestId;
57 std::queue<std::string> m_failReason;
61 m_currentTestCase(nullptr)
63 , m_allowChildLogs(false)
66 void beginPerformanceTestTime(std::chrono::system_clock::duration maxTimeInMicroseconds);
67 void endPerformanceTestTime();
68 void getCurrentTestCasePerformanceResult(bool& isPerformanceTest,
69 std::chrono::system_clock::duration& result,
70 std::chrono::system_clock::duration& resultMax);
71 void setCurrentTestCasePerformanceResult(bool isPerformanceTest,
72 std::chrono::system_clock::duration result,
73 std::chrono::system_clock::duration resultMax);
75 void addFailReason(const std::string &reason);
77 typedef void (*TestCase)();
85 bool m_isPerformanceTest;
86 std::chrono::system_clock::time_point m_performanceTestStartTime;
87 std::chrono::system_clock::duration m_performanceTestDurationTime;
88 std::chrono::system_clock::duration m_performanceMaxTime;
90 bool operator <(const TestCaseStruct &other) const
92 return name < other.name;
95 bool operator ==(const TestCaseStruct &other) const
97 return name == other.name;
100 TestCaseStruct(const std::string &n, TestCase p) :
103 m_isPerformanceTest(false)
107 typedef std::list<TestCaseStruct> TestCaseStructList;
108 typedef std::map<std::string, TestCaseStructList> TestCaseGroupMap;
109 TestCaseGroupMap m_testGroups;
111 TestCaseStruct * m_currentTestCase;
113 typedef std::set<std::string> SelectedTestNameSet;
114 SelectedTestNameSet m_selectedTestNamesSet;
115 typedef std::set<std::string> SelectedTestGroupSet;
116 SelectedTestGroupSet m_selectedTestGroupSet;
117 std::string m_currentGroup;
119 DPL::Atomic m_totalAssertions;
121 // Terminate without any logs.
122 // Some test requires to call fork function.
123 // Child process must not produce any logs and should die quietly.
125 bool m_allowChildLogs;
128 void InvalidArgs(const std::string& message = "Invalid arguments!");
131 bool filterGroupsByXmls(const std::vector<std::string> & files);
132 bool filterByXML(std::map<std::string, bool> & casesMap);
133 void normalizeXMLTag(std::string& str, const std::string& testcase);
135 enum Status { FAILED, IGNORED, PASS };
137 Status RunTestCase(const TestCaseStruct& testCase);
139 void setCurrentTestCase(TestCaseStruct* testCase);
140 TestCaseStruct *getCurrentTestCase();
144 std::string getConcatedFailReason(const std::string &reason);
146 void CollectResult(const std::string& id,
147 const TestResultsCollectorBase::FailStatus status
148 = TestResultsCollectorBase::FailStatus::NONE,
149 const std::string& reason = std::string(),
150 const bool& isPerformanceTest = false,
151 const std::chrono::system_clock::duration& performanceTime = std::chrono::microseconds::zero(),
152 const std::chrono::system_clock::duration& performanceMaxTime = std::chrono::microseconds::zero());
158 std::string m_message;
164 //! \brief Failed test message creator
166 //! \param[in] aTest string for tested expression
167 //! \param[in] aFile source file name
168 //! \param[in] aLine source file line
169 //! \param[in] aMessage error message
170 TestFailed(const char* aTest,
173 const std::string &aMessage);
175 TestFailed(const std::string &message);
177 std::string GetMessage() const
186 std::string m_message;
192 Ignored(const std::string &message) :
196 std::string GetMessage() const
202 void MarkAssertion();
204 void RegisterTest(const char *testName, TestCase proc);
205 void InitGroup(const char* name);
207 int ExecTestRunner(int argc, char *argv[]);
208 typedef std::vector<std::string> ArgsList;
209 int ExecTestRunner(ArgsList args);
210 bool getRunIgnored() const;
211 // The runner will terminate as soon as possible (after current test).
213 bool GetAllowChildLogs();
216 typedef DPL::Singleton<TestRunner> TestRunnerSingleton;
220 #define RUNNER_TEST_GROUP_INIT(GroupName) \
221 static int Static##GroupName##Init() \
223 DPL::Test::TestRunnerSingleton::Instance().InitGroup(#GroupName); \
226 const int DPL_UNUSED Static##GroupName##InitVar = \
227 Static##GroupName##Init();
229 #define RUNNER_TEST(Proc) \
231 static int Static##Proc##Init() \
233 DPL::Test::TestRunnerSingleton::Instance().RegisterTest(#Proc, &Proc); \
236 const int DPL_UNUSED Static##Proc##InitVar = Static##Proc##Init(); \
243 * Use them to create assertions in test cases. To do that put them inside test
244 * body. Failing assertion indicates failing test.
247 #define RUNNER_ASSERT_MSG(test, message) \
250 DPL::Test::TestRunnerSingleton::Instance().MarkAssertion(); \
254 std::ostringstream assertMsg; \
255 assertMsg << message << DPL::gdbbacktrace(); \
256 DPL::Test::TestRunner::TestFailed e(#test, \
260 if (!std::uncaught_exception()) \
262 DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
266 #define RUNNER_ASSERT_ERRNO_MSG(test, message) \
269 DPL::Test::TestRunnerSingleton::Instance().MarkAssertion(); \
273 const char *err = strerror(errno); \
274 std::ostringstream assertMsg; \
275 assertMsg << message; \
276 if (!assertMsg.str().empty()) \
278 assertMsg << err << DPL::gdbbacktrace(); \
279 DPL::Test::TestRunner::TestFailed e(#test, \
283 if (!std::uncaught_exception()) \
285 DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
289 #define RUNNER_ASSERT_ERRNO(test) \
290 RUNNER_ASSERT_ERRNO_MSG(test, "")
292 #define RUNNER_FAIL_MSG(message) \
293 RUNNER_ASSERT_MSG(false, message)
295 #define RUNNER_ASSERT(test) \
296 RUNNER_ASSERT_MSG(test, "")
301 * When test reaches this macro call, its furhter code will be ignored.
302 * To ignore whole test, put this macro call at the beginning of this tests
306 #define RUNNER_IGNORED_MSG(message) \
309 std::ostringstream assertMsg; \
310 assertMsg << message; \
311 throw DPL::Test::TestRunner::Ignored(assertMsg.str()); \
317 * Use these macros to do the time measurement. The first macro will start time measurement,
318 * the second will gather the result. These macros can be used only once per test-case.
319 * The result of time measurement will be displayed only if the test will pass.
320 * Notice that these macros will work only if will be used in parent process. If these
321 * macros will be used in child process then there will be no time measure results printed.
322 * This macro in multiprocess tests has effect only if used in parent process. This macro
323 * used in child process in multiprocess test has no effect.
324 * The precision of measurement is 1 microsecond - the smallest time value that can be
325 * measured is 0.000001s.
326 * The time measure results will be printed only specific output format:
332 #define RUNNER_PERF_TEST_BEGIN(maxTime) \
334 DPL::Test::TestRunnerSingleton::Instance().beginPerformanceTestTime( \
335 std::chrono::microseconds{static_cast<long long int>(maxTime*1000000.0)}); \
338 #define RUNNER_PERF_TEST_END() \
340 DPL::Test::TestRunnerSingleton::Instance().endPerformanceTestTime(); \
346 * Use these macros to print error messages during test run time
349 #define RUNNER_ERROR_MSG(message) \
351 std::cerr << DPL::Colors::Text::RED_BEGIN << message \
352 << DPL::Colors::Text::RED_END << std::endl; \
355 #endif // DPL_TEST_RUNNER_H