Added RUNNER_PERF_TEST_BEGIN(max_time) and RUNNER_PERF_TEST_END() macros.
Add both of these macros in test when you want to do the time measurement.
The first macro is used to start the time measurement, the second is used to
end the measuement. The result will be displayed if and only if the test will
pass. RUNNER_PERF_TEST_BEGIN(time), takes one parameter (type of double) - the
expected time. If the measured time will be shorter than expected then result
will be print in console in green color, otherwise the result will be print
in red. If other output method will be choosen then measured time and max time
(if defined) will be displayed. In TAP output format there is no preformance
results. If you don't want to give any param as the expected time then put 0 or
any negative value (any value <= 0) as a param (RUNNER_PERF_TEST_BEGIN(0)) and
the param will be ignored. In that case the result in console will be always
printed in white.
The precision of measurement is 1 microsecond - the smallest time value
that can be measured is 0.000001s.
Remarks:
* The result of time measurement will be displayed only if the test will pass.
* Make sure that you use each of these macros at most ONCE in each test. In
the other case the result of first measurement will be overrwriten by the
second.
* Make sure that you use macros in right order.
* In case of RUNNER_MULTIPROCESS_TEST the time measurement will wokrs only if
the macors will be used in parent process. If you will use these macros
in the child code then you will see no result.
* The performance results will be displayed only in these output format:
- text
- html
- xml
- csv
[Problem] No framework for performance tests
[Cause] N/A
[Solution] Added RUNNER_PERF_TEST_BEGIN(expected_time) and RUNNER_PERF_TEST_END()
macros.
[Verification] Add these macros to a few passing tests and a few failing tests,
and check if results are displayed correctly.
Change-Id: I9eebaade094fbdf1d2af34e7da2871b7307f89c6
Signed-off-by: Janusz Kozerski <j.kozerski@samsung.com>
#include <vector>
#include <list>
#include <map>
+#include <chrono>
#include <string>
#include <memory>
virtual void CollectResult(const std::string& id,
const std::string& description,
const FailStatus::Type status = FailStatus::NONE,
- const std::string& reason = "") = 0;
+ const std::string& reason = "",
+ const bool& isPerformanceTest = false,
+ const std::chrono::system_clock::duration& performanceTime = std::chrono::microseconds::zero(),
+ const std::chrono::system_clock::duration& performanceMaxTime = std::chrono::microseconds::zero()) = 0;
virtual std::string CollectorSpecificHelp() const
{
return "";
#include <sstream>
#include <string>
#include <vector>
+#include <chrono>
#include <list>
#include <set>
#include <map>
public:
TestRunner() :
- m_terminate(false)
+ m_currentTestCase(NULL)
+ , m_terminate(false)
, m_allowChildLogs(false)
{}
+ void beginPerformanceTestTime(std::chrono::system_clock::duration maxTimeInMicroseconds);
+ void endPerformanceTestTime();
+ void getCurrentTestCasePerformanceResult(bool& isPerformanceTest,
+ std::chrono::system_clock::duration& result,
+ std::chrono::system_clock::duration& resultMax);
+ void setCurrentTestCasePerformanceResult(bool isPerformanceTest,
+ std::chrono::system_clock::duration result,
+ std::chrono::system_clock::duration resultMax);
+
typedef void (*TestCase)();
private:
std::string name;
TestCase proc;
+ bool m_isPerformanceTest;
+ std::chrono::system_clock::time_point m_performanceTestStartTime;
+ std::chrono::system_clock::duration m_performanceTestDurationTime;
+ std::chrono::system_clock::duration m_performanceMaxTime;
+
bool operator <(const TestCaseStruct &other) const
{
return name < other.name;
TestCaseStruct(const std::string &n, TestCase p) :
name(n),
- proc(p)
+ proc(p),
+ m_isPerformanceTest(false)
{}
};
typedef std::map<std::string, TestCaseStructList> TestCaseGroupMap;
TestCaseGroupMap m_testGroups;
+ TestCaseStruct * m_currentTestCase;
+
typedef std::set<std::string> SelectedTestNameSet;
SelectedTestNameSet m_selectedTestNamesSet;
typedef std::set<std::string> SelectedTestGroupSet;
Status RunTestCase(const TestCaseStruct& testCase);
+ void setCurrentTestCase(TestCaseStruct* testCase);
+ TestCaseStruct *getCurrentTestCase();
+
void RunTests();
void CollectResult(const std::string& id,
const std::string& description,
const TestResultsCollectorBase::FailStatus::Type status
= TestResultsCollectorBase::FailStatus::NONE,
- const std::string& reason = std::string());
+ const std::string& reason = std::string(),
+ const bool& isPerformanceTest = false,
+ const std::chrono::system_clock::duration& performanceTime = std::chrono::microseconds::zero(),
+ const std::chrono::system_clock::duration& performanceMaxTime = std::chrono::microseconds::zero());
public:
class TestFailed
assertMsg.str()); \
} while (0)
+/*
+ * Use these macros to do the time measurement. The first macro will start time measurement,
+ * the second will gather the result. These macros can be used only once per test-case.
+ * The result of time measurement will be displayed only if the test will pass.
+ * Notice that these macros will work only if will be used in parent process. If these
+ * macros will be used in child process then there will be no time measure results printed.
+ * This macro in multiprocess tests has effect only if used in parent process. This macro
+ * used in child process in multiprocess test has no effect.
+ * The precision of measurement is 1 microsecond - the smallest time value that can be
+ * measured is 0.000001s.
+ * The time measure results will be printed only specific output format:
+ * - text
+ * - html
+ * - xml
+ * - csv
+ * In TAP format performance result will not be displayed.
+ */
+#define RUNNER_PERF_TEST_BEGIN(maxTime) \
+ do { \
+ DPL::Test::TestRunnerSingleton::Instance().beginPerformanceTestTime( \
+ std::chrono::microseconds{static_cast<long long int>(maxTime*1000000.0)}); \
+ } while (0)
+
+#define RUNNER_PERF_TEST_END() \
+ do { \
+ DPL::Test::TestRunnerSingleton::Instance().endPerformanceTestTime(); \
+ } while (0)
+
#endif // DPL_TEST_RUNNER_H
virtual ~PipeWrapper();
Status send(int code, std::string &message);
+ Status sendTime(int code,
+ std::chrono::system_clock::duration time,
+ std::chrono::system_clock::duration timeMax);
- Status receive(int &code, std::string &data, time_t deadline);
+ Status receive(int &code,
+ int &msgType,
+ std::string &data,
+ std::chrono::system_clock::duration &time,
+ std::chrono::system_clock::duration &timeMax,
+ time_t deadline);
void closeAll();
protected:
std::string toBinaryString(int data);
+ std::string toBinaryString(std::chrono::system_clock::duration data);
void closeHelp(int desc);
*/
#include <cstddef>
#include <dpl/test/test_results_collector.h>
+#include <dpl/availability.h>
#include <dpl/colors.h>
#include <dpl/assert.h>
#include <dpl/foreach.h>
#define GREEN_RESULT_OK "[%s%s%s]\n", BOLD_GREEN_BEGIN, " OK ", \
BOLD_GREEN_END
+#define GREEN_RESULT_OK_TIME "[%s%s%s] [elapsed: %0.3fms]\n", BOLD_GREEN_BEGIN, \
+ " OK ", BOLD_GREEN_END
+#define GREEN_RESULT_OK_TIME_MAX(elapsed, max) \
+ "[%s%s%s] %s[elapsed: %0.3fms, expected < %0.3fms]%s\n", BOLD_GREEN_BEGIN, \
+ " OK ", BOLD_GREEN_END, BOLD_GREEN_BEGIN, elapsed, max, BOLD_GREEN_END
+#define GREEN_RESULT_OK_TIME_TOO_LONG(elapsed, max) \
+ "[%s%s%s] %s[elapsed: %0.3fms, expected < %0.3fms]%s\n", BOLD_GREEN_BEGIN, \
+ " OK ", BOLD_GREEN_END, BOLD_RED_BEGIN, elapsed, max, BOLD_RED_END
+
+namespace { /* anonymous namespace */
+// Get duration as a fraction of millisecond (max precision is 1 microsecond)
+double get_milliseconds (const std::chrono::system_clock::duration& performanceTime)
+{
+ return (static_cast<double>(std::chrono::duration_cast<std::chrono::microseconds>
+ (performanceTime).count()))/1000.0;
+}
+} /* anonymous namespace */
namespace DPL {
namespace Test {
virtual void CollectResult(const std::string& id,
const std::string& /*description*/,
const FailStatus::Type status = FailStatus::NONE,
- const std::string& reason = "")
+ const std::string& reason = "",
+ const bool& isPerformanceTest = true,
+ const std::chrono::system_clock::duration& performanceTime = std::chrono::microseconds::zero(),
+ const std::chrono::system_clock::duration& performanceMaxTime = std::chrono::microseconds::zero())
{
using namespace DPL::Colors::Text;
std::string tmp = "'" + id + "' ...";
printf("Running test case %-60s", tmp.c_str());
switch (status) {
case TestResultsCollectorBase::FailStatus::NONE:
+ if (isPerformanceTest) {
+ if (performanceMaxTime <= std::chrono::microseconds::zero()) {
+ printf(GREEN_RESULT_OK_TIME,
+ get_milliseconds(performanceTime));
+ break;
+ }
+ else {
+ if (performanceTime > performanceMaxTime)
+ printf(GREEN_RESULT_OK_TIME_TOO_LONG(
+ get_milliseconds(performanceTime),
+ get_milliseconds(performanceMaxTime)));
+ else
+ printf(GREEN_RESULT_OK_TIME_MAX(
+ get_milliseconds(performanceTime),
+ get_milliseconds(performanceMaxTime)));
+ break;
+ }
+ }
printf(GREEN_RESULT_OK);
break;
case TestResultsCollectorBase::FailStatus::FAILED:
virtual void CollectResult(const std::string& id,
const std::string& /*description*/,
const FailStatus::Type status = FailStatus::NONE,
- const std::string& reason = "")
+ const std::string& reason = "",
+ const bool& isPerformanceTest = false,
+ const std::chrono::system_clock::duration& performanceTime = std::chrono::microseconds::zero(),
+ const std::chrono::system_clock::duration& performanceMaxTime = std::chrono::microseconds::zero())
{
using namespace DPL::Colors::Html;
std::string tmp = "'" + id + "' ...";
fprintf(m_fp.Get(), "Running test case %-100s", tmp.c_str());
switch (status) {
case TestResultsCollectorBase::FailStatus::NONE:
+ if (isPerformanceTest) {
+ if (performanceMaxTime <= std::chrono::microseconds::zero()) {
+ fprintf(m_fp.Get(), GREEN_RESULT_OK_TIME,
+ get_milliseconds(performanceTime));
+ break;
+ } else {
+ if (performanceTime > performanceMaxTime)
+ fprintf(m_fp.Get(), GREEN_RESULT_OK_TIME_TOO_LONG(
+ get_milliseconds(performanceTime),
+ get_milliseconds(performanceMaxTime)));
+ else
+ fprintf(m_fp.Get(), GREEN_RESULT_OK_TIME_MAX(
+ get_milliseconds(performanceTime),
+ get_milliseconds(performanceMaxTime)));
+ break;
+ }
+ }
fprintf(m_fp.Get(), GREEN_RESULT_OK);
break;
case TestResultsCollectorBase::FailStatus::FAILED:
virtual void CollectResult(const std::string& id,
const std::string& /*description*/,
const FailStatus::Type status = FailStatus::NONE,
- const std::string& reason = "")
+ const std::string& reason = "",
+ const bool& isPerformanceTest = false,
+ const std::chrono::system_clock::duration& performanceTime = std::chrono::microseconds::zero(),
+ const std::chrono::system_clock::duration& performanceMaxTime = std::chrono::microseconds::zero())
{
m_resultBuffer.erase();
m_resultBuffer.append("\t\t<testcase name=\"");
m_resultBuffer.append("\"");
switch (status) {
case TestResultsCollectorBase::FailStatus::NONE:
+ if (isPerformanceTest) {
+ if (performanceMaxTime <= std::chrono::microseconds::zero()) {
+ m_resultBuffer.append(" status=\"OK\" time=\"");
+ std::ostringstream ostr;
+ ostr << performanceTime.count();
+ m_resultBuffer.append(ostr.str());
+ m_resultBuffer.append("\"/>\n");
+ break;
+ } else {
+ m_resultBuffer.append(" status=\"OK\" time=\"");
+ std::ostringstream ostr;
+ ostr << performanceTime.count();
+ m_resultBuffer.append(ostr.str());
+ m_resultBuffer.append("\" time_expected=\"");
+ ostr.str("");
+ ostr << performanceMaxTime.count();
+ m_resultBuffer.append(ostr.str());
+ m_resultBuffer.append("\"/>\n");
+ break;
+ }
+ }
m_resultBuffer.append(" status=\"OK\"/>\n");
break;
case TestResultsCollectorBase::FailStatus::FAILED:
virtual void Start()
{
- printf("GROUP;ID;RESULT;REASON\n");
+ printf("GROUP;ID;RESULT;REASON;ELAPSED [s];EXPECTED [s]\n");
}
virtual void CollectCurrentTestGroupName(const std::string& name)
virtual void CollectResult(const std::string& id,
const std::string& /*description*/,
const FailStatus::Type status = FailStatus::NONE,
- const std::string& reason = "")
+ const std::string& reason = "",
+ const bool& isPerformanceTest = false,
+ const std::chrono::system_clock::duration& performanceTime = std::chrono::microseconds::zero(),
+ const std::chrono::system_clock::duration& performanceMaxTime = std::chrono::microseconds::zero())
{
std::string statusMsg = "";
switch (status) {
- case TestResultsCollectorBase::FailStatus::NONE: statusMsg = "OK";
+ case TestResultsCollectorBase::FailStatus::NONE:
+ statusMsg = "OK";
+ if (isPerformanceTest) {
+ statusMsg.append(";;");
+ std::ostringstream ostr;
+ ostr << performanceTime.count();
+ statusMsg.append(ostr.str());
+ if (performanceMaxTime <= std::chrono::microseconds::zero()) {
+ statusMsg.append(";");
+ ostr.str("");
+ ostr << performanceMaxTime.count();
+ statusMsg.append(ostr.str());
+ }
+ }
break;
case TestResultsCollectorBase::FailStatus::FAILED: statusMsg = "FAILED";
break;
virtual void CollectResult(const std::string& id,
const std::string& description,
const FailStatus::Type status = FailStatus::NONE,
- const std::string& reason = "")
+ const std::string& reason = "",
+ const bool& isPerformanceTest = false,
+ const std::chrono::system_clock::duration& performanceTime = std::chrono::microseconds::zero(),
+ const std::chrono::system_clock::duration& performanceMaxTime = std::chrono::microseconds::zero())
{
+ /* Remove unused variable warning */
+ DPL_UNUSED_PARAM(isPerformanceTest);
+ DPL_UNUSED_PARAM(performanceTime);
+ DPL_UNUSED_PARAM(performanceMaxTime);
+
m_testIndex++;
switch (status) {
case TestResultsCollectorBase::FailStatus::NONE:
TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase)
{
+ setCurrentTestCase(&(const_cast<TestCaseStruct &>(testCase)));
try {
testCase.proc();
} catch (const TestFailed &e) {
"",
TestResultsCollectorBase::FailStatus::FAILED,
e.GetMessage());
+
+ setCurrentTestCase(NULL);
return FAILED;
} catch (const Ignored &e) {
if (m_runIgnored) {
e.GetMessage());
}
+ setCurrentTestCase(NULL);
return IGNORED;
} catch (const DPL::Exception &e) {
// DPL exception failure
TestResultsCollectorBase::FailStatus::INTERNAL,
"DPL exception:" + e.GetMessage());
+ setCurrentTestCase(NULL);
return FAILED;
} catch (const std::exception &) {
// std exception failure
TestResultsCollectorBase::FailStatus::INTERNAL,
"std exception");
+ setCurrentTestCase(NULL);
return FAILED;
} catch (...) {
// Unknown exception failure
TestResultsCollectorBase::FailStatus::INTERNAL,
"unknown exception");
+ setCurrentTestCase(NULL);
return FAILED;
}
CollectResult(testCase.name,
"",
- TestResultsCollectorBase::FailStatus::NONE);
+ TestResultsCollectorBase::FailStatus::NONE,
+ "",
+ testCase.m_isPerformanceTest,
+ testCase.m_performanceTestDurationTime,
+ testCase.m_performanceMaxTime);
+ setCurrentTestCase(NULL);
// Everything OK
return PASS;
fprintf(stderr, "%s%s%s\n\n", GREEN_BEGIN, "Finished", GREEN_END);
}
+TestRunner::TestCaseStruct *TestRunner::getCurrentTestCase()
+{
+ return m_currentTestCase;
+}
+
+void TestRunner::setCurrentTestCase(TestCaseStruct* testCase)
+{
+ m_currentTestCase = testCase;
+}
+
+void TestRunner::beginPerformanceTestTime(std::chrono::system_clock::duration maxTimeInMicroseconds)
+{
+ TestCaseStruct* testCase = getCurrentTestCase();
+ if (!testCase)
+ return;
+
+ testCase->m_isPerformanceTest = true;
+ testCase->m_performanceMaxTime = maxTimeInMicroseconds;
+ testCase->m_performanceTestStartTime = std::chrono::system_clock::now();
+
+ // Set result to 0 microseconds. Display 0ms result when end macro is missing.
+ testCase->m_performanceTestDurationTime = std::chrono::microseconds::zero();
+}
+
+void TestRunner::endPerformanceTestTime()
+{
+ TestCaseStruct* testCase = getCurrentTestCase();
+ if (!testCase)
+ return;
+
+ testCase->m_performanceTestDurationTime = std::chrono::system_clock::now() -
+ testCase->m_performanceTestStartTime;
+}
+
+void TestRunner::getCurrentTestCasePerformanceResult(bool& isPerformanceTest,
+ std::chrono::system_clock::duration& result,
+ std::chrono::system_clock::duration& resultMax)
+{
+ TestCaseStruct* testCase = getCurrentTestCase();
+ if (!testCase || !(testCase->m_isPerformanceTest)){
+ isPerformanceTest = false;
+ return;
+ }
+
+ isPerformanceTest = testCase->m_isPerformanceTest;
+ result = testCase->m_performanceTestDurationTime;
+ resultMax = testCase->m_performanceMaxTime;
+}
+
+void TestRunner::setCurrentTestCasePerformanceResult(bool isPerformanceTest,
+ std::chrono::system_clock::duration result,
+ std::chrono::system_clock::duration resultMax)
+{
+ TestCaseStruct* testCase = getCurrentTestCase();
+ if (!testCase)
+ return;
+
+ testCase->m_isPerformanceTest = isPerformanceTest;
+ testCase->m_performanceTestDurationTime = result;
+ testCase->m_performanceMaxTime = resultMax;
+}
+
+
void TestRunner::CollectResult(
const std::string& id,
const std::string& description,
const TestResultsCollectorBase::FailStatus::Type status,
- const std::string& reason)
+ const std::string& reason,
+ const bool& isPerformanceTest,
+ const std::chrono::system_clock::duration& performanceTestDurationTime,
+ const std::chrono::system_clock::duration& performanceMaxTime)
{
std::for_each(m_collectors.begin(),
m_collectors.end(),
collector.second->CollectResult(id,
description,
status,
- reason);
+ reason,
+ isPerformanceTest,
+ performanceTestDurationTime,
+ performanceMaxTime);
});
}
const int CHILD_TEST_PASS = 1;
const int CHILD_TEST_IGNORED = 2;
+const int MSG_TYPE_MESSAGE = 0; // sizeof(Message) + Message
+const int MSG_TYPE_PERF_TIME = 1; // perfTime + maxTime
+
int closeOutput() {
int devnull;
int retcode = -1;
std::ostringstream output;
output << toBinaryString(code);
+ output << toBinaryString(MSG_TYPE_MESSAGE);
output << toBinaryString(static_cast<int>(message.size()));
output << message;
return SUCCESS;
}
-PipeWrapper::Status PipeWrapper::receive(int &code, std::string &data, time_t deadline)
+PipeWrapper::Status PipeWrapper::sendTime(int code,
+ std::chrono::system_clock::duration time,
+ std::chrono::system_clock::duration timeMax)
+{
+ if (m_pipefd[1] == PIPE_CLOSED) {
+ return ERROR;
+ }
+
+ std::ostringstream output;
+ output << toBinaryString(code);
+ output << toBinaryString(MSG_TYPE_PERF_TIME);
+ output << toBinaryString(time);
+ output << toBinaryString(timeMax);
+
+ std::string binary = output.str();
+ int size = binary.size();
+
+ if ((writeHelp(&size,
+ sizeof(int)) == ERROR) ||
+ (writeHelp(binary.c_str(), size) == ERROR))
+ {
+ return ERROR;
+ }
+ return SUCCESS;
+}
+
+PipeWrapper::Status PipeWrapper::receive(int &code,
+ int &msgType,
+ std::string &data,
+ std::chrono::system_clock::duration &time,
+ std::chrono::system_clock::duration &timeMax,
+ time_t deadline)
{
if (m_pipefd[0] == PIPE_CLOSED) {
return ERROR;
queue.AppendCopy(&buffer[0], size);
queue.FlattenConsume(&code, sizeof(int));
- queue.FlattenConsume(&size, sizeof(int));
-
- buffer.resize(size);
-
- queue.FlattenConsume(&buffer[0], size);
- data.assign(buffer.begin(), buffer.end());
+ queue.FlattenConsume(&msgType, sizeof(int));
+
+ switch (msgType) {
+ case MSG_TYPE_MESSAGE:
+ queue.FlattenConsume(&size, sizeof(int));
+
+ buffer.resize(size);
+
+ queue.FlattenConsume(&buffer[0], size);
+ data.assign(buffer.begin(), buffer.end());
+ break;
+ case MSG_TYPE_PERF_TIME:
+ queue.FlattenConsume(&time, sizeof(std::chrono::system_clock::duration));
+ queue.FlattenConsume(&timeMax, sizeof(std::chrono::system_clock::duration));
+ break;
+ default:
+ return ERROR;
+ }
} catch (DPL::BinaryQueue::Exception::Base &e) {
return ERROR;
}
return std::string(buffer, buffer + sizeof(int));
}
+std::string PipeWrapper::toBinaryString(std::chrono::system_clock::duration data)
+{
+ char buffer[sizeof(std::chrono::system_clock::duration)];
+ memcpy(buffer, &data, sizeof(std::chrono::system_clock::duration));
+ return std::string(buffer, buffer + sizeof(std::chrono::system_clock::duration));
+}
+
void PipeWrapper::closeHelp(int desc)
{
if (m_pipefd[desc] != PIPE_CLOSED) {
pipe.setUsage(PipeWrapper::READONLY);
int code;
+ int msgType;
+ std::chrono::system_clock::duration time_m;
+ std::chrono::system_clock::duration timeMax_m;
std::string message;
- int pipeReturn = pipe.receive(code, message, time(0) + 10);
+ int pipeReturn = pipe.receive(code, msgType, message, time_m, timeMax_m, time(0) + 10);
if (pipeReturn != PipeWrapper::SUCCESS) { // Timeout or reading error
pipe.closeAll();
throw TestRunner::TestFailed("Reading pipe error");
}
+ if (code == CHILD_TEST_PASS && msgType == MSG_TYPE_PERF_TIME) {
+ DPL::Test::TestRunnerSingleton::Instance().setCurrentTestCasePerformanceResult(true,
+ time_m,
+ timeMax_m);
+ }
+
if (code == CHILD_TEST_FAIL) {
throw TestRunner::TestFailed(message);
} else if (code == CHILD_TEST_IGNORED) {
int code = CHILD_TEST_PASS;
std::string msg;
+ bool isPerformanceTest;
+ std::chrono::system_clock::duration time_m;
+ std::chrono::system_clock::duration timeMax_m;
bool allowLogs = TestRunnerSingleton::Instance().GetAllowChildLogs();
closeOutput();
}
- pipe.send(code, msg);
+ DPL::Test::TestRunnerSingleton::Instance().getCurrentTestCasePerformanceResult(isPerformanceTest,
+ time_m,
+ timeMax_m);
+
+ if (code == CHILD_TEST_PASS && isPerformanceTest){
+ pipe.sendTime(code,
+ time_m,
+ timeMax_m);
+ } else {
+ pipe.send(code, msg);
+ }
}
}
} // namespace Test
projects / platform / framework / web / wrt-commons.git / commitdiff