2 * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 * @file test_results_collector.h
18 * @author Lukasz Wrzosek (l.wrzosek@samsung.com)
20 * @brief Implementation file some concrete TestResulstsCollector
23 #include <dpl/test/test_results_collector.h>
24 #include <dpl/colors.h>
25 #include <dpl/assert.h>
26 #include <dpl/foreach.h>
27 #include <dpl/scoped_fclose.h>
28 #include <dpl/exception.h>
29 #include <dpl/errno_string.h>
30 #include <dpl/lexical_cast.h>
31 #include <dpl/availability.h>
40 #define GREEN_RESULT_OK "[%s%s%s]\n", BOLD_GREEN_BEGIN, " OK ", \
46 const char *DEFAULT_HTML_FILE_NAME = "index.html";
47 const char *DEFAULT_TAP_FILE_NAME = "results.tap";
48 const char *DEFAULT_XML_FILE_NAME = "results.xml";
50 bool ParseCollectorFileArg(const std::string &arg, std::string &filename)
52 const std::string argname = "--file=";
53 if (arg.find(argname) == 0 ) {
54 filename = arg.substr(argname.size());
70 void AddTest(TestResultsCollectorBase::FailStatus::Type type)
74 case TestResultsCollectorBase::FailStatus::INTERNAL:
75 case TestResultsCollectorBase::FailStatus::FAILED: ++m_failed;
77 case TestResultsCollectorBase::FailStatus::IGNORED: ++m_ignored;
79 case TestResultsCollectorBase::FailStatus::NONE: ++m_passed;
82 Assert(false && "Bad FailStatus");
86 std::size_t GetTotal() const
90 std::size_t GetPassed() const
94 std::size_t GetSuccesed() const
98 std::size_t GetFailed() const
102 std::size_t GetIgnored() const
106 float GetPassedOrIgnoredPercend() const
108 float passIgnoredPercent =
109 100.0f * (static_cast<float>(m_passed)
110 + static_cast<float>(m_ignored))
111 / static_cast<float>(m_count);
112 return passIgnoredPercent;
116 std::size_t m_failed;
117 std::size_t m_ignored;
118 std::size_t m_passed;
122 class ConsoleCollector :
123 public TestResultsCollectorBase
126 static TestResultsCollectorBase* Constructor();
129 ConsoleCollector() {}
131 virtual void CollectCurrentTestGroupName(const std::string& name)
133 printf("Starting group %s\n", name.c_str());
134 m_currentGroup = name;
137 virtual void Finish()
139 using namespace VcoreDPL::Colors::Text;
142 FOREACH(group, m_groupsStats) {
143 PrintStats(group->first, group->second);
145 PrintStats("All tests together", m_stats);
148 virtual void CollectResult(const std::string& id,
149 const std::string& /*description*/,
150 const FailStatus::Type status = FailStatus::NONE,
151 const std::string& reason = "")
153 using namespace VcoreDPL::Colors::Text;
154 std::string tmp = "'" + id + "' ...";
156 printf("Running test case %-60s", tmp.c_str());
158 case TestResultsCollectorBase::FailStatus::NONE:
159 printf(GREEN_RESULT_OK);
161 case TestResultsCollectorBase::FailStatus::FAILED:
162 PrintfErrorMessage(" FAILED ", reason, true);
164 case TestResultsCollectorBase::FailStatus::IGNORED:
165 PrintfIgnoredMessage("Ignored ", reason, true);
167 case TestResultsCollectorBase::FailStatus::INTERNAL:
168 PrintfErrorMessage("INTERNAL", reason, true);
171 Assert(false && "Bad status");
173 m_stats.AddTest(status);
174 m_groupsStats[m_currentGroup].AddTest(status);
177 void PrintfErrorMessage(const char* type,
178 const std::string& message,
181 using namespace VcoreDPL::Colors::Text;
183 printf("[%s%s%s] %s%s%s\n",
198 void PrintfIgnoredMessage(const char* type,
199 const std::string& message,
202 using namespace VcoreDPL::Colors::Text;
204 printf("[%s%s%s] %s%s%s\n",
219 void PrintStats(const std::string& title, const Statistic& stats)
221 using namespace VcoreDPL::Colors::Text;
222 printf("\n%sResults [%s]: %s\n", BOLD_GREEN_BEGIN,
223 title.c_str(), BOLD_GREEN_END);
224 printf("%s%s%3d%s\n",
229 printf(" %s%s%3d%s\n",
234 printf(" %s%s%3d%s\n",
239 printf(" %s%s%3d%s\n",
247 std::map<std::string, Statistic> m_groupsStats;
248 std::string m_currentGroup;
251 TestResultsCollectorBase* ConsoleCollector::Constructor()
253 return new ConsoleCollector();
256 class HtmlCollector :
257 public TestResultsCollectorBase
260 static TestResultsCollectorBase* Constructor();
263 HtmlCollector() : m_filename(DEFAULT_HTML_FILE_NAME) {}
265 virtual void CollectCurrentTestGroupName(const std::string& name)
267 fprintf(m_fp.Get(), "<b>Starting group %s", name.c_str());
268 m_currentGroup = name;
271 virtual bool Configure()
273 m_fp.Reset(fopen(m_filename.c_str(), "w"));
279 virtual std::string CollectorSpecificHelp() const
281 return "--file=<filename> - name of file for output\n"
282 " default - index.html\n";
285 virtual void Start(int count)
287 DPL_UNUSED_PARAM(count);
288 AssertMsg(!!m_fp, "File handle must not be null");
290 "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0"
291 "Transitional//EN\" "
292 "\"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\""
295 "<html xmlns=\"http://www.w3.org/1999/xhtml\" "
296 "lang=\"en\" dir=\"ltr\">\n");
297 fprintf(m_fp.Get(), "<body style=\"background-color: black;\">\n");
298 fprintf(m_fp.Get(), "<pre>\n");
299 fprintf(m_fp.Get(), "<font color=\"white\">\n");
302 virtual void Finish()
304 using namespace VcoreDPL::Colors::Html;
306 FOREACH(group, m_groupsStats) {
307 PrintStats(group->first, group->second);
309 PrintStats("All tests together", m_stats);
310 fprintf(m_fp.Get(), "</font>\n");
311 fprintf(m_fp.Get(), "</pre>\n");
312 fprintf(m_fp.Get(), "</body>\n");
313 fprintf(m_fp.Get(), "</html>\n");
316 virtual bool ParseCollectorSpecificArg(const std::string& arg)
318 return ParseCollectorFileArg(arg, m_filename);
321 virtual void CollectResult(const std::string& id,
322 const std::string& /*description*/,
323 const FailStatus::Type status = FailStatus::NONE,
324 const std::string& reason = "")
326 using namespace VcoreDPL::Colors::Html;
327 std::string tmp = "'" + id + "' ...";
329 fprintf(m_fp.Get(), "Running test case %-100s", tmp.c_str());
331 case TestResultsCollectorBase::FailStatus::NONE:
332 fprintf(m_fp.Get(), GREEN_RESULT_OK);
334 case TestResultsCollectorBase::FailStatus::FAILED:
335 PrintfErrorMessage(" FAILED ", reason, true);
337 case TestResultsCollectorBase::FailStatus::IGNORED:
338 PrintfIgnoredMessage("Ignored ", reason, true);
340 case TestResultsCollectorBase::FailStatus::INTERNAL:
341 PrintfErrorMessage("INTERNAL", reason, true);
344 Assert(false && "Bad status");
346 m_groupsStats[m_currentGroup].AddTest(status);
347 m_stats.AddTest(status);
350 void PrintfErrorMessage(const char* type,
351 const std::string& message,
354 using namespace VcoreDPL::Colors::Html;
373 void PrintfIgnoredMessage(const char* type,
374 const std::string& message,
377 using namespace VcoreDPL::Colors::Html;
397 void PrintStats(const std::string& name, const Statistic& stats)
399 using namespace VcoreDPL::Colors::Html;
401 m_fp.Get(), "\n%sResults [%s]:%s\n", BOLD_GREEN_BEGIN,
402 name.c_str(), BOLD_GREEN_END);
404 m_fp.Get(), "%s%s%3d%s\n", CYAN_BEGIN,
405 "Total tests: ", stats.GetTotal(), CYAN_END);
407 m_fp.Get(), " %s%s%3d%s\n", CYAN_BEGIN,
408 "Succeeded: ", stats.GetPassed(), CYAN_END);
410 m_fp.Get(), " %s%s%3d%s\n", CYAN_BEGIN,
411 "Failed: ", stats.GetFailed(), CYAN_END);
413 m_fp.Get(), " %s%s%3d%s\n", CYAN_BEGIN,
414 "Ignored: ", stats.GetIgnored(), CYAN_END);
417 std::string m_filename;
420 std::string m_currentGroup;
421 std::map<std::string, Statistic> m_groupsStats;
424 TestResultsCollectorBase* HtmlCollector::Constructor()
426 return new HtmlCollector();
430 public TestResultsCollectorBase
433 static TestResultsCollectorBase* Constructor();
436 XmlCollector() : m_filename(DEFAULT_XML_FILE_NAME) {}
438 virtual void CollectCurrentTestGroupName(const std::string& name)
440 std::size_t pos = GetCurrentGroupPosition();
441 if (std::string::npos != pos) {
444 m_stats = Statistic();
447 pos = m_outputBuffer.find("</testsuites>");
448 if (std::string::npos == pos) {
449 ThrowMsg(VcoreDPL::Exception, "Could not find test suites closing tag");
451 GroupStart(pos, name);
454 void GroupStart(const std::size_t pos, const std::string& name)
456 std::stringstream groupHeader;
457 groupHeader << "\n\t<testsuite";
458 groupHeader << " name=\"" << EscapeSpecialCharacters(name) << "\"";
459 groupHeader << R"( tests="1")"; // include SegFault
460 groupHeader << R"( failures="1")"; // include SegFault
461 groupHeader << R"( skipped="0")";
464 groupHeader << "\n\t\t<testcase name=\"unknown\" status=\"FAILED\">";
466 "\n\t\t\t<failure type=\"FAILED\" message=\"segmentation fault\"/>";
467 groupHeader << "\n\t\t</testcase>";
469 groupHeader << "\n\t</testsuite>";
471 m_outputBuffer.insert(pos - 1, groupHeader.str());
474 virtual bool Configure()
476 m_fp.Reset(fopen(m_filename.c_str(), "w"));
483 virtual std::string CollectorSpecificHelp() const
485 return "--file=<filename> - name of file for output\n"
486 " default - results.xml\n";
489 virtual void Start(int count)
491 AssertMsg(!!m_fp, "File handle must not be null");
492 m_outputBuffer.append("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n");
493 m_outputBuffer.append("<testsuites ");
496 m_outputBuffer.append("total=\"");
497 m_outputBuffer.append(VcoreDPL::lexical_cast<std::string>(count));
498 m_outputBuffer.append("\"");
500 m_outputBuffer.append(" >\n</testsuites>");
504 virtual void Finish()
506 std::size_t pos = GetCurrentGroupPosition();
507 if (std::string::npos != pos) {
513 virtual bool ParseCollectorSpecificArg(const std::string& arg)
515 return ParseCollectorFileArg(arg, m_filename);
518 virtual void CollectResult(const std::string& id,
519 const std::string& /*description*/,
520 const FailStatus::Type status = FailStatus::NONE,
521 const std::string& reason = "")
523 m_resultBuffer.erase();
524 m_resultBuffer.append("\t\t<testcase name=\"");
525 m_resultBuffer.append(EscapeSpecialCharacters(id));
526 m_resultBuffer.append("\"");
528 case TestResultsCollectorBase::FailStatus::NONE:
529 m_resultBuffer.append(" status=\"OK\"/>\n");
531 case TestResultsCollectorBase::FailStatus::FAILED:
532 m_resultBuffer.append(" status=\"FAILED\">\n");
533 PrintfErrorMessage("FAILED", EscapeSpecialCharacters(reason), true);
534 m_resultBuffer.append("\t\t</testcase>\n");
536 case TestResultsCollectorBase::FailStatus::IGNORED:
537 m_resultBuffer.append(" status=\"Ignored\">\n");
538 PrintfIgnoredMessage("Ignored", EscapeSpecialCharacters(
540 m_resultBuffer.append("\t\t</testcase>\n");
542 case TestResultsCollectorBase::FailStatus::INTERNAL:
543 m_resultBuffer.append(" status=\"FAILED\">\n");
544 PrintfErrorMessage("INTERNAL", EscapeSpecialCharacters(
546 m_resultBuffer.append("\t\t</testcase>");
549 Assert(false && "Bad status");
551 std::size_t group_pos = GetCurrentGroupPosition();
552 if (std::string::npos == group_pos) {
553 ThrowMsg(VcoreDPL::Exception, "No current group set");
556 std::size_t last_case_pos = m_outputBuffer.find(
557 "<testcase name=\"unknown\"",
559 if (std::string::npos == last_case_pos) {
560 ThrowMsg(VcoreDPL::Exception, "Could not find SegFault test case");
562 m_outputBuffer.insert(last_case_pos - 2, m_resultBuffer);
564 m_stats.AddTest(status);
566 UpdateGroupHeader(group_pos,
567 m_stats.GetTotal() + 1, // include SegFault
568 m_stats.GetFailed() + 1, // include SegFault
569 m_stats.GetIgnored());
573 std::size_t GetCurrentGroupPosition() const
575 return m_outputBuffer.rfind("<testsuite ");
578 void UpdateGroupHeader(const std::size_t groupPosition,
579 const unsigned int tests,
580 const unsigned int failures,
581 const unsigned int skipped)
583 UpdateElementAttribute(groupPosition, "tests", UIntToString(tests));
584 UpdateElementAttribute(groupPosition, "failures", UIntToString(failures));
585 UpdateElementAttribute(groupPosition, "skipped", UIntToString(skipped));
588 void UpdateElementAttribute(const std::size_t elementPosition,
589 const std::string& name,
590 const std::string& value)
592 std::string pattern = name + "=\"";
594 std::size_t start = m_outputBuffer.find(pattern, elementPosition);
595 if (std::string::npos == start) {
596 ThrowMsg(VcoreDPL::Exception,
597 "Could not find attribute " << name << " beginning");
600 std::size_t end = m_outputBuffer.find("\"", start + pattern.length());
601 if (std::string::npos == end) {
602 ThrowMsg(VcoreDPL::Exception,
603 "Could not find attribute " << name << " end");
606 m_outputBuffer.replace(start + pattern.length(),
607 end - start - pattern.length(),
611 std::string UIntToString(const unsigned int value)
613 std::stringstream result;
618 void GroupFinish(const std::size_t groupPosition)
620 std::size_t segFaultStart =
621 m_outputBuffer.find("<testcase name=\"unknown\"", groupPosition);
622 if (std::string::npos == segFaultStart) {
623 ThrowMsg(VcoreDPL::Exception,
624 "Could not find SegFault test case start position");
626 segFaultStart -= 2; // to erase tabs
628 std::string closeTag = "</testcase>";
629 std::size_t segFaultEnd = m_outputBuffer.find(closeTag, segFaultStart);
630 if (std::string::npos == segFaultEnd) {
631 ThrowMsg(VcoreDPL::Exception,
632 "Could not find SegFault test case end position");
634 segFaultEnd += closeTag.length() + 1; // to erase new line
636 m_outputBuffer.erase(segFaultStart, segFaultEnd - segFaultStart);
638 UpdateGroupHeader(groupPosition,
641 m_stats.GetIgnored());
646 int fd = fileno(m_fp.Get());
649 ThrowMsg(VcoreDPL::Exception, VcoreDPL::GetErrnoString(error));
652 if (-1 == TEMP_FAILURE_RETRY(ftruncate(fd, 0L))) {
654 ThrowMsg(VcoreDPL::Exception, VcoreDPL::GetErrnoString(error));
657 if (-1 == TEMP_FAILURE_RETRY(fseek(m_fp.Get(), 0L, SEEK_SET))) {
659 ThrowMsg(VcoreDPL::Exception, VcoreDPL::GetErrnoString(error));
662 if (m_outputBuffer.size() !=
663 fwrite(m_outputBuffer.c_str(), 1, m_outputBuffer.size(),
667 ThrowMsg(VcoreDPL::Exception, VcoreDPL::GetErrnoString(error));
670 if (-1 == TEMP_FAILURE_RETRY(fflush(m_fp.Get()))) {
672 ThrowMsg(VcoreDPL::Exception, VcoreDPL::GetErrnoString(error));
676 void PrintfErrorMessage(const char* type,
677 const std::string& message,
681 m_resultBuffer.append("\t\t\t<failure type=\"");
682 m_resultBuffer.append(EscapeSpecialCharacters(type));
683 m_resultBuffer.append("\" message=\"");
684 m_resultBuffer.append(EscapeSpecialCharacters(message));
685 m_resultBuffer.append("\"/>\n");
687 m_resultBuffer.append("\t\t\t<failure type=\"");
688 m_resultBuffer.append(EscapeSpecialCharacters(type));
689 m_resultBuffer.append("\"/>\n");
693 void PrintfIgnoredMessage(const char* type,
694 const std::string& message,
698 m_resultBuffer.append("\t\t\t<skipped type=\"");
699 m_resultBuffer.append(EscapeSpecialCharacters(type));
700 m_resultBuffer.append("\" message=\"");
701 m_resultBuffer.append(EscapeSpecialCharacters(message));
702 m_resultBuffer.append("\"/>\n");
704 m_resultBuffer.append("\t\t\t<skipped type=\"");
705 m_resultBuffer.append(EscapeSpecialCharacters(type));
706 m_resultBuffer.append("\"/>\n");
710 std::string EscapeSpecialCharacters(std::string s)
712 for (unsigned int i = 0; i < s.size();) {
716 s.insert(i, """);
722 s.insert(i, "&");
740 s.insert(i, "'");
751 std::string m_filename;
754 std::string m_outputBuffer;
755 std::string m_resultBuffer;
758 TestResultsCollectorBase* XmlCollector::Constructor()
760 return new XmlCollector();
764 public TestResultsCollectorBase
767 static TestResultsCollectorBase* Constructor();
772 virtual void Start(int count)
774 DPL_UNUSED_PARAM(count);
775 printf("GROUP;ID;RESULT;REASON\n");
778 virtual void CollectCurrentTestGroupName(const std::string& name)
780 m_currentGroup = name;
783 virtual void CollectResult(const std::string& id,
784 const std::string& /*description*/,
785 const FailStatus::Type status = FailStatus::NONE,
786 const std::string& reason = "")
788 std::string statusMsg = "";
790 case TestResultsCollectorBase::FailStatus::NONE: statusMsg = "OK";
792 case TestResultsCollectorBase::FailStatus::FAILED: statusMsg = "FAILED";
794 case TestResultsCollectorBase::FailStatus::IGNORED: statusMsg =
797 case TestResultsCollectorBase::FailStatus::INTERNAL: statusMsg =
801 Assert(false && "Bad status");
803 printf("%s;%s;%s;%s\n",
804 m_currentGroup.c_str(),
810 std::string m_currentGroup;
813 TestResultsCollectorBase* CSVCollector::Constructor()
815 return new CSVCollector();
820 public TestResultsCollectorBase
823 static TestResultsCollectorBase* Constructor();
826 TAPCollector() : m_filename(DEFAULT_TAP_FILE_NAME) {}
828 virtual bool Configure()
830 m_output.open(m_filename.c_str(), std::ios_base::trunc);
836 virtual std::string CollectorSpecificHelp() const
838 std::string retVal = "--file=<filename> - name of file for output\n"
840 retVal += DEFAULT_TAP_FILE_NAME;
845 virtual void Start(int count)
847 DPL_UNUSED_PARAM(count);
848 AssertMsg(m_output.good(), "Output file must be opened.");
849 m_output << "TAP version 13" << std::endl;
853 virtual void Finish()
855 m_output << "1.." << m_testIndex << std::endl;
856 m_output << m_collectedData.rdbuf();
860 virtual bool ParseCollectorSpecificArg(const std::string& arg)
862 return ParseCollectorFileArg(arg, m_filename);
865 virtual void CollectResult(const std::string& id,
866 const std::string& description,
867 const FailStatus::Type status = FailStatus::NONE,
868 const std::string& reason = "")
872 case TestResultsCollectorBase::FailStatus::NONE:
873 LogBasicTAP(true, id, description);
876 case TestResultsCollectorBase::FailStatus::FAILED:
877 LogBasicTAP(false, id, description);
880 case TestResultsCollectorBase::FailStatus::IGNORED:
881 LogBasicTAP(true, id, description);
882 m_collectedData << " # skip " << reason;
885 case TestResultsCollectorBase::FailStatus::INTERNAL:
886 LogBasicTAP(true, id, description);
888 m_collectedData << " ---" << std::endl;
889 m_collectedData << " message: " << reason << std::endl;
890 m_collectedData << " severity: Internal" << std::endl;
891 m_collectedData << " ..." << std::endl;
894 Assert(false && "Bad status");
898 void LogBasicTAP(bool isOK, const std::string& id,
899 const std::string& description)
902 m_collectedData << "not ";
904 m_collectedData << "ok " << m_testIndex << " [" <<
905 id << "] " << description;
910 m_collectedData << std::endl;
913 std::string m_filename;
914 std::stringstream m_collectedData;
915 std::ofstream m_output;
919 TestResultsCollectorBase* TAPCollector::Constructor()
921 return new TAPCollector();
924 void TestResultsCollectorBase::RegisterCollectorConstructor(
925 const std::string& name,
926 TestResultsCollectorBase::CollectorConstructorFunc func)
928 Assert(m_constructorsMap.find(name) == m_constructorsMap.end());
929 m_constructorsMap[name] = func;
932 TestResultsCollectorBase* TestResultsCollectorBase::Create(
933 const std::string& name)
935 ConstructorsMap::iterator found = m_constructorsMap.find(name);
936 if (found != m_constructorsMap.end()) {
937 return found->second();
943 std::vector<std::string> TestResultsCollectorBase::GetCollectorsNames()
945 std::vector<std::string> list;
946 FOREACH(it, m_constructorsMap)
948 list.push_back(it->first);
953 TestResultsCollectorBase::ConstructorsMap TestResultsCollectorBase::
957 static int RegisterCollectorConstructors();
958 static const int RegisterHelperVariable = RegisterCollectorConstructors();
959 int RegisterCollectorConstructors()
961 (void)RegisterHelperVariable;
963 TestResultsCollectorBase::RegisterCollectorConstructor(
965 &ConsoleCollector::Constructor);
966 TestResultsCollectorBase::RegisterCollectorConstructor(
968 &HtmlCollector::Constructor);
969 TestResultsCollectorBase::RegisterCollectorConstructor(
971 &CSVCollector::Constructor);
972 TestResultsCollectorBase::RegisterCollectorConstructor(
974 &TAPCollector::Constructor);
975 TestResultsCollectorBase::RegisterCollectorConstructor(
977 &XmlCollector::Constructor);
984 #undef GREEN_RESULT_OK