2 * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 * @file test_results_collector.h
18 * @author Lukasz Wrzosek (l.wrzosek@samsung.com)
20 * @brief Implementation file some concrete TestResulstsCollector
23 #include <dpl/test/test_results_collector.h>
24 #include <dpl/colors.h>
25 #include <dpl/assert.h>
26 #include <dpl/foreach.h>
27 #include <dpl/scoped_fclose.h>
28 #include <dpl/exception.h>
29 #include <dpl/errno_string.h>
38 #define GREEN_RESULT_OK "[%s%s%s]\n", BOLD_GREEN_BEGIN, " OK ", BOLD_GREEN_END
47 const char *DEFAULT_HTML_FILE_NAME = "index.html";
48 const char *DEFAULT_TAP_FILE_NAME = "results.tap";
49 const char *DEFAULT_XML_FILE_NAME = "results.xml";
51 bool ParseCollectorFileArg(const std::string &arg, std::string &filename)
53 const std::string argname = "--file=";
54 if (0 == arg.find(argname)) {
55 filename = arg.substr(argname.size());
72 void AddTest(TestResultsCollectorBase::FailStatus::Type type)
76 case TestResultsCollectorBase::FailStatus::INTERNAL:
77 case TestResultsCollectorBase::FailStatus::FAILED: ++m_failed; break;
78 case TestResultsCollectorBase::FailStatus::IGNORED: ++m_ignored; break;
79 case TestResultsCollectorBase::FailStatus::NONE: ++m_passed; break;
81 Assert(false && "Bad FailStatus");
85 std::size_t GetTotal() const { return m_count; }
86 std::size_t GetPassed() const { return m_passed; }
87 std::size_t GetSuccesed() const { return m_passed; }
88 std::size_t GetFailed() const { return m_failed; }
89 std::size_t GetIgnored() const { return m_ignored; }
90 float GetPassedOrIgnoredPercend() const
92 float passIgnoredPercent =
93 100.0f * (static_cast<float>(m_passed)
94 + static_cast<float>(m_ignored))
95 / static_cast<float>(m_count);
96 return passIgnoredPercent;
100 std::size_t m_failed;
101 std::size_t m_ignored;
102 std::size_t m_passed;
106 class ConsoleCollector
107 : public TestResultsCollectorBase
110 static TestResultsCollectorBase* Constructor();
113 ConsoleCollector() {}
115 virtual void CollectCurrentTestGroupName(const std::string& name)
117 printf("Starting group %s\n", name.c_str());
118 m_currentGroup = name;
121 virtual void Finish()
123 using namespace DPL::Colors::Text;
126 FOREACH(group, m_groupsStats) {
127 PrintStats(group->first, group->second);
129 PrintStats("All tests together", m_stats);
132 virtual void CollectResult(const std::string& id,
133 const std::string& /*description*/,
134 const FailStatus::Type status = FailStatus::NONE,
135 const std::string& reason = "")
137 using namespace DPL::Colors::Text;
138 std::string tmp = "'" + id + "' ...";
140 printf("Running test case %-60s", tmp.c_str());
142 case TestResultsCollectorBase::FailStatus::NONE:
143 printf(GREEN_RESULT_OK); break;
144 case TestResultsCollectorBase::FailStatus::FAILED:
145 PrintfErrorMessage( " FAILED ", reason, true); break;
146 case TestResultsCollectorBase::FailStatus::IGNORED:
147 PrintfIgnoredMessage("Ignored ", reason, true); break;
148 case TestResultsCollectorBase::FailStatus::INTERNAL:
149 PrintfErrorMessage( "INTERNAL", reason, true); break;
151 Assert(false && "Bad status");
153 m_stats.AddTest(status);
154 m_groupsStats[m_currentGroup].AddTest(status);
157 void PrintfErrorMessage(const char* type,
158 const std::string& message,
161 using namespace DPL::Colors::Text;
163 printf("[%s%s%s] %s%s%s\n",
178 void PrintfIgnoredMessage(const char* type,
179 const std::string& message,
182 using namespace DPL::Colors::Text;
184 printf("[%s%s%s] %s%s%s\n",
199 void PrintStats(const std::string& title, const Statistic& stats)
201 using namespace DPL::Colors::Text;
202 printf("\n%sResults [%s]: %s\n", BOLD_GREEN_BEGIN, title.c_str(), BOLD_GREEN_END);
203 printf("%s%s%3d%s\n", CYAN_BEGIN, "Total tests: ", stats.GetTotal(), CYAN_END);
204 printf(" %s%s%3d%s\n", CYAN_BEGIN, "Succeeded: ", stats.GetPassed(), CYAN_END);
205 printf(" %s%s%3d%s\n", CYAN_BEGIN, "Failed: ", stats.GetFailed(), CYAN_END);
206 printf(" %s%s%3d%s\n", CYAN_BEGIN, "Ignored: ", stats.GetIgnored(), CYAN_END);
210 std::map<std::string, Statistic> m_groupsStats;
211 std::string m_currentGroup;
215 TestResultsCollectorBase* ConsoleCollector::Constructor()
217 return new ConsoleCollector();
222 : public TestResultsCollectorBase
225 static TestResultsCollectorBase* Constructor();
228 HtmlCollector() : m_filename(DEFAULT_HTML_FILE_NAME) {}
230 virtual void CollectCurrentTestGroupName(const std::string& name)
232 fprintf(m_fp.Get(),"<b>Starting group %s", name.c_str());
233 m_currentGroup = name;
236 virtual bool Configure()
238 m_fp.Reset(fopen (m_filename.c_str(), "w"));
240 LogPedantic("Could not open file " << m_filename << " for writing");
245 virtual std::string CollectorSpecificHelp() const
247 return "--file=<filename> - name of file for output\n"
248 " default - index.html\n";
253 Assert(!!m_fp && "File handle must not be null");
255 "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0"
256 "Transitional//EN\" "
257 "\"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\""
260 "<html xmlns=\"http://www.w3.org/1999/xhtml\" "
261 "lang=\"en\" dir=\"ltr\">\n");
262 fprintf(m_fp.Get(), "<body style=\"background-color: black;\">\n");
263 fprintf(m_fp.Get(), "<pre>\n");
264 fprintf(m_fp.Get(), "<font color=\"white\">\n");
267 virtual void Finish()
269 using namespace DPL::Colors::Html;
271 FOREACH(group, m_groupsStats) {
272 PrintStats(group->first, group->second);
274 PrintStats("All tests together", m_stats);
275 fprintf(m_fp.Get(), "</font>\n");
276 fprintf(m_fp.Get(), "</pre>\n");
277 fprintf(m_fp.Get(), "</body>\n");
278 fprintf(m_fp.Get(), "</html>\n");
281 virtual bool ParseCollectorSpecificArg(const std::string& arg)
283 return ParseCollectorFileArg(arg, m_filename);
286 virtual void CollectResult(const std::string& id,
287 const std::string& /*description*/,
288 const FailStatus::Type status = FailStatus::NONE,
289 const std::string& reason = "")
291 using namespace DPL::Colors::Html;
292 std::string tmp = "'" + id + "' ...";
294 fprintf(m_fp.Get(), "Running test case %-100s", tmp.c_str());
296 case TestResultsCollectorBase::FailStatus::NONE:
297 fprintf(m_fp.Get(), GREEN_RESULT_OK); break;
298 case TestResultsCollectorBase::FailStatus::FAILED:
299 PrintfErrorMessage( " FAILED ", reason, true); break;
300 case TestResultsCollectorBase::FailStatus::IGNORED:
301 PrintfIgnoredMessage("Ignored ", reason, true); break;
302 case TestResultsCollectorBase::FailStatus::INTERNAL:
303 PrintfErrorMessage( "INTERNAL", reason, true); break;
305 Assert(false && "Bad status");
307 m_groupsStats[m_currentGroup].AddTest(status);
308 m_stats.AddTest(status);
311 void PrintfErrorMessage(const char* type,
312 const std::string& message,
315 using namespace DPL::Colors::Html;
334 void PrintfIgnoredMessage(const char* type,
335 const std::string& message,
338 using namespace DPL::Colors::Html;
358 void PrintStats(const std::string& name, const Statistic& stats)
360 using namespace DPL::Colors::Html;
361 fprintf(m_fp.Get(), "\n%sResults [%s]:%s\n", BOLD_GREEN_BEGIN, name.c_str(), BOLD_GREEN_END);
362 fprintf(m_fp.Get(), "%s%s%3d%s\n", CYAN_BEGIN, "Total tests: ", stats.GetTotal(), CYAN_END);
363 fprintf(m_fp.Get(), " %s%s%3d%s\n", CYAN_BEGIN, "Succeeded: ", stats.GetPassed(), CYAN_END);
364 fprintf(m_fp.Get(), " %s%s%3d%s\n", CYAN_BEGIN, "Failed: ", stats.GetFailed(), CYAN_END);
365 fprintf(m_fp.Get(), " %s%s%3d%s\n", CYAN_BEGIN, "Ignored: ", stats.GetIgnored(), CYAN_END);
368 std::string m_filename;
371 std::string m_currentGroup;
372 std::map<std::string, Statistic> m_groupsStats;
375 TestResultsCollectorBase* HtmlCollector::Constructor()
377 return new HtmlCollector();
382 : public TestResultsCollectorBase
385 static TestResultsCollectorBase* Constructor();
388 XmlCollector() : m_filename(DEFAULT_XML_FILE_NAME) {}
390 virtual void CollectCurrentTestGroupName(const std::string& name)
392 std::size_t pos = GetCurrentGroupPosition();
393 if (std::string::npos != pos)
397 m_stats = Statistic();
400 pos = m_outputBuffer.find("</testsuites>");
401 if (std::string::npos == pos)
403 ThrowMsg(DPL::Exception, "Could not find test suites closing tag");
405 GroupStart(pos, name);
408 void GroupStart(const std::size_t pos, const std::string& name)
410 std::stringstream groupHeader;
411 groupHeader << "\n\t<testsuite";
412 groupHeader << " name=\"" << EscapeSpecialCharacters(name) << "\"";
413 groupHeader << R"( tests="1")"; // include SegFault
414 groupHeader << R"( failures="1")"; // include SegFault
415 groupHeader << R"( skipped="0")";
418 groupHeader << "\n\t\t<testcase name=\"unknown\" status=\"FAILED\">";
419 groupHeader << "\n\t\t\t<failure type=\"FAILED\" message=\"segmentation fault\"/>";
420 groupHeader << "\n\t\t</testcase>";
422 groupHeader << "\n\t</testsuite>";
424 m_outputBuffer.insert(pos - 1, groupHeader.str());
427 virtual bool Configure()
429 m_fp.Reset(fopen (m_filename.c_str(), "w"));
431 LogPedantic("Could not open file " << m_filename << " for writing");
437 virtual std::string CollectorSpecificHelp() const
439 return "--file=<filename> - name of file for output\n"
440 " default - results.xml\n";
445 Assert(!!m_fp && "File handle must not be null");
446 m_outputBuffer.append("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n");
447 m_outputBuffer.append("<testsuites>\n</testsuites>");
451 virtual void Finish()
453 std::size_t pos = GetCurrentGroupPosition();
454 if (std::string::npos != pos)
461 virtual bool ParseCollectorSpecificArg(const std::string& arg)
463 return ParseCollectorFileArg(arg, m_filename);
466 virtual void CollectResult(const std::string& id,
467 const std::string& /*description*/,
468 const FailStatus::Type status = FailStatus::NONE,
469 const std::string& reason = "")
471 m_resultBuffer.erase();
472 m_resultBuffer.append("\t\t<testcase name=\"");
473 m_resultBuffer.append(EscapeSpecialCharacters(id));
474 m_resultBuffer.append("\"");
476 case TestResultsCollectorBase::FailStatus::NONE:
477 m_resultBuffer.append(" status=\"OK\"/>\n");
479 case TestResultsCollectorBase::FailStatus::FAILED:
480 m_resultBuffer.append(" status=\"FAILED\">\n");
481 PrintfErrorMessage("FAILED", EscapeSpecialCharacters(reason), true);
482 m_resultBuffer.append("\t\t</testcase>\n");
484 case TestResultsCollectorBase::FailStatus::IGNORED:
485 m_resultBuffer.append(" status=\"Ignored\">\n");
486 PrintfIgnoredMessage("Ignored", EscapeSpecialCharacters(reason), true);
487 m_resultBuffer.append("\t\t</testcase>\n");
489 case TestResultsCollectorBase::FailStatus::INTERNAL:
490 m_resultBuffer.append(" status=\"FAILED\">\n");
491 PrintfErrorMessage("INTERNAL", EscapeSpecialCharacters(reason), true);
492 m_resultBuffer.append("\t\t</testcase>");
495 Assert(false && "Bad status");
497 std::size_t group_pos = GetCurrentGroupPosition();
498 if (std::string::npos == group_pos)
500 ThrowMsg(DPL::Exception, "No current group set");
503 std::size_t last_case_pos = m_outputBuffer.find("<testcase name=\"unknown\"", group_pos);
504 if (std::string::npos == last_case_pos)
506 ThrowMsg(DPL::Exception, "Could not find SegFault test case");
508 m_outputBuffer.insert(last_case_pos - 2, m_resultBuffer);
510 m_stats.AddTest(status);
512 UpdateGroupHeader(group_pos,
513 m_stats.GetTotal() + 1, // include SegFault
514 m_stats.GetFailed() + 1, // include SegFault
515 m_stats.GetIgnored());
519 std::size_t GetCurrentGroupPosition() const
521 return m_outputBuffer.rfind("<testsuite ");
524 void UpdateGroupHeader(const std::size_t groupPosition,
525 const unsigned int tests,
526 const unsigned int failures,
527 const unsigned int skipped)
529 UpdateElementAttribute(groupPosition, "tests", UIntToString(tests));
530 UpdateElementAttribute(groupPosition, "failures", UIntToString(failures));
531 UpdateElementAttribute(groupPosition, "skipped", UIntToString(skipped));
534 void UpdateElementAttribute(const std::size_t elementPosition,
535 const std::string& name,
536 const std::string& value)
538 std::string pattern = name + "=\"";
540 std::size_t start = m_outputBuffer.find(pattern, elementPosition);
541 if (std::string::npos == start)
543 ThrowMsg(DPL::Exception, "Could not find attribute " << name << " beginning");
546 std::size_t end = m_outputBuffer.find("\"", start + pattern.length());
547 if (std::string::npos == end)
549 ThrowMsg(DPL::Exception, "Could not find attribute " << name << " end");
552 m_outputBuffer.replace(start + pattern.length(),
553 end - start - pattern.length(),
557 std::string UIntToString(const unsigned int value)
559 std::stringstream result;
564 void GroupFinish(const std::size_t groupPosition)
566 std::size_t segFaultStart =
567 m_outputBuffer.find("<testcase name=\"unknown\"", groupPosition);
568 if (std::string::npos == segFaultStart)
570 ThrowMsg(DPL::Exception, "Could not find SegFault test case start position");
572 segFaultStart -= 2; // to erase tabs
574 std::string closeTag = "</testcase>";
575 std::size_t segFaultEnd = m_outputBuffer.find(closeTag, segFaultStart);
576 if (std::string::npos == segFaultEnd)
578 ThrowMsg(DPL::Exception, "Could not find SegFault test case end position");
580 segFaultEnd += closeTag.length() + 1; // to erase new line
582 m_outputBuffer.erase(segFaultStart, segFaultEnd - segFaultStart);
584 UpdateGroupHeader(groupPosition,
587 m_stats.GetIgnored());
592 int fd = fileno(m_fp.Get());
596 ThrowMsg(DPL::Exception, DPL::GetErrnoString(error));
599 if (-1 == TEMP_FAILURE_RETRY(ftruncate(fd, 0L)))
602 ThrowMsg(DPL::Exception, DPL::GetErrnoString(error));
605 if (-1 == TEMP_FAILURE_RETRY(fseek(m_fp.Get(), 0L, SEEK_SET)))
608 ThrowMsg(DPL::Exception, DPL::GetErrnoString(error));
611 if (0 > fprintf(m_fp.Get(), m_outputBuffer.c_str()))
614 ThrowMsg(DPL::Exception, DPL::GetErrnoString(error));
617 if (-1 == TEMP_FAILURE_RETRY(fflush(m_fp.Get())))
620 ThrowMsg(DPL::Exception, DPL::GetErrnoString(error));
624 void PrintfErrorMessage(const char* type,
625 const std::string& message,
629 m_resultBuffer.append("\t\t\t<failure type=\"");
630 m_resultBuffer.append(EscapeSpecialCharacters(type));
631 m_resultBuffer.append("\" message=\"");
632 m_resultBuffer.append(EscapeSpecialCharacters(message));
633 m_resultBuffer.append("\"/>\n");
637 m_resultBuffer.append("\t\t\t<failure type=\"");
638 m_resultBuffer.append(EscapeSpecialCharacters(type));
639 m_resultBuffer.append("\"/>\n");
643 void PrintfIgnoredMessage(const char* type,
644 const std::string& message,
648 m_resultBuffer.append("\t\t\t<skipped type=\"");
649 m_resultBuffer.append(EscapeSpecialCharacters(type));
650 m_resultBuffer.append("\" message=\"");
651 m_resultBuffer.append(EscapeSpecialCharacters(message));
652 m_resultBuffer.append("\"/>\n");
655 m_resultBuffer.append("\t\t\t<skipped type=\"");
656 m_resultBuffer.append(EscapeSpecialCharacters(type));
657 m_resultBuffer.append("\"/>\n");
661 std::string EscapeSpecialCharacters(std::string s)
663 for (unsigned int i = 0; i < s.size();) {
667 s.insert(i, """);
673 s.insert(i, "&");
691 s.insert(i, "'");
702 std::string m_filename;
705 std::string m_outputBuffer;
706 std::string m_resultBuffer;
709 TestResultsCollectorBase* XmlCollector::Constructor()
711 return new XmlCollector();
717 : public TestResultsCollectorBase
720 static TestResultsCollectorBase* Constructor();
725 virtual void Start() {
726 printf("GROUP;ID;RESULT;REASON\n");
729 virtual void CollectCurrentTestGroupName(const std::string& name)
731 m_currentGroup = name;
734 virtual void CollectResult(const std::string& id,
735 const std::string& /*description*/,
736 const FailStatus::Type status = FailStatus::NONE,
737 const std::string& reason = "")
739 std::string statusMsg = "";
741 case TestResultsCollectorBase::FailStatus::NONE: statusMsg = "OK"; break;
742 case TestResultsCollectorBase::FailStatus::FAILED: statusMsg = "FAILED"; break;
743 case TestResultsCollectorBase::FailStatus::IGNORED: statusMsg = "IGNORED"; break;
744 case TestResultsCollectorBase::FailStatus::INTERNAL: statusMsg = "FAILED"; break;
746 Assert(false && "Bad status");
748 printf("%s;%s;%s;%s\n",
749 m_currentGroup.c_str(),
755 std::string m_currentGroup;
759 TestResultsCollectorBase* CSVCollector::Constructor()
761 return new CSVCollector();
767 : public TestResultsCollectorBase
770 static TestResultsCollectorBase* Constructor();
773 TAPCollector() : m_filename(DEFAULT_TAP_FILE_NAME) {}
775 virtual bool Configure()
777 m_output.open(m_filename.c_str(), std::ios_base::trunc);
778 if (m_output.fail()) {
779 LogError("Can't open output file: " << m_filename);
784 virtual std::string CollectorSpecificHelp() const
786 std::string retVal = "--file=<filename> - name of file for output\n"
788 retVal += DEFAULT_TAP_FILE_NAME;
795 Assert(m_output.good() && "Output file must be opened.");
796 m_output << "TAP version 13" << std::endl;
800 virtual void Finish()
802 m_output << "1.." << m_testIndex << std::endl;
803 m_output << m_collectedData.rdbuf();
807 virtual bool ParseCollectorSpecificArg(const std::string& arg)
809 return ParseCollectorFileArg(arg, m_filename);
812 virtual void CollectResult(const std::string& id,
813 const std::string& description,
814 const FailStatus::Type status = FailStatus::NONE,
815 const std::string& reason = "")
819 case TestResultsCollectorBase::FailStatus::NONE:
820 LogBasicTAP(true, id, description);
823 case TestResultsCollectorBase::FailStatus::FAILED:
824 LogBasicTAP(false, id, description);
827 case TestResultsCollectorBase::FailStatus::IGNORED:
828 LogBasicTAP(true, id, description);
829 m_collectedData << " # skip " << reason;
832 case TestResultsCollectorBase::FailStatus::INTERNAL:
833 LogBasicTAP(true, id, description);
835 m_collectedData << " ---" << std::endl;
836 m_collectedData << " message: " << reason << std::endl;
837 m_collectedData << " severity: Internal" << std::endl;
838 m_collectedData << " ..." << std::endl;
841 Assert(false && "Bad status");
845 void LogBasicTAP(bool isOK, const std::string& id,
846 const std::string& description)
849 m_collectedData << "not ";
851 m_collectedData << "ok " << m_testIndex << " [" <<
852 id << "] " << description;
857 m_collectedData << std::endl;
861 std::string m_filename;
862 std::stringstream m_collectedData;
863 std::ofstream m_output;
868 TestResultsCollectorBase* TAPCollector::Constructor()
870 return new TAPCollector();
874 void TestResultsCollectorBase::RegisterCollectorConstructor(
875 const std::string& name,
876 TestResultsCollectorBase::CollectorConstructorFunc func)
878 Assert(m_constructorsMap.find(name) == m_constructorsMap.end());
879 m_constructorsMap[name] = func;
882 TestResultsCollectorBase* TestResultsCollectorBase::Create(
883 const std::string& name)
885 ConstructorsMap::iterator found = m_constructorsMap.find(name);
886 if (found != m_constructorsMap.end())
887 return found->second();
892 std::vector<std::string> TestResultsCollectorBase::GetCollectorsNames()
894 std::vector<std::string> list;
895 FOREACH(it, m_constructorsMap)
897 list.push_back(it->first);
902 TestResultsCollectorBase::ConstructorsMap TestResultsCollectorBase::m_constructorsMap;
906 static int RegisterCollectorConstructors();
907 static const int RegisterHelperVariable = RegisterCollectorConstructors();
908 int RegisterCollectorConstructors()
910 (void)RegisterHelperVariable;
912 TestResultsCollectorBase::RegisterCollectorConstructor(
914 &ConsoleCollector::Constructor);
915 TestResultsCollectorBase::RegisterCollectorConstructor(
917 &HtmlCollector::Constructor);
918 TestResultsCollectorBase::RegisterCollectorConstructor(
920 &CSVCollector::Constructor);
921 TestResultsCollectorBase::RegisterCollectorConstructor(
923 &TAPCollector::Constructor);
924 TestResultsCollectorBase::RegisterCollectorConstructor(
926 &XmlCollector::Constructor);
935 #undef GREEN_RESULT_OK