Define a base TestException class
[platform/core/test/security-tests.git] / src / framework / src / test_runner.cpp
1 /*
2  * Copyright (c) 2014-2015 Samsung Electronics Co., Ltd All Rights Reserved
3  *
4  *    Licensed under the Apache License, Version 2.0 (the "License");
5  *    you may not use this file except in compliance with the License.
6  *    You may obtain a copy of the License at
7  *
8  *        http://www.apache.org/licenses/LICENSE-2.0
9  *
10  *    Unless required by applicable law or agreed to in writing, software
11  *    distributed under the License is distributed on an "AS IS" BASIS,
12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  *    See the License for the specific language governing permissions and
14  *    limitations under the License.
15  */
16 /**
17  * @file        test_runner.cpp
18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
20  * @version     1.0
21  * @brief       This file is the implementation file of test runner
22  */
23 #include <stddef.h>
24 #include <dpl/test/test_failed.h>
25 #include <dpl/test/test_ignored.h>
26 #include <dpl/test/test_runner.h>
27 #include <dpl/test/test_results_collector.h>
28 #include <dpl/exception.h>
29 #include <dpl/scoped_free.h>
30 #include <dpl/log/log.h>
31 #include <dpl/colors.h>
32 #include <pcrecpp.h>
33 #include <algorithm>
34 #include <cstdio>
35 #include <memory.h>
36
37 #include <libxml/xpath.h>
38 #include <libxml/xpathInternals.h>
39 #include <libxml/parser.h>
40 #include <libxml/tree.h>
41
42 #include <dpl/singleton_impl.h>
43 IMPLEMENT_SINGLETON(DPL::Test::TestRunner)
44
45 namespace {
46
47 std::string getXMLNode(xmlNodePtr node)
48 {
49     std::string ret;
50     xmlChar * value = xmlNodeGetContent(node);
51     ret = std::string(reinterpret_cast<char*>(value));
52     xmlFree(value);
53     return ret;
54 }
55
56 }
57
58
59 namespace DPL {
60 namespace Test {
61
62 void TestRunner::RegisterTest(const char *testName, TestCase proc)
63 {
64     m_testGroups[m_currentGroup].push_back(TestCaseStruct(testName, proc));
65 }
66
67 void TestRunner::InitGroup(const char* name)
68 {
69     m_currentGroup = name;
70 }
71
72 void TestRunner::normalizeXMLTag(std::string& str, const std::string& testcase)
73 {
74     //Add testcase if missing
75     std::string::size_type pos = str.find(testcase);
76     if(pos != 0)
77     {
78         str = testcase + "_" + str;
79     }
80
81     //dpl test runner cannot have '-' character in name so it have to be replaced
82     // for TCT case to make comparision works
83     std::replace(str.begin(), str.end(), '-', '_');
84 }
85
86 bool TestRunner::filterGroupsByXmls(const std::vector<std::string> & files)
87 {
88     DECLARE_EXCEPTION_TYPE(DPL::Exception, XMLError)
89
90     const std::string idPath = "/test_definition/suite/set/testcase/@id";
91
92     bool success = true;
93     std::map<std::string, bool> casesMap;
94
95     std::string testsuite;
96     if(!m_testGroups.empty())
97     {
98         for(TestCaseGroupMap::const_iterator cit = m_testGroups.begin(); cit != m_testGroups.end(); ++cit)
99         {
100             if(!cit->second.empty())
101             {
102                 for(TestCaseStructList::const_iterator cj = cit->second.begin(); cj != cit->second.end(); ++cj)
103                 {
104                     std::string name = cj->name;
105                     std::string::size_type st = name.find('_');
106                     if(st != std::string::npos)
107                     {
108                         name = name.substr(0, st);
109                         testsuite = name;
110                         break;
111                     }
112                 }
113                 if(!testsuite.empty()) break;
114             }
115         }
116     }
117
118     xmlInitParser();
119     LIBXML_TEST_VERSION
120     xmlXPathInit();
121
122     Try
123     {
124         for (const std::string &file : files)
125         {
126             xmlDocPtr doc;
127             xmlXPathContextPtr xpathCtx;
128
129             doc = xmlReadFile(file.c_str(), nullptr, 0);
130             if (doc == nullptr) {
131                 ThrowMsg(XMLError, "File Problem");
132             } else {
133                 //context
134                 xpathCtx = xmlXPathNewContext(doc);
135                 if (xpathCtx == nullptr) {
136                     ThrowMsg(XMLError,
137                              "Error: unable to create new XPath context\n");
138                 }
139                 xpathCtx->node = xmlDocGetRootElement(doc);
140             }
141
142             std::string result;
143             xmlXPathObjectPtr xpathObject;
144             //get requested node's values
145             xpathObject = xmlXPathEvalExpression(BAD_CAST idPath.c_str(), xpathCtx);
146             if (xpathObject == nullptr)
147             {
148                 ThrowMsg(XMLError, "XPath evaluation failure: " << idPath);
149             }
150             xmlNodeSetPtr nodes = xpathObject->nodesetval;
151             unsigned size = (nodes) ? nodes->nodeNr : 0;
152             LogDebug("Found " << size << " nodes matching xpath");
153             for(unsigned i = 0; i < size; ++i)
154             {
155                 LogPedantic("Type: " << nodes->nodeTab[i]->type);
156                 if (nodes->nodeTab[i]->type == XML_ATTRIBUTE_NODE) {
157                     xmlNodePtr curNode = nodes->nodeTab[i];
158                     result = getXMLNode(curNode);
159                     LogPedantic("Result: " << result);
160                     normalizeXMLTag(result, testsuite);
161                     casesMap.insert(make_pair(result, false));
162                 }
163             }
164             //Cleanup of XPath data
165             xmlXPathFreeObject(xpathObject);
166             xmlXPathFreeContext(xpathCtx);
167             xmlFreeDoc(doc);
168         }
169     }
170     Catch(XMLError)
171     {
172         LogError("Libxml error: " << _rethrown_exception.DumpToString());
173         success = false;
174     }
175     xmlCleanupParser();
176
177     if(!filterByXML(casesMap))
178     {
179         success = false;
180     }
181
182     return success;
183 }
184
185 bool TestRunner::filterByXML(std::map<std::string, bool> & casesMap)
186 {
187     for (auto &group : m_testGroups) {
188         TestCaseStructList newList;
189         for (auto &tc : group.second)
190         {
191             if (casesMap.find(tc.name) != casesMap.end()) {
192                 casesMap[tc.name] = true;
193                 newList.push_back(tc);
194             }
195         }
196         group.second = newList;
197     }
198     for (auto &cs : casesMap)
199     {
200         if(cs.second == false)
201         {
202             LogError("Cannot find testcase from XML file: " << cs.first);
203             return false;
204         }
205     }
206     return true;
207 }
208
209 TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase)
210 {
211     setCurrentTestCase(&(const_cast<TestCaseStruct &>(testCase)));
212     try {
213         testCase.proc();
214     } catch (const TestFailed &e) {
215         // Simple test failure
216         CollectResult(testCase.name,
217                       TestResultsCollectorBase::FailStatus::FAILED,
218                       getConcatedFailReason(e.GetMessage()));
219
220         setCurrentTestCase(nullptr);
221         return FAILED;
222     } catch (const TestIgnored &e) {
223         if (m_runIgnored) {
224             // Simple test have to be implemented
225             CollectResult(testCase.name,
226                           TestResultsCollectorBase::FailStatus::IGNORED,
227                           e.GetMessage());
228         }
229
230         setCurrentTestCase(nullptr);
231         return IGNORED;
232     } catch (const std::exception &) {
233         // std exception failure
234         CollectResult(testCase.name,
235                       TestResultsCollectorBase::FailStatus::FAILED,
236                       "std exception");
237
238         setCurrentTestCase(nullptr);
239         return FAILED;
240     } catch (...) {
241         // Unknown exception failure
242         CollectResult(testCase.name,
243                       TestResultsCollectorBase::FailStatus::FAILED,
244                       "unknown exception");
245
246         setCurrentTestCase(nullptr);
247         return FAILED;
248     }
249
250     CollectResult(testCase.name,
251                   TestResultsCollectorBase::FailStatus::NONE,
252                   "",
253                   testCase.m_isPerformanceTest,
254                   testCase.m_performanceTestDurationTime,
255                   testCase.m_performanceMaxTime);
256     setCurrentTestCase(nullptr);
257
258     // Everything OK
259     return PASS;
260 }
261
262 void TestRunner::RunTests()
263 {
264     using namespace DPL::Colors::Text;
265
266     Banner();
267     for (auto &collector : m_collectors) {
268         collector.second->Start();
269     }
270
271     unsigned count = 0;
272     for (auto &group : m_testGroups) {
273         count += group.second.size();
274     }
275     fprintf(stderr, "%sFound %d testcases...%s\n", GREEN_BEGIN, count, GREEN_END);
276     fprintf(stderr, "%s%s%s\n", GREEN_BEGIN, "Running tests...", GREEN_END);
277     for (auto &group : m_testGroups) {
278         TestCaseStructList list = group.second;
279         if (!list.empty()) {
280             for (auto &collector : m_collectors) {
281                 collector.second->CollectCurrentTestGroupName(group.first);
282             }
283             list.sort();
284
285             for (TestCaseStructList::const_iterator iterator = list.begin();
286                  iterator != list.end();
287                  ++iterator)
288             {
289                 TestCaseStruct test = *iterator;
290                 if (m_startTestId == test.name) {
291                     m_startTestId = "";
292                 }
293
294                 if (m_startTestId.empty()) {
295                     RunTestCase(test);
296                 }
297                 if (m_terminate == true) {
298                     // Terminate quietly without any logs
299                     return;
300                 }
301             }
302         }
303     }
304
305     std::for_each(m_collectors.begin(),
306                   m_collectors.end(),
307                   [] (const TestResultsCollectors::value_type & collector)
308                   {
309                       collector.second->Finish();
310                   });
311
312     // Finished
313     fprintf(stderr, "%s%s%s\n\n", GREEN_BEGIN, "Finished", GREEN_END);
314 }
315
316 TestRunner::TestCaseStruct *TestRunner::getCurrentTestCase()
317 {
318     return m_currentTestCase;
319 }
320
321 void TestRunner::setCurrentTestCase(TestCaseStruct* testCase)
322 {
323     m_currentTestCase = testCase;
324 }
325
326 void TestRunner::beginPerformanceTestTime(std::chrono::system_clock::duration maxTimeInMicroseconds)
327 {
328     TestCaseStruct* testCase = getCurrentTestCase();
329     if (!testCase)
330         return;
331
332     testCase->m_isPerformanceTest = true;
333     testCase->m_performanceMaxTime = maxTimeInMicroseconds;
334     testCase->m_performanceTestStartTime = std::chrono::system_clock::now();
335
336     // Set result to 0 microseconds. Display 0ms result when end macro is missing.
337     testCase->m_performanceTestDurationTime = std::chrono::microseconds::zero();
338 }
339
340 void TestRunner::endPerformanceTestTime()
341 {
342     TestCaseStruct* testCase = getCurrentTestCase();
343     if (!testCase)
344         return;
345
346     testCase->m_performanceTestDurationTime = std::chrono::system_clock::now() -
347             testCase->m_performanceTestStartTime;
348 }
349
350 void TestRunner::getCurrentTestCasePerformanceResult(bool& isPerformanceTest,
351                                                      std::chrono::system_clock::duration& result,
352                                                      std::chrono::system_clock::duration& resultMax)
353 {
354     TestCaseStruct* testCase = getCurrentTestCase();
355     if (!testCase || !(testCase->m_isPerformanceTest)){
356         isPerformanceTest = false;
357         return;
358     }
359
360     isPerformanceTest = testCase->m_isPerformanceTest;
361     result = testCase->m_performanceTestDurationTime;
362     resultMax = testCase->m_performanceMaxTime;
363 }
364
365 void TestRunner::setCurrentTestCasePerformanceResult(bool isPerformanceTest,
366                                                      std::chrono::system_clock::duration result,
367                                                      std::chrono::system_clock::duration resultMax)
368 {
369     TestCaseStruct* testCase = getCurrentTestCase();
370     if (!testCase)
371         return;
372
373     testCase->m_isPerformanceTest = isPerformanceTest;
374     testCase->m_performanceTestDurationTime = result;
375     testCase->m_performanceMaxTime = resultMax;
376 }
377
378 void TestRunner::addFailReason(const std::string &reason)
379 {
380     m_failReason.push(reason);
381 }
382
383 std::string TestRunner::getConcatedFailReason(const std::string &reason)
384 {
385     std::string ret;
386     while (!m_failReason.empty())
387     {
388         ret += m_failReason.front();
389         m_failReason.pop();
390     }
391     return reason + ret;
392 }
393
394 void TestRunner::CollectResult(
395     const std::string& id,
396     const TestResultsCollectorBase::FailStatus status,
397     const std::string& reason,
398     const bool& isPerformanceTest,
399     const std::chrono::system_clock::duration& performanceTestDurationTime,
400     const std::chrono::system_clock::duration& performanceMaxTime)
401 {
402     std::for_each(m_collectors.begin(),
403                   m_collectors.end(),
404                   [&](const TestResultsCollectors::value_type & collector)
405                   {
406                       collector.second->CollectResult(id,
407                                                       status,
408                                                       reason,
409                                                       isPerformanceTest,
410                                                       performanceTestDurationTime,
411                                                       performanceMaxTime);
412                   });
413 }
414
415 void TestRunner::Banner()
416 {
417     using namespace DPL::Colors::Text;
418     fprintf(stderr,
419             "%s%s%s\n",
420             BOLD_GREEN_BEGIN,
421             "DPL tests runner",
422             BOLD_GREEN_END);
423     fprintf(stderr,
424             "%s%s%s%s\n\n",
425             GREEN_BEGIN,
426             "Build: ",
427             __TIMESTAMP__,
428             GREEN_END);
429 }
430
431 void TestRunner::InvalidArgs(const std::string& message)
432 {
433     using namespace DPL::Colors::Text;
434     fprintf(stderr,
435             "%s%s%s\n",
436             BOLD_RED_BEGIN,
437             message.c_str(),
438             BOLD_RED_END);
439 }
440
441 void TestRunner::Usage()
442 {
443     fprintf(stderr, "Usage: runner [options]\n\n");
444     fprintf(stderr, "Output type:\n");
445     fprintf(stderr, "  --output=<output type> --output=<output type> ...\n");
446     fprintf(stderr, "\n  possible output types:\n");
447     for (std::string &type : TestResultsCollectorBase::GetCollectorsNames()) {
448         fprintf(stderr, "    --output=%s\n", type.c_str());
449     }
450     fprintf(stderr, "\n  example:\n");
451     fprintf(stderr,
452             "    test-binary --output=text --output=xml --file=output.xml\n\n");
453     fprintf(stderr, "Other parameters:\n");
454     fprintf(stderr,
455             "  --regexp='regexp'\t Only selected tests"
456             " which names match regexp run\n\n");
457     fprintf(stderr, "  --start=<test id>\tStart from concrete test id");
458     fprintf(stderr, "  --group=<group name>\t Run tests only from one group\n");
459     fprintf(stderr, "  --runignored\t Run also ignored tests\n");
460     fprintf(stderr, "  --list\t Show a list of Test IDs\n");
461     fprintf(stderr, "  --listgroups\t Show a list of Test Group names \n");
462     fprintf(stderr, "  --only-from-xml=<xml file>\t Run only testcases specified in XML file \n"
463                     "       XML name is taken from attribute id=\"part1_part2\" as whole.\n"
464                     "       If part1 is not found (no _) then it is implicitily "
465                            "set according to suite part1 from binary tests\n");
466     fprintf(
467         stderr,
468         "  --listingroup=<group name>\t Show a list of Test IDS in one group\n");
469     fprintf(stderr, "  --allowchildlogs\t Allow to print logs from child process on screen.\n");
470     fprintf(stderr, "       When active child process will be able to print logs on stdout and stderr.\n");
471     fprintf(stderr, "       Both descriptors will be closed after test.\n");
472     fprintf(stderr, "  --help\t This help\n\n");
473     std::for_each(m_collectors.begin(),
474                   m_collectors.end(),
475                   [] (const TestResultsCollectors::value_type & collector)
476                   {
477                       fprintf(stderr,
478                               "Output %s has specific args:\n",
479                               collector.first.c_str());
480                       fprintf(stderr,
481                               "%s\n",
482                               collector.second->
483                                   CollectorSpecificHelp().c_str());
484                   });
485     fprintf(stderr, "For bug reporting, please write to:\n");
486     fprintf(stderr, "<p.dobrowolsk@samsung.com>\n");
487 }
488
489 int TestRunner::ExecTestRunner(int argc, char *argv[])
490 {
491     std::vector<std::string> args;
492     for (int i = 0; i < argc; ++i) {
493         args.push_back(argv[i]);
494     }
495     return ExecTestRunner(args);
496 }
497
498 void TestRunner::MarkAssertion()
499 {
500     ++m_totalAssertions;
501 }
502
503 int TestRunner::ExecTestRunner(ArgsList args)
504 {
505     m_runIgnored = false;
506     // Parse command line
507
508     args.erase(args.begin());
509
510     bool showHelp = false;
511     bool justList = false;
512     std::vector<std::string> xmlFiles;
513
514     TestResultsCollectorBasePtr currentCollector;
515
516     // Parse each argument
517     for(std::string &arg : args)
518     {
519         const std::string regexp = "--regexp=";
520         const std::string output = "--output=";
521         const std::string groupId = "--group=";
522         const std::string runIgnored = "--runignored";
523         const std::string listCmd = "--list";
524         const std::string startCmd = "--start=";
525         const std::string listGroupsCmd = "--listgroups";
526         const std::string listInGroup = "--listingroup=";
527         const std::string allowChildLogs = "--allowchildlogs";
528         const std::string onlyFromXML = "--only-from-xml=";
529
530         if (currentCollector) {
531             if (currentCollector->ParseCollectorSpecificArg(arg)) {
532                 continue;
533             }
534         }
535
536         if (arg.find(startCmd) == 0) {
537             arg.erase(0, startCmd.length());
538             for (auto &group : m_testGroups) {
539                 for (auto &tc : group.second) {
540                     if (tc.name == arg) {
541                         m_startTestId = arg;
542                         break;
543                     }
544                 }
545                 if (!m_startTestId.empty()) {
546                     break;
547                 }
548             }
549             if (!m_startTestId.empty()) {
550                 continue;
551             }
552             InvalidArgs();
553             fprintf(stderr, "Start test id has not been found\n");
554             Usage();
555             return 0;
556         } else if (arg.find(groupId) == 0) {
557             arg.erase(0, groupId.length());
558             TestCaseGroupMap::iterator found = m_testGroups.find(arg);
559             if (found != m_testGroups.end()) {
560                 std::string name = found->first;
561                 TestCaseStructList newList = found->second;
562                 m_testGroups.clear();
563                 m_testGroups[name] = newList;
564             } else {
565                 fprintf(stderr, "Group %s not found\n", arg.c_str());
566                 InvalidArgs();
567                 Usage();
568                 return -1;
569             }
570         } else if (arg == runIgnored) {
571             m_runIgnored = true;
572         } else if (arg == listCmd) {
573             justList = true;
574         } else if (arg == listGroupsCmd) {
575             for (auto &group : m_testGroups) {
576                 printf("GR:%s\n", group.first.c_str());
577             }
578             return 0;
579         } else if (arg.find(listInGroup) == 0) {
580             arg.erase(0, listInGroup.length());
581             for (auto &test : m_testGroups[arg]) {
582                 printf("ID:%s\n", test.name.c_str());
583             }
584             return 0;
585         } else if (arg.find(allowChildLogs) == 0) {
586             arg.erase(0, allowChildLogs.length());
587             m_allowChildLogs = true;
588         } else if (arg == "--help") {
589             showHelp = true;
590         } else if (arg.find(output) == 0) {
591             arg.erase(0, output.length());
592             if (m_collectors.find(arg) != m_collectors.end()) {
593                 InvalidArgs(
594                     "Multiple outputs of the same type are not supported!");
595                 Usage();
596                 return -1;
597             }
598             currentCollector.reset(TestResultsCollectorBase::Create(arg));
599             if (!currentCollector) {
600                 InvalidArgs("Unsupported output type!");
601                 Usage();
602                 return -1;
603             }
604             m_collectors[arg] = currentCollector;
605         } else if (arg.find(regexp) == 0) {
606             arg.erase(0, regexp.length());
607             if (arg.length() == 0) {
608                 InvalidArgs();
609                 Usage();
610                 return -1;
611             }
612
613             if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
614                 arg.erase(0);
615                 arg.erase(arg.length() - 1);
616             }
617
618             if (arg.length() == 0) {
619                 InvalidArgs();
620                 Usage();
621                 return -1;
622             }
623
624             pcrecpp::RE re(arg.c_str());
625             for (auto &group : m_testGroups) {
626                 TestCaseStructList newList;
627                 for (auto &tc : group.second)
628                 {
629                     if (re.PartialMatch(tc.name)) {
630                         newList.push_back(tc);
631                     }
632                 }
633                 group.second = newList;
634             }
635         } else if(arg.find(onlyFromXML) == 0) {
636             arg.erase(0, onlyFromXML.length());
637             if (arg.length() == 0) {
638                 InvalidArgs();
639                 Usage();
640                 return -1;
641             }
642
643             if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
644                 arg.erase(0);
645                 arg.erase(arg.length() - 1);
646             }
647
648             if (arg.length() == 0) {
649                 InvalidArgs();
650                 Usage();
651                 return -1;
652             }
653
654             xmlFiles.push_back(arg);
655         } else {
656             InvalidArgs();
657             Usage();
658             return -1;
659         }
660     }
661
662     if(!xmlFiles.empty())
663     {
664         if(!filterGroupsByXmls(xmlFiles))
665         {
666             fprintf(stderr, "XML file is not correct\n");
667             return 0;
668         }
669     }
670
671     if(justList)
672     {
673         for (auto &group : m_testGroups) {
674             for (auto &tc : group.second) {
675                 printf("ID:%s:%s\n", group.first.c_str(), tc.name.c_str());
676             }
677         }
678         return 0;
679     }
680
681     currentCollector.reset();
682
683     // Show help
684     if (showHelp) {
685         Usage();
686         return 0;
687     }
688
689     if (m_collectors.empty()) {
690         TestResultsCollectorBasePtr collector(
691             TestResultsCollectorBase::Create("text"));
692         m_collectors["text"] = collector;
693     }
694
695     for (auto &collector : m_collectors) {
696         if (!collector.second->Configure()) {
697             fprintf(stderr, "Could not configure selected output");
698             return 0;
699         }
700     }
701
702     // Run tests
703     RunTests();
704
705     return 0;
706 }
707
708 bool TestRunner::getRunIgnored() const
709 {
710     return m_runIgnored;
711 }
712
713 void TestRunner::Terminate()
714 {
715     m_terminate = true;
716 }
717
718 bool TestRunner::GetAllowChildLogs()
719 {
720     return m_allowChildLogs;
721 }
722
723 }
724 } // namespace DPL