modules/test/src/test_runner.cpp

   1 /*
   2  * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.cpp
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @version     1.0
  21  * @brief       This file is the implementation file of test runner
  22  */
  23 #include <stddef.h>
  24 #include <dpl/test/test_runner.h>
  25 #include <dpl/test/test_results_collector.h>
  26 #include <dpl/exception.h>
  27 #include <dpl/free_deleter.h>
  28 #include <memory>
  29 #include <dpl/foreach.h>
  30 #include <dpl/log/wrt_log.h>
  31 #include <dpl/colors.h>
  32 #include <pcrecpp.h>
  33 #include <algorithm>
  34 #include <cstdio>
  35 #include <memory.h>
  36 #include <libgen.h>
  37 #include <cstring>
  38 #include <cstdlib>
  39 #include <dpl/utils/wrt_global_settings.h>
  40
  41 #include <libxml/xpath.h>
  42 #include <libxml/xpathInternals.h>
  43 #include <libxml/parser.h>
  44 #include <libxml/tree.h>
  45
  46 #include <dpl/singleton_impl.h>
  47 IMPLEMENT_SINGLETON(DPL::Test::TestRunner)
  48
  49 namespace {
  50
  51 std::string getXMLNode(xmlNodePtr node)
  52 {
  53     std::string ret;
  54     xmlChar * value = xmlNodeGetContent(node);
  55     ret = std::string(reinterpret_cast<char*>(value));
  56     xmlFree(value);
  57     return ret;
  58 }
  59
  60 }
  61
  62
  63 namespace DPL {
  64 namespace Test {
  65 namespace // anonymous
  66 {
  67 std::string BaseName(std::string aPath)
  68 {
  69     std::unique_ptr<char[],free_deleter> path(strdup(aPath.c_str()));
  70     if (NULL == path.get()) {
  71         throw std::bad_alloc();
  72     }
  73     char* baseName = basename(path.get());
  74     std::string retValue = baseName;
  75     return retValue;
  76 }
  77 } // namespace anonymous
  78
  79 //! \brief Failed test message creator
  80 //!
  81 //! \param[in] aTest string for tested expression
  82 //! \param[in] aFile source file name
  83 //! \param[in] aLine source file line
  84 //! \param[in] aMessage error message
  85 TestRunner::TestFailed::TestFailed(const char* aTest,
  86                                    const char* aFile,
  87                                    int aLine,
  88                                    const std::string &aMessage)
  89 {
  90     std::ostringstream assertMsg;
  91     assertMsg << "[" << BaseName(aFile) << ":" << aLine
  92               << "] Assertion failed ("
  93               << aTest << ") " << aMessage;
  94     m_message = assertMsg.str();
  95 }
  96
  97 TestRunner::TestFailed::TestFailed(const std::string &message)
  98 {
  99     m_message = message;
 100 }
 101
 102 void TestRunner::RegisterTest(const char *testName, TestCase proc)
 103 {
 104     m_testGroups[m_currentGroup].push_back(TestCaseStruct(testName, proc));
 105 }
 106
 107 void TestRunner::InitGroup(const char* name)
 108 {
 109     m_currentGroup = name;
 110 }
 111
 112 void TestRunner::normalizeXMLTag(std::string& str, const std::string& testcase)
 113 {
 114     //Add testcase if missing
 115     std::string::size_type pos = str.find(testcase);
 116     if(pos != 0)
 117     {
 118         str = testcase + "_" + str;
 119     }
 120
 121     //dpl test runner cannot have '-' character in name so it have to be replaced
 122     // for TCT case to make comparision works
 123     std::replace(str.begin(), str.end(), '-', '_');
 124 }
 125
 126 bool TestRunner::filterGroupsByXmls(const std::vector<std::string> & files)
 127 {
 128     DECLARE_EXCEPTION_TYPE(DPL::Exception, XMLError)
 129
 130     const std::string idPath = "/test_definition/suite/set/testcase/@id";
 131
 132     bool success = true;
 133     std::map<std::string, bool> casesMap;
 134
 135     std::string testsuite;
 136     if(!m_testGroups.empty())
 137     {
 138         for(TestCaseGroupMap::const_iterator cit = m_testGroups.begin(); cit != m_testGroups.end(); ++cit)
 139         {
 140             if(!cit->second.empty())
 141             {
 142                 for(TestCaseStructList::const_iterator cj = cit->second.begin(); cj != cit->second.end(); ++cj)
 143                 {
 144                     std::string name = cj->name;
 145                     std::string::size_type st = name.find('_');
 146                     if(st != std::string::npos)
 147                     {
 148                         name = name.substr(0, st);
 149                         testsuite = name;
 150                         break;
 151                     }
 152                 }
 153                 if(!testsuite.empty()) break;
 154             }
 155         }
 156     }
 157
 158     xmlInitParser();
 159     LIBXML_TEST_VERSION
 160     xmlXPathInit();
 161
 162     Try
 163     {
 164         FOREACH(file, files)
 165         {
 166             xmlDocPtr doc;
 167             xmlXPathContextPtr xpathCtx;
 168
 169             doc = xmlReadFile(file->c_str(), NULL, 0);
 170             if (doc == NULL) {
 171                 ThrowMsg(XMLError, "File Problem");
 172             } else {
 173                 //context
 174                 xpathCtx = xmlXPathNewContext(doc);
 175                 if (xpathCtx == NULL) {
 176                     ThrowMsg(XMLError,
 177                              "Error: unable to create new XPath context\n");
 178                 }
 179                 xpathCtx->node = xmlDocGetRootElement(doc);
 180             }
 181
 182             std::string result;
 183             xmlXPathObjectPtr xpathObject;
 184             //get requested node's values
 185             xpathObject = xmlXPathEvalExpression(BAD_CAST idPath.c_str(), xpathCtx);
 186             if (xpathObject == NULL)
 187             {
 188                 ThrowMsg(XMLError, "XPath evaluation failure: " << idPath);
 189             }
 190             xmlNodeSetPtr nodes = xpathObject->nodesetval;
 191             unsigned size = (nodes) ? nodes->nodeNr : 0;
 192             WrtLogD("Found %i nodes matching xpath", size);
 193             for(unsigned i = 0; i < size; ++i)
 194             {
 195                 WrtLogD("Type: %i", nodes->nodeTab[i]->type);
 196                 if (nodes->nodeTab[i]->type == XML_ATTRIBUTE_NODE) {
 197                     xmlNodePtr curNode = nodes->nodeTab[i];
 198                     result = getXMLNode(curNode);
 199                     WrtLogD("Result: %s", result.c_str());
 200                     normalizeXMLTag(result, testsuite);
 201                     casesMap.insert(make_pair(result, false));
 202                 }
 203             }
 204             //Cleanup of XPath data
 205             xmlXPathFreeObject(xpathObject);
 206             xmlXPathFreeContext(xpathCtx);
 207             xmlFreeDoc(doc);
 208         }
 209     }
 210     Catch(XMLError)
 211     {
 212         WrtLogE("Libxml error: %s", _rethrown_exception.DumpToString().c_str());
 213         success = false;
 214     }
 215     xmlCleanupParser();
 216
 217     if(!filterByXML(casesMap))
 218     {
 219         success = false;
 220     }
 221
 222     return success;
 223 }
 224
 225 bool TestRunner::filterByXML(std::map<std::string, bool> & casesMap)
 226 {
 227     FOREACH(group, m_testGroups) {
 228         TestCaseStructList newList;
 229         FOREACH(iterator, group->second)
 230         {
 231             if (casesMap.find(iterator->name) != casesMap.end()) {
 232                 casesMap[iterator->name] = true;
 233                 newList.push_back(*iterator);
 234             }
 235         }
 236         group->second = newList;
 237     }
 238     FOREACH(cs, casesMap)
 239     {
 240         if(cs->second == false)
 241         {
 242             WrtLogE("Cannot find testcase from XML file: %s", cs->first.c_str());
 243             return false;
 244         }
 245     }
 246     return true;
 247 }
 248
 249 TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase)
 250 {
 251     try {
 252         testCase.proc();
 253     } catch (const TestFailed &e) {
 254         // Simple test failure
 255         CollectResult(testCase.name,
 256                       "",
 257                       TestResultsCollectorBase::FailStatus::FAILED,
 258                       e.GetMessage());
 259         return FAILED;
 260     } catch (const Ignored &e) {
 261         if (m_runIgnored) {
 262             // Simple test have to be implemented
 263             CollectResult(testCase.name,
 264                           "",
 265                           TestResultsCollectorBase::FailStatus::IGNORED,
 266                           e.GetMessage());
 267         }
 268
 269         return IGNORED;
 270     } catch (const DPL::Exception &e) {
 271         // DPL exception failure
 272         CollectResult(testCase.name,
 273                       "",
 274                       TestResultsCollectorBase::FailStatus::INTERNAL,
 275                       "DPL exception:" + e.GetMessage() + "\n" + e.DumpToString());
 276
 277         return FAILED;
 278     } catch (const std::exception &) {
 279         // std exception failure
 280         CollectResult(testCase.name,
 281                       "",
 282                       TestResultsCollectorBase::FailStatus::INTERNAL,
 283                       "std exception");
 284
 285         return FAILED;
 286     } catch (...) {
 287         // Unknown exception failure
 288         CollectResult(testCase.name,
 289                       "",
 290                       TestResultsCollectorBase::FailStatus::INTERNAL,
 291                       "unknown exception");
 292
 293         return FAILED;
 294     }
 295
 296     CollectResult(testCase.name,
 297                   "",
 298                   TestResultsCollectorBase::FailStatus::NONE);
 299
 300     // Everything OK
 301     return PASS;
 302 }
 303
 304 void TestRunner::RunTests()
 305 {
 306     using namespace DPL::Colors::Text;
 307
 308     Banner();
 309
 310     unsigned count = 0;
 311     FOREACH(group, m_testGroups) {
 312         count += group->second.size();
 313     }
 314
 315     std::for_each(m_collectors.begin(),
 316                   m_collectors.end(),
 317                   [count] (const TestResultsCollectors::value_type & collector)
 318                   {
 319                       collector.second->Start(count);
 320                   });
 321
 322     fprintf(stderr, "%sFound %d testcases...%s\n", GREEN_BEGIN, count, GREEN_END);
 323     fprintf(stderr, "%s%s%s\n", GREEN_BEGIN, "Running tests...", GREEN_END);
 324     FOREACH(group, m_testGroups) {
 325         TestCaseStructList list = group->second;
 326         if (!list.empty()) {
 327             std::for_each(
 328                 m_collectors.begin(),
 329                 m_collectors.end(),
 330                 [&group](const TestResultsCollectors::value_type & collector)
 331                 {
 332                     collector.second->
 333                         CollectCurrentTestGroupName(group->first);
 334                 });
 335             list.sort();
 336
 337             for (TestCaseStructList::const_iterator iterator = list.begin();
 338                  iterator != list.end();
 339                  ++iterator)
 340             {
 341                 TestCaseStruct test = *iterator;
 342                 if (m_startTestId == test.name) {
 343                     m_startTestId = "";
 344                 }
 345
 346                 if (m_startTestId.empty()) {
 347                     RunTestCase(test);
 348                 }
 349                 if (m_terminate == true) {
 350                     // Terminate quietly without any logs
 351                     return;
 352                 }
 353             }
 354         }
 355     }
 356
 357     std::for_each(m_collectors.begin(),
 358                   m_collectors.end(),
 359                   [] (const TestResultsCollectors::value_type & collector)
 360                   {
 361                       collector.second->Finish();
 362                   });
 363
 364     // Finished
 365     fprintf(stderr, "%s%s%s\n\n", GREEN_BEGIN, "Finished", GREEN_END);
 366 }
 367
 368 void TestRunner::CollectResult(
 369     const std::string& id,
 370     const std::string& description,
 371     const TestResultsCollectorBase::FailStatus::Type status,
 372     const std::string& reason)
 373 {
 374     std::for_each(m_collectors.begin(),
 375                   m_collectors.end(),
 376                   [&](const TestResultsCollectors::value_type & collector)
 377                   {
 378                       collector.second->CollectResult(id,
 379                                                       description,
 380                                                       status,
 381                                                       reason);
 382                   });
 383 }
 384
 385 void TestRunner::Banner()
 386 {
 387     using namespace DPL::Colors::Text;
 388     fprintf(stderr,
 389             "%s%s%s\n",
 390             BOLD_GREEN_BEGIN,
 391             "DPL tests runner",
 392             BOLD_GREEN_END);
 393     fprintf(stderr,
 394             "%s%s%s%s\n\n",
 395             GREEN_BEGIN,
 396             "Build: ",
 397             __TIMESTAMP__,
 398             GREEN_END);
 399 }
 400
 401 void TestRunner::InvalidArgs(const std::string& message)
 402 {
 403     using namespace DPL::Colors::Text;
 404     fprintf(stderr,
 405             "%s%s%s\n",
 406             BOLD_RED_BEGIN,
 407             message.c_str(),
 408             BOLD_RED_END);
 409 }
 410
 411 void TestRunner::Usage()
 412 {
 413     fprintf(stderr, "Usage: runner [options]\n\n");
 414     fprintf(stderr, "Output type:\n");
 415     fprintf(stderr, "  --output=<output type> --output=<output type> ...\n");
 416     fprintf(stderr, "\n  possible output types:\n");
 417     FOREACH(type, TestResultsCollectorBase::GetCollectorsNames()) {
 418         fprintf(stderr, "    --output=%s\n", type->c_str());
 419     }
 420     fprintf(stderr, "\n  example:\n");
 421     fprintf(stderr,
 422             "    test-binary --output=text --output=xml --file=output.xml\n\n");
 423     fprintf(stderr, "Other parameters:\n");
 424     fprintf(stderr,
 425             "  --regexp='regexp'\t Only selected tests"
 426             " which names match regexp run\n\n");
 427     fprintf(stderr, "  --start=<test id>\tStart from concrete test id");
 428     fprintf(stderr, "  --group=<group name>\t Run tests only from one group\n");
 429     fprintf(stderr, "  --runignored\t Run also ignored tests\n");
 430     fprintf(stderr, "  --list\t Show a list of Test IDs\n");
 431     fprintf(stderr, "  --listgroups\t Show a list of Test Group names \n");
 432     fprintf(stderr, "  --only-from-xml=<xml file>\t Run only testcases specified in XML file \n"
 433                     "       XML name is taken from attribute id=\"part1_part2\" as whole.\n"
 434                     "       If part1 is not found (no _) then it is implicitily "
 435                            "set according to suite part1 from binary tests\n");
 436     fprintf(
 437         stderr,
 438         "  --listingroup=<group name>\t Show a list of Test IDS in one group\n");
 439     fprintf(stderr, "  --allowchildlogs\t Allow to print logs from child process on screen.\n");
 440     fprintf(stderr, "       When active child process will be able to print logs on stdout and stderr.\n");
 441     fprintf(stderr, "       Both descriptors will be closed after test.\n");
 442     fprintf(stderr, "  --help\t This help\n\n");
 443     std::for_each(m_collectors.begin(),
 444                   m_collectors.end(),
 445                   [] (const TestResultsCollectors::value_type & collector)
 446                   {
 447                       fprintf(stderr,
 448                               "Output %s has specific args:\n",
 449                               collector.first.c_str());
 450                       fprintf(stderr,
 451                               "%s\n",
 452                               collector.second->
 453                                   CollectorSpecificHelp().c_str());
 454                   });
 455     fprintf(stderr, "For bug reporting, please write to:\n");
 456     fprintf(stderr, "<p.dobrowolsk@samsung.com>\n");
 457 }
 458
 459 int TestRunner::ExecTestRunner(int argc, char *argv[])
 460 {
 461     std::vector<std::string> args;
 462     for (int i = 0; i < argc; ++i) {
 463         args.push_back(argv[i]);
 464     }
 465     return ExecTestRunner(args);
 466 }
 467
 468 void TestRunner::MarkAssertion()
 469 {
 470     ++m_totalAssertions;
 471 }
 472
 473 int TestRunner::ExecTestRunner(const ArgsList& value)
 474 {
 475     m_runIgnored = false;
 476     ArgsList args = value;
 477     // Parse command line
 478     if (args.size() == 1) {
 479         InvalidArgs();
 480         Usage();
 481         return -1;
 482     }
 483
 484     args.erase(args.begin());
 485
 486     bool showHelp = false;
 487     bool justList = false;
 488     std::vector<std::string> xmlFiles;
 489
 490     TestResultsCollectorBasePtr currentCollector;
 491
 492     // Parse each argument
 493     FOREACH(it, args)
 494     {
 495         std::string arg = *it;
 496         const std::string regexp = "--regexp=";
 497         const std::string output = "--output=";
 498         const std::string groupId = "--group=";
 499         const std::string runIgnored = "--runignored";
 500         const std::string listCmd = "--list";
 501         const std::string startCmd = "--start=";
 502         const std::string listGroupsCmd = "--listgroups";
 503         const std::string listInGroup = "--listingroup=";
 504         const std::string allowChildLogs = "--allowchildlogs";
 505         const std::string onlyFromXML = "--only-from-xml=";
 506
 507         if (currentCollector) {
 508             if (currentCollector->ParseCollectorSpecificArg(arg)) {
 509                 continue;
 510             }
 511         }
 512
 513         if (arg.find(startCmd) == 0) {
 514             arg.erase(0, startCmd.length());
 515             FOREACH(group, m_testGroups) {
 516                 FOREACH(tc, group->second) {
 517                     if (tc->name == arg) {
 518                         m_startTestId = arg;
 519                         break;
 520                     }
 521                 }
 522                 if (!m_startTestId.empty()) {
 523                     break;
 524                 }
 525             }
 526             if (!m_startTestId.empty()) {
 527                 continue;
 528             }
 529             InvalidArgs();
 530             fprintf(stderr, "Start test id has not been found\n");
 531             Usage();
 532             return 0;
 533         } else if (arg.find(groupId) == 0) {
 534             arg.erase(0, groupId.length());
 535             TestCaseGroupMap::iterator found = m_testGroups.find(arg);
 536             if (found != m_testGroups.end()) {
 537                 std::string name = found->first;
 538                 TestCaseStructList newList = found->second;
 539                 m_testGroups.clear();
 540                 m_testGroups[name] = newList;
 541             } else {
 542                 fprintf(stderr, "Group %s not found\n", arg.c_str());
 543                 InvalidArgs();
 544                 Usage();
 545                 return -1;
 546             }
 547         } else if (arg == runIgnored) {
 548             m_runIgnored = true;
 549         } else if (arg == listCmd) {
 550             justList = true;
 551         } else if (arg == listGroupsCmd) {
 552             FOREACH(group, m_testGroups) {
 553                 printf("GR:%s\n", group->first.c_str());
 554             }
 555             return 0;
 556         } else if (arg.find(listInGroup) == 0) {
 557             arg.erase(0, listInGroup.length());
 558             FOREACH(test, m_testGroups[arg]) {
 559                 printf("ID:%s\n", test->name.c_str());
 560             }
 561             return 0;
 562         } else if (arg.find(allowChildLogs) == 0) {
 563             arg.erase(0, allowChildLogs.length());
 564             m_allowChildLogs = true;
 565         } else if (arg == "--help") {
 566             showHelp = true;
 567         } else if (arg.find(output) == 0) {
 568             arg.erase(0, output.length());
 569             if (m_collectors.find(arg) != m_collectors.end()) {
 570                 InvalidArgs(
 571                     "Multiple outputs of the same type are not supported!");
 572                 Usage();
 573                 return -1;
 574             }
 575             currentCollector.reset(TestResultsCollectorBase::Create(arg));
 576             if (!currentCollector) {
 577                 InvalidArgs("Unsupported output type!");
 578                 Usage();
 579                 return -1;
 580             }
 581             m_collectors[arg] = currentCollector;
 582         } else if (arg.find(regexp) == 0) {
 583             arg.erase(0, regexp.length());
 584             if (arg.length() == 0) {
 585                 InvalidArgs();
 586                 Usage();
 587                 return -1;
 588             }
 589
 590             if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
 591                 arg.erase(0);
 592                 arg.erase(arg.length() - 1);
 593             }
 594
 595             if (arg.length() == 0) {
 596                 InvalidArgs();
 597                 Usage();
 598                 return -1;
 599             }
 600
 601             pcrecpp::RE re(arg.c_str());
 602             FOREACH(group, m_testGroups) {
 603                 TestCaseStructList newList;
 604                 FOREACH(iterator, group->second)
 605                 {
 606                     if (re.PartialMatch(iterator->name)) {
 607                         newList.push_back(*iterator);
 608                     }
 609                 }
 610                 group->second = newList;
 611             }
 612         } else if(arg.find(onlyFromXML) == 0) {
 613             arg.erase(0, onlyFromXML.length());
 614             if (arg.length() == 0) {
 615                 InvalidArgs();
 616                 Usage();
 617                 return -1;
 618             }
 619
 620             if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
 621                 arg.erase(0);
 622                 arg.erase(arg.length() - 1);
 623             }
 624
 625             if (arg.length() == 0) {
 626                 InvalidArgs();
 627                 Usage();
 628                 return -1;
 629             }
 630
 631             xmlFiles.push_back(arg);
 632         } else {
 633             InvalidArgs();
 634             Usage();
 635             return -1;
 636         }
 637     }
 638
 639     if(!xmlFiles.empty())
 640     {
 641         if(!filterGroupsByXmls(xmlFiles))
 642         {
 643             fprintf(stderr, "XML file is not correct\n");
 644             return 0;
 645         }
 646     }
 647
 648     if(justList)
 649     {
 650         FOREACH(group, m_testGroups) {
 651             FOREACH(test, group->second) {
 652                 printf("ID:%s:%s\n", group->first.c_str(), test->name.c_str());
 653             }
 654         }
 655         return 0;
 656     }
 657
 658     currentCollector.reset();
 659
 660     // Show help
 661     if (showHelp) {
 662         Usage();
 663         return 0;
 664     }
 665
 666     if (m_collectors.empty()) {
 667         TestResultsCollectorBasePtr collector(
 668             TestResultsCollectorBase::Create("text"));
 669         m_collectors["text"] = collector;
 670     }
 671
 672     for (auto it = m_collectors.begin(); it != m_collectors.end(); ++it) {
 673         if (!it->second->Configure()) {
 674             fprintf(stderr, "Could not configure selected output");
 675             return 0;
 676         }
 677     }
 678
 679     // Run tests
 680     RunTests();
 681
 682     return 0;
 683 }
 684
 685 bool TestRunner::getRunIgnored() const
 686 {
 687     return m_runIgnored;
 688 }
 689
 690 void TestRunner::Terminate()
 691 {
 692     m_terminate = true;
 693 }
 694
 695 bool TestRunner::GetAllowChildLogs()
 696 {
 697     return m_allowChildLogs;
 698 }
 699
 700 }
 701 } // namespace DPL