vcore/src/dpl/test/src/test_runner.cpp

   1 /*
   2  * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.cpp
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @version     1.0
  21  * @brief       This file is the implementation file of test runner
  22  */
  23 #include <stddef.h>
  24 #include <dpl/test/test_runner.h>
  25 #include <dpl/test/test_results_collector.h>
  26 #include <dpl/exception.h>
  27 #include <dpl/free_deleter.h>
  28 #include <memory>
  29 #include <dpl/foreach.h>
  30 #include <dpl/colors.h>
  31 #include <pcrecpp.h>
  32 #include <algorithm>
  33 #include <cstdio>
  34 #include <memory.h>
  35 #include <libgen.h>
  36 #include <cstring>
  37 #include <cstdlib>
  38
  39 #include <libxml/xpath.h>
  40 #include <libxml/xpathInternals.h>
  41 #include <libxml/parser.h>
  42 #include <libxml/tree.h>
  43
  44 #include <dpl/singleton_impl.h>
  45 IMPLEMENT_SINGLETON(VcoreDPL::Test::TestRunner)
  46
  47 namespace {
  48
  49 std::string getXMLNode(xmlNodePtr node)
  50 {
  51     std::string ret;
  52     xmlChar * value = xmlNodeGetContent(node);
  53     if (value != NULL) {
  54         ret = std::string(reinterpret_cast<char*>(value));
  55         xmlFree(value);
  56     }
  57     return ret;
  58 }
  59
  60 }
  61
  62
  63 namespace VcoreDPL {
  64 namespace Test {
  65 namespace // anonymous
  66 {
  67 std::string BaseName(std::string aPath)
  68 {
  69     std::unique_ptr<char[],free_deleter> path(strdup(aPath.c_str()));
  70     if (NULL == path.get()) {
  71         throw std::bad_alloc();
  72     }
  73     char* baseName = basename(path.get());
  74     std::string retValue = baseName;
  75     return retValue;
  76 }
  77 } // namespace anonymous
  78
  79 //! \brief Failed test message creator
  80 //!
  81 //! \param[in] aTest string for tested expression
  82 //! \param[in] aFile source file name
  83 //! \param[in] aLine source file line
  84 //! \param[in] aMessage error message
  85 TestRunner::TestFailed::TestFailed(const char* aTest,
  86                                    const char* aFile,
  87                                    int aLine,
  88                                    const std::string &aMessage)
  89 {
  90     std::ostringstream assertMsg;
  91     assertMsg << "[" << BaseName(aFile) << ":" << aLine
  92               << "] Assertion failed ("
  93               << aTest << ") " << aMessage;
  94     m_message = assertMsg.str();
  95 }
  96
  97 TestRunner::TestFailed::TestFailed(const std::string &message)
  98 {
  99     m_message = message;
 100 }
 101
 102 void TestRunner::RegisterTest(const char *testName, TestCase proc)
 103 {
 104     m_testGroups[m_currentGroup].push_back(TestCaseStruct(testName, proc));
 105 }
 106
 107 void TestRunner::InitGroup(const char* name)
 108 {
 109     m_currentGroup = name;
 110 }
 111
 112 void TestRunner::normalizeXMLTag(std::string& str, const std::string& testcase)
 113 {
 114     //Add testcase if missing
 115     std::string::size_type pos = str.find(testcase);
 116     if(pos != 0)
 117     {
 118         str = testcase + "_" + str;
 119     }
 120
 121     //dpl test runner cannot have '-' character in name so it have to be replaced
 122     // for TCT case to make comparision works
 123     std::replace(str.begin(), str.end(), '-', '_');
 124 }
 125
 126 bool TestRunner::filterGroupsByXmls(const std::vector<std::string> & files)
 127 {
 128     DECLARE_EXCEPTION_TYPE(VcoreDPL::Exception, XMLError)
 129
 130     const std::string idPath = "/test_definition/suite/set/testcase/@id";
 131
 132     bool success = true;
 133     std::map<std::string, bool> casesMap;
 134
 135     std::string testsuite;
 136     if(!m_testGroups.empty())
 137     {
 138         for(TestCaseGroupMap::const_iterator cit = m_testGroups.begin(); cit != m_testGroups.end(); ++cit)
 139         {
 140             if(!cit->second.empty())
 141             {
 142                 for(TestCaseStructList::const_iterator cj = cit->second.begin(); cj != cit->second.end(); ++cj)
 143                 {
 144                     std::string name = cj->name;
 145                     std::string::size_type st = name.find('_');
 146                     if(st != std::string::npos)
 147                     {
 148                         name = name.substr(0, st);
 149                         testsuite = name;
 150                         break;
 151                     }
 152                 }
 153                 if(!testsuite.empty()) break;
 154             }
 155         }
 156     }
 157
 158     xmlInitParser();
 159     LIBXML_TEST_VERSION
 160     xmlXPathInit();
 161
 162     Try
 163     {
 164         FOREACH(file, files)
 165         {
 166             xmlDocPtr doc;
 167             xmlXPathContextPtr xpathCtx;
 168
 169             doc = xmlReadFile(file->c_str(), NULL, 0);
 170             if (doc == NULL) {
 171                 ThrowMsg(XMLError, "File Problem");
 172             } else {
 173                 //context
 174                 xpathCtx = xmlXPathNewContext(doc);
 175                 if (xpathCtx == NULL) {
 176                     ThrowMsg(XMLError,
 177                              "Error: unable to create new XPath context\n");
 178                 }
 179                 xpathCtx->node = xmlDocGetRootElement(doc);
 180             }
 181
 182             std::string result;
 183             xmlXPathObjectPtr xpathObject;
 184             //get requested node's values
 185             xpathObject = xmlXPathEvalExpression(BAD_CAST idPath.c_str(), xpathCtx);
 186             if (xpathObject == NULL)
 187             {
 188                 ThrowMsg(XMLError, "XPath evaluation failure: " << idPath);
 189             }
 190             xmlNodeSetPtr nodes = xpathObject->nodesetval;
 191             unsigned size = (nodes) ? nodes->nodeNr : 0;
 192             for(unsigned i = 0; i < size; ++i)
 193             {
 194                 if (nodes->nodeTab[i]->type == XML_ATTRIBUTE_NODE) {
 195                     xmlNodePtr curNode = nodes->nodeTab[i];
 196                     result = getXMLNode(curNode);
 197                     normalizeXMLTag(result, testsuite);
 198                     casesMap.insert(make_pair(result, false));
 199                 }
 200             }
 201             //Cleanup of XPath data
 202             xmlXPathFreeObject(xpathObject);
 203             xmlXPathFreeContext(xpathCtx);
 204             xmlFreeDoc(doc);
 205         }
 206     }
 207     Catch(XMLError)
 208     {
 209         success = false;
 210     }
 211     xmlCleanupParser();
 212
 213     if(!filterByXML(casesMap))
 214     {
 215         success = false;
 216     }
 217
 218     return success;
 219 }
 220
 221 bool TestRunner::filterByXML(std::map<std::string, bool> & casesMap)
 222 {
 223     FOREACH(group, m_testGroups) {
 224         TestCaseStructList newList;
 225         FOREACH(iterator, group->second)
 226         {
 227             if (casesMap.find(iterator->name) != casesMap.end()) {
 228                 casesMap[iterator->name] = true;
 229                 newList.push_back(*iterator);
 230             }
 231         }
 232         group->second = newList;
 233     }
 234     FOREACH(cs, casesMap)
 235     {
 236         if(cs->second == false)
 237         {
 238             return false;
 239         }
 240     }
 241     return true;
 242 }
 243
 244 TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase)
 245 {
 246     try {
 247         testCase.proc();
 248     } catch (const TestFailed &e) {
 249         // Simple test failure
 250         CollectResult(testCase.name,
 251                       "",
 252                       TestResultsCollectorBase::FailStatus::FAILED,
 253                       e.GetMessage());
 254         return FAILED;
 255     } catch (const Ignored &e) {
 256         if (m_runIgnored) {
 257             // Simple test have to be implemented
 258             CollectResult(testCase.name,
 259                           "",
 260                           TestResultsCollectorBase::FailStatus::IGNORED,
 261                           e.GetMessage());
 262         }
 263
 264         return IGNORED;
 265     } catch (const VcoreDPL::Exception &e) {
 266         // DPL exception failure
 267         CollectResult(testCase.name,
 268                       "",
 269                       TestResultsCollectorBase::FailStatus::INTERNAL,
 270                       "DPL exception:" + e.GetMessage() + "\n" + e.DumpToString());
 271
 272         return FAILED;
 273     } catch (const std::exception &) {
 274         // std exception failure
 275         CollectResult(testCase.name,
 276                       "",
 277                       TestResultsCollectorBase::FailStatus::INTERNAL,
 278                       "std exception");
 279
 280         return FAILED;
 281     } catch (...) {
 282         // Unknown exception failure
 283         CollectResult(testCase.name,
 284                       "",
 285                       TestResultsCollectorBase::FailStatus::INTERNAL,
 286                       "unknown exception");
 287
 288         return FAILED;
 289     }
 290
 291     CollectResult(testCase.name,
 292                   "",
 293                   TestResultsCollectorBase::FailStatus::NONE);
 294
 295     // Everything OK
 296     return PASS;
 297 }
 298
 299 void TestRunner::RunTests()
 300 {
 301     using namespace VcoreDPL::Colors::Text;
 302
 303     Banner();
 304
 305     unsigned count = 0;
 306     FOREACH(group, m_testGroups) {
 307         count += group->second.size();
 308     }
 309
 310     std::for_each(m_collectors.begin(),
 311                   m_collectors.end(),
 312                   [count] (const TestResultsCollectors::value_type & collector)
 313                   {
 314                       collector.second->Start(count);
 315                   });
 316
 317     fprintf(stderr, "%sFound %d testcases...%s\n", GREEN_BEGIN, count, GREEN_END);
 318     fprintf(stderr, "%s%s%s\n", GREEN_BEGIN, "Running tests...", GREEN_END);
 319     FOREACH(group, m_testGroups) {
 320         TestCaseStructList list = group->second;
 321         if (!list.empty()) {
 322             std::for_each(
 323                 m_collectors.begin(),
 324                 m_collectors.end(),
 325                 [&group](const TestResultsCollectors::value_type & collector)
 326                 {
 327                     collector.second->
 328                         CollectCurrentTestGroupName(group->first);
 329                 });
 330             list.sort();
 331
 332             for (TestCaseStructList::const_iterator iterator = list.begin();
 333                  iterator != list.end();
 334                  ++iterator)
 335             {
 336                 TestCaseStruct test = *iterator;
 337                 if (m_startTestId == test.name) {
 338                     m_startTestId = "";
 339                 }
 340
 341                 if (m_startTestId.empty()) {
 342                     RunTestCase(test);
 343                 }
 344                 if (m_terminate == true) {
 345                     // Terminate quietly without any logs
 346                     return;
 347                 }
 348             }
 349         }
 350     }
 351
 352     std::for_each(m_collectors.begin(),
 353                   m_collectors.end(),
 354                   [] (const TestResultsCollectors::value_type & collector)
 355                   {
 356                       collector.second->Finish();
 357                   });
 358
 359     // Finished
 360     fprintf(stderr, "%s%s%s\n\n", GREEN_BEGIN, "Finished", GREEN_END);
 361 }
 362
 363 void TestRunner::CollectResult(
 364     const std::string& id,
 365     const std::string& description,
 366     const TestResultsCollectorBase::FailStatus::Type status,
 367     const std::string& reason)
 368 {
 369     std::for_each(m_collectors.begin(),
 370                   m_collectors.end(),
 371                   [&](const TestResultsCollectors::value_type & collector)
 372                   {
 373                       collector.second->CollectResult(id,
 374                                                       description,
 375                                                       status,
 376                                                       reason);
 377                   });
 378 }
 379
 380 void TestRunner::Banner()
 381 {
 382     using namespace VcoreDPL::Colors::Text;
 383     fprintf(stderr,
 384             "%s%s%s\n",
 385             BOLD_GREEN_BEGIN,
 386             "DPL tests runner",
 387             BOLD_GREEN_END);
 388     fprintf(stderr,
 389             "%s%s%s%s\n\n",
 390             GREEN_BEGIN,
 391             "Build: ",
 392             __TIMESTAMP__,
 393             GREEN_END);
 394 }
 395
 396 void TestRunner::InvalidArgs(const std::string& message)
 397 {
 398     using namespace VcoreDPL::Colors::Text;
 399     fprintf(stderr,
 400             "%s%s%s\n",
 401             BOLD_RED_BEGIN,
 402             message.c_str(),
 403             BOLD_RED_END);
 404 }
 405
 406 void TestRunner::Usage()
 407 {
 408     fprintf(stderr, "Usage: runner [options]\n\n");
 409     fprintf(stderr, "Output type:\n");
 410     fprintf(stderr, "  --output=<output type> --output=<output type> ...\n");
 411     fprintf(stderr, "\n  possible output types:\n");
 412     FOREACH(type, TestResultsCollectorBase::GetCollectorsNames()) {
 413         fprintf(stderr, "    --output=%s\n", type->c_str());
 414     }
 415     fprintf(stderr, "\n  example:\n");
 416     fprintf(stderr,
 417             "    test-binary --output=text --output=xml --file=output.xml\n\n");
 418     fprintf(stderr, "Other parameters:\n");
 419     fprintf(stderr,
 420             "  --regexp='regexp'\t Only selected tests"
 421             " which names match regexp run\n\n");
 422     fprintf(stderr, "  --start=<test id>\tStart from concrete test id");
 423     fprintf(stderr, "  --group=<group name>\t Run tests only from one group\n");
 424     fprintf(stderr, "  --runignored\t Run also ignored tests\n");
 425     fprintf(stderr, "  --list\t Show a list of Test IDs\n");
 426     fprintf(stderr, "  --listgroups\t Show a list of Test Group names \n");
 427     fprintf(stderr, "  --only-from-xml=<xml file>\t Run only testcases specified in XML file \n"
 428                     "       XML name is taken from attribute id=\"part1_part2\" as whole.\n"
 429                     "       If part1 is not found (no _) then it is implicitily "
 430                            "set according to suite part1 from binary tests\n");
 431     fprintf(
 432         stderr,
 433         "  --listingroup=<group name>\t Show a list of Test IDS in one group\n");
 434     fprintf(stderr, "  --allowchildlogs\t Allow to print logs from child process on screen.\n");
 435     fprintf(stderr, "       When active child process will be able to print logs on stdout and stderr.\n");
 436     fprintf(stderr, "       Both descriptors will be closed after test.\n");
 437     fprintf(stderr, "  --help\t This help\n\n");
 438     std::for_each(m_collectors.begin(),
 439                   m_collectors.end(),
 440                   [] (const TestResultsCollectors::value_type & collector)
 441                   {
 442                       fprintf(stderr,
 443                               "Output %s has specific args:\n",
 444                               collector.first.c_str());
 445                       fprintf(stderr,
 446                               "%s\n",
 447                               collector.second->
 448                                   CollectorSpecificHelp().c_str());
 449                   });
 450     fprintf(stderr, "For bug reporting, please write to:\n");
 451     fprintf(stderr, "<p.dobrowolsk@samsung.com>\n");
 452 }
 453
 454 int TestRunner::ExecTestRunner(int argc, char *argv[])
 455 {
 456     std::vector<std::string> args;
 457     for (int i = 0; i < argc; ++i) {
 458         args.push_back(argv[i]);
 459     }
 460     return ExecTestRunner(args);
 461 }
 462
 463 void TestRunner::MarkAssertion()
 464 {
 465     ++m_totalAssertions;
 466 }
 467
 468 int TestRunner::ExecTestRunner(const ArgsList& value)
 469 {
 470     m_runIgnored = false;
 471     ArgsList args = value;
 472     // Parse command line
 473     if (args.size() == 1) {
 474         InvalidArgs();
 475         Usage();
 476         return -1;
 477     }
 478
 479     args.erase(args.begin());
 480
 481     bool showHelp = false;
 482     bool justList = false;
 483     std::vector<std::string> xmlFiles;
 484
 485     TestResultsCollectorBasePtr currentCollector;
 486
 487     // Parse each argument
 488     FOREACH(it, args)
 489     {
 490         std::string arg = *it;
 491         const std::string regexp = "--regexp=";
 492         const std::string output = "--output=";
 493         const std::string groupId = "--group=";
 494         const std::string runIgnored = "--runignored";
 495         const std::string listCmd = "--list";
 496         const std::string startCmd = "--start=";
 497         const std::string listGroupsCmd = "--listgroups";
 498         const std::string listInGroup = "--listingroup=";
 499         const std::string allowChildLogs = "--allowchildlogs";
 500         const std::string onlyFromXML = "--only-from-xml=";
 501
 502         if (currentCollector) {
 503             if (currentCollector->ParseCollectorSpecificArg(arg)) {
 504                 continue;
 505             }
 506         }
 507
 508         if (arg.find(startCmd) == 0) {
 509             arg.erase(0, startCmd.length());
 510             FOREACH(group, m_testGroups) {
 511                 FOREACH(tc, group->second) {
 512                     if (tc->name == arg) {
 513                         m_startTestId = arg;
 514                         break;
 515                     }
 516                 }
 517                 if (!m_startTestId.empty()) {
 518                     break;
 519                 }
 520             }
 521             if (!m_startTestId.empty()) {
 522                 continue;
 523             }
 524             InvalidArgs();
 525             fprintf(stderr, "Start test id has not been found\n");
 526             Usage();
 527             return 0;
 528         } else if (arg.find(groupId) == 0) {
 529             arg.erase(0, groupId.length());
 530             TestCaseGroupMap::iterator found = m_testGroups.find(arg);
 531             if (found != m_testGroups.end()) {
 532                 std::string name = found->first;
 533                 TestCaseStructList newList = found->second;
 534                 m_testGroups.clear();
 535                 m_testGroups[name] = newList;
 536             } else {
 537                 fprintf(stderr, "Group %s not found\n", arg.c_str());
 538                 InvalidArgs();
 539                 Usage();
 540                 return -1;
 541             }
 542         } else if (arg == runIgnored) {
 543             m_runIgnored = true;
 544         } else if (arg == listCmd) {
 545             justList = true;
 546         } else if (arg == listGroupsCmd) {
 547             FOREACH(group, m_testGroups) {
 548                 printf("GR:%s\n", group->first.c_str());
 549             }
 550             return 0;
 551         } else if (arg.find(listInGroup) == 0) {
 552             arg.erase(0, listInGroup.length());
 553             FOREACH(test, m_testGroups[arg]) {
 554                 printf("ID:%s\n", test->name.c_str());
 555             }
 556             return 0;
 557         } else if (arg.find(allowChildLogs) == 0) {
 558             arg.erase(0, allowChildLogs.length());
 559             m_allowChildLogs = true;
 560         } else if (arg == "--help") {
 561             showHelp = true;
 562         } else if (arg.find(output) == 0) {
 563             arg.erase(0, output.length());
 564             if (m_collectors.find(arg) != m_collectors.end()) {
 565                 InvalidArgs(
 566                     "Multiple outputs of the same type are not supported!");
 567                 Usage();
 568                 return -1;
 569             }
 570             currentCollector.reset(TestResultsCollectorBase::Create(arg));
 571             if (!currentCollector) {
 572                 InvalidArgs("Unsupported output type!");
 573                 Usage();
 574                 return -1;
 575             }
 576             m_collectors[arg] = currentCollector;
 577         } else if (arg.find(regexp) == 0) {
 578             arg.erase(0, regexp.length());
 579             if (arg.length() == 0) {
 580                 InvalidArgs();
 581                 Usage();
 582                 return -1;
 583             }
 584
 585             if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
 586                 arg.erase(0);
 587                 arg.erase(arg.length() - 1);
 588             }
 589
 590             if (arg.length() == 0) {
 591                 InvalidArgs();
 592                 Usage();
 593                 return -1;
 594             }
 595
 596             pcrecpp::RE re(arg.c_str());
 597             FOREACH(group, m_testGroups) {
 598                 TestCaseStructList newList;
 599                 FOREACH(iterator, group->second)
 600                 {
 601                     if (re.PartialMatch(iterator->name)) {
 602                         newList.push_back(*iterator);
 603                     }
 604                 }
 605                 group->second = newList;
 606             }
 607         } else if(arg.find(onlyFromXML) == 0) {
 608             arg.erase(0, onlyFromXML.length());
 609             if (arg.length() == 0) {
 610                 InvalidArgs();
 611                 Usage();
 612                 return -1;
 613             }
 614
 615             if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
 616                 arg.erase(0);
 617                 arg.erase(arg.length() - 1);
 618             }
 619
 620             if (arg.length() == 0) {
 621                 InvalidArgs();
 622                 Usage();
 623                 return -1;
 624             }
 625
 626             xmlFiles.push_back(arg);
 627         } else {
 628             InvalidArgs();
 629             Usage();
 630             return -1;
 631         }
 632     }
 633
 634     if(!xmlFiles.empty())
 635     {
 636         if(!filterGroupsByXmls(xmlFiles))
 637         {
 638             fprintf(stderr, "XML file is not correct\n");
 639             return 0;
 640         }
 641     }
 642
 643     if(justList)
 644     {
 645         FOREACH(group, m_testGroups) {
 646             FOREACH(test, group->second) {
 647                 printf("ID:%s:%s\n", group->first.c_str(), test->name.c_str());
 648             }
 649         }
 650         return 0;
 651     }
 652
 653     currentCollector.reset();
 654
 655     // Show help
 656     if (showHelp) {
 657         Usage();
 658         return 0;
 659     }
 660
 661     if (m_collectors.empty()) {
 662         TestResultsCollectorBasePtr collector(
 663             TestResultsCollectorBase::Create("text"));
 664         m_collectors["text"] = collector;
 665     }
 666
 667     for (auto it = m_collectors.begin(); it != m_collectors.end(); ++it) {
 668         if (!it->second->Configure()) {
 669             fprintf(stderr, "Could not configure selected output");
 670             return 0;
 671         }
 672     }
 673
 674     // Run tests
 675     RunTests();
 676
 677     return 0;
 678 }
 679
 680 bool TestRunner::getRunIgnored() const
 681 {
 682     return m_runIgnored;
 683 }
 684
 685 void TestRunner::Terminate()
 686 {
 687     m_terminate = true;
 688 }
 689
 690 bool TestRunner::GetAllowChildLogs()
 691 {
 692     return m_allowChildLogs;
 693 }
 694
 695 }
 696 } // namespace VcoreDPL