modules_wearable/test/src/test_runner.cpp

   1 /*
   2  * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.cpp
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @version     1.0
  21  * @brief       This file is the implementation file of test runner
  22  */
  23 #include <stddef.h>
  24 #include <dpl/test/test_runner.h>
  25 #include <dpl/test/test_results_collector.h>
  26 #include <dpl/exception.h>
  27 #include <dpl/scoped_free.h>
  28 #include <dpl/foreach.h>
  29 #include <dpl/log/log.h>
  30 #include <dpl/colors.h>
  31 #include <pcrecpp.h>
  32 #include <algorithm>
  33 #include <cstdio>
  34 #include <memory.h>
  35 #include <libgen.h>
  36 #include <cstring>
  37 #include <cstdlib>
  38 #include <dpl/utils/wrt_global_settings.h>
  39
  40 #include <libxml/xpath.h>
  41 #include <libxml/xpathInternals.h>
  42 #include <libxml/parser.h>
  43 #include <libxml/tree.h>
  44
  45 #include <dpl/singleton_impl.h>
  46 IMPLEMENT_SINGLETON(DPL::Test::TestRunner)
  47
  48 namespace {
  49
  50 std::string getXMLNode(xmlNodePtr node)
  51 {
  52     std::string ret;
  53     xmlChar * value = xmlNodeGetContent(node);
  54     ret = std::string(reinterpret_cast<char*>(value));
  55     xmlFree(value);
  56     return ret;
  57 }
  58
  59 }
  60
  61
  62 namespace DPL {
  63 namespace Test {
  64 namespace // anonymous
  65 {
  66 std::string BaseName(std::string aPath)
  67 {
  68     ScopedFree<char> path(strdup(aPath.c_str()));
  69     if (NULL == path.Get()) {
  70         throw std::bad_alloc();
  71     }
  72     char* baseName = basename(path.Get());
  73     std::string retValue = baseName;
  74     return retValue;
  75 }
  76 } // namespace anonymous
  77
  78 //! \brief Failed test message creator
  79 //!
  80 //! \param[in] aTest string for tested expression
  81 //! \param[in] aFile source file name
  82 //! \param[in] aLine source file line
  83 //! \param[in] aMessage error message
  84 TestRunner::TestFailed::TestFailed(const char* aTest,
  85                                    const char* aFile,
  86                                    int aLine,
  87                                    const std::string &aMessage)
  88 {
  89     std::ostringstream assertMsg;
  90     assertMsg << "[" << BaseName(aFile) << ":" << aLine
  91               << "] Assertion failed ("
  92               << aTest << ") " << aMessage;
  93     m_message = assertMsg.str();
  94 }
  95
  96 TestRunner::TestFailed::TestFailed(const std::string &message)
  97 {
  98     m_message = message;
  99 }
 100
 101 void TestRunner::RegisterTest(const char *testName, TestCase proc)
 102 {
 103     m_testGroups[m_currentGroup].push_back(TestCaseStruct(testName, proc));
 104 }
 105
 106 void TestRunner::InitGroup(const char* name)
 107 {
 108     m_currentGroup = name;
 109 }
 110
 111 void TestRunner::normalizeXMLTag(std::string& str, const std::string& testcase)
 112 {
 113     //Add testcase if missing
 114     std::string::size_type pos = str.find(testcase);
 115     if(pos != 0)
 116     {
 117         str = testcase + "_" + str;
 118     }
 119
 120     //dpl test runner cannot have '-' character in name so it have to be replaced
 121     // for TCT case to make comparision works
 122     std::replace(str.begin(), str.end(), '-', '_');
 123 }
 124
 125 bool TestRunner::filterGroupsByXmls(const std::vector<std::string> & files)
 126 {
 127     DECLARE_EXCEPTION_TYPE(DPL::Exception, XMLError)
 128
 129     const std::string idPath = "/test_definition/suite/set/testcase/@id";
 130
 131     bool success = true;
 132     std::map<std::string, bool> casesMap;
 133
 134     std::string testsuite;
 135     if(!m_testGroups.empty())
 136     {
 137         for(TestCaseGroupMap::const_iterator cit = m_testGroups.begin(); cit != m_testGroups.end(); ++cit)
 138         {
 139             if(!cit->second.empty())
 140             {
 141                 for(TestCaseStructList::const_iterator cj = cit->second.begin(); cj != cit->second.end(); ++cj)
 142                 {
 143                     std::string name = cj->name;
 144                     std::string::size_type st = name.find('_');
 145                     if(st != std::string::npos)
 146                     {
 147                         name = name.substr(0, st);
 148                         testsuite = name;
 149                         break;
 150                     }
 151                 }
 152                 if(!testsuite.empty()) break;
 153             }
 154         }
 155     }
 156
 157     xmlInitParser();
 158     LIBXML_TEST_VERSION
 159     xmlXPathInit();
 160
 161     Try
 162     {
 163         FOREACH(file, files)
 164         {
 165             xmlDocPtr doc;
 166             xmlXPathContextPtr xpathCtx;
 167
 168             doc = xmlReadFile(file->c_str(), NULL, 0);
 169             if (doc == NULL) {
 170                 ThrowMsg(XMLError, "File Problem");
 171             } else {
 172                 //context
 173                 xpathCtx = xmlXPathNewContext(doc);
 174                 if (xpathCtx == NULL) {
 175                     ThrowMsg(XMLError,
 176                              "Error: unable to create new XPath context\n");
 177                 }
 178                 xpathCtx->node = xmlDocGetRootElement(doc);
 179             }
 180
 181             std::string result;
 182             xmlXPathObjectPtr xpathObject;
 183             //get requested node's values
 184             xpathObject = xmlXPathEvalExpression(BAD_CAST idPath.c_str(), xpathCtx);
 185             if (xpathObject == NULL)
 186             {
 187                 ThrowMsg(XMLError, "XPath evaluation failure: " << idPath);
 188             }
 189             xmlNodeSetPtr nodes = xpathObject->nodesetval;
 190             unsigned size = (nodes) ? nodes->nodeNr : 0;
 191             LogDebug("Found " << size << " nodes matching xpath");
 192             for(unsigned i = 0; i < size; ++i)
 193             {
 194                 LogPedantic("Type: " << nodes->nodeTab[i]->type);
 195                 if (nodes->nodeTab[i]->type == XML_ATTRIBUTE_NODE) {
 196                     xmlNodePtr curNode = nodes->nodeTab[i];
 197                     result = getXMLNode(curNode);
 198                     LogPedantic("Result: " << result);
 199                     normalizeXMLTag(result, testsuite);
 200                     casesMap.insert(make_pair(result, false));
 201                 }
 202             }
 203             //Cleanup of XPath data
 204             xmlXPathFreeObject(xpathObject);
 205             xmlXPathFreeContext(xpathCtx);
 206             xmlFreeDoc(doc);
 207         }
 208     }
 209     Catch(XMLError)
 210     {
 211         LogError("Libxml error: " << _rethrown_exception.DumpToString());
 212         success = false;
 213     }
 214     xmlCleanupParser();
 215
 216     if(!filterByXML(casesMap))
 217     {
 218         success = false;
 219     }
 220
 221     return success;
 222 }
 223
 224 bool TestRunner::filterByXML(std::map<std::string, bool> & casesMap)
 225 {
 226     FOREACH(group, m_testGroups) {
 227         TestCaseStructList newList;
 228         FOREACH(iterator, group->second)
 229         {
 230             if (casesMap.find(iterator->name) != casesMap.end()) {
 231                 casesMap[iterator->name] = true;
 232                 newList.push_back(*iterator);
 233             }
 234         }
 235         group->second = newList;
 236     }
 237     FOREACH(cs, casesMap)
 238     {
 239         if(cs->second == false)
 240         {
 241             LogError("Cannot find testcase from XML file: " << cs->first);
 242             return false;
 243         }
 244     }
 245     return true;
 246 }
 247
 248 TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase)
 249 {
 250     try {
 251         testCase.proc();
 252     } catch (const TestFailed &e) {
 253         // Simple test failure
 254         CollectResult(testCase.name,
 255                       "",
 256                       TestResultsCollectorBase::FailStatus::FAILED,
 257                       e.GetMessage());
 258         return FAILED;
 259     } catch (const Ignored &e) {
 260         if (m_runIgnored) {
 261             // Simple test have to be implemented
 262             CollectResult(testCase.name,
 263                           "",
 264                           TestResultsCollectorBase::FailStatus::IGNORED,
 265                           e.GetMessage());
 266         }
 267
 268         return IGNORED;
 269     } catch (const DPL::Exception &e) {
 270         // DPL exception failure
 271         CollectResult(testCase.name,
 272                       "",
 273                       TestResultsCollectorBase::FailStatus::INTERNAL,
 274                       "DPL exception:" + e.GetMessage() + "\n" + e.DumpToString());
 275
 276         return FAILED;
 277     } catch (const std::exception &) {
 278         // std exception failure
 279         CollectResult(testCase.name,
 280                       "",
 281                       TestResultsCollectorBase::FailStatus::INTERNAL,
 282                       "std exception");
 283
 284         return FAILED;
 285     } catch (...) {
 286         // Unknown exception failure
 287         CollectResult(testCase.name,
 288                       "",
 289                       TestResultsCollectorBase::FailStatus::INTERNAL,
 290                       "unknown exception");
 291
 292         return FAILED;
 293     }
 294
 295     CollectResult(testCase.name,
 296                   "",
 297                   TestResultsCollectorBase::FailStatus::NONE);
 298
 299     // Everything OK
 300     return PASS;
 301 }
 302
 303 void TestRunner::RunTests()
 304 {
 305     using namespace DPL::Colors::Text;
 306
 307     Banner();
 308
 309     unsigned count = 0;
 310     FOREACH(group, m_testGroups) {
 311         count += group->second.size();
 312     }
 313
 314     std::for_each(m_collectors.begin(),
 315                   m_collectors.end(),
 316                   [count] (const TestResultsCollectors::value_type & collector)
 317                   {
 318                       collector.second->Start(count);
 319                   });
 320
 321     fprintf(stderr, "%sFound %d testcases...%s\n", GREEN_BEGIN, count, GREEN_END);
 322     fprintf(stderr, "%s%s%s\n", GREEN_BEGIN, "Running tests...", GREEN_END);
 323     FOREACH(group, m_testGroups) {
 324         TestCaseStructList list = group->second;
 325         if (!list.empty()) {
 326             std::for_each(
 327                 m_collectors.begin(),
 328                 m_collectors.end(),
 329                 [&group](const TestResultsCollectors::value_type & collector)
 330                 {
 331                     collector.second->
 332                         CollectCurrentTestGroupName(group->first);
 333                 });
 334             list.sort();
 335
 336             for (TestCaseStructList::const_iterator iterator = list.begin();
 337                  iterator != list.end();
 338                  ++iterator)
 339             {
 340                 TestCaseStruct test = *iterator;
 341                 if (m_startTestId == test.name) {
 342                     m_startTestId = "";
 343                 }
 344
 345                 if (m_startTestId.empty()) {
 346                     RunTestCase(test);
 347                 }
 348                 if (m_terminate == true) {
 349                     // Terminate quietly without any logs
 350                     return;
 351                 }
 352             }
 353         }
 354     }
 355
 356     std::for_each(m_collectors.begin(),
 357                   m_collectors.end(),
 358                   [] (const TestResultsCollectors::value_type & collector)
 359                   {
 360                       collector.second->Finish();
 361                   });
 362
 363     // Finished
 364     fprintf(stderr, "%s%s%s\n\n", GREEN_BEGIN, "Finished", GREEN_END);
 365 }
 366
 367 void TestRunner::CollectResult(
 368     const std::string& id,
 369     const std::string& description,
 370     const TestResultsCollectorBase::FailStatus::Type status,
 371     const std::string& reason)
 372 {
 373     std::for_each(m_collectors.begin(),
 374                   m_collectors.end(),
 375                   [&](const TestResultsCollectors::value_type & collector)
 376                   {
 377                       collector.second->CollectResult(id,
 378                                                       description,
 379                                                       status,
 380                                                       reason);
 381                   });
 382 }
 383
 384 void TestRunner::Banner()
 385 {
 386     using namespace DPL::Colors::Text;
 387     fprintf(stderr,
 388             "%s%s%s\n",
 389             BOLD_GREEN_BEGIN,
 390             "DPL tests runner",
 391             BOLD_GREEN_END);
 392     fprintf(stderr,
 393             "%s%s%s%s\n\n",
 394             GREEN_BEGIN,
 395             "Build: ",
 396             __TIMESTAMP__,
 397             GREEN_END);
 398 }
 399
 400 void TestRunner::InvalidArgs(const std::string& message)
 401 {
 402     using namespace DPL::Colors::Text;
 403     fprintf(stderr,
 404             "%s%s%s\n",
 405             BOLD_RED_BEGIN,
 406             message.c_str(),
 407             BOLD_RED_END);
 408 }
 409
 410 void TestRunner::Usage()
 411 {
 412     fprintf(stderr, "Usage: runner [options]\n\n");
 413     fprintf(stderr, "Output type:\n");
 414     fprintf(stderr, "  --output=<output type> --output=<output type> ...\n");
 415     fprintf(stderr, "\n  possible output types:\n");
 416     FOREACH(type, TestResultsCollectorBase::GetCollectorsNames()) {
 417         fprintf(stderr, "    --output=%s\n", type->c_str());
 418     }
 419     fprintf(stderr, "\n  example:\n");
 420     fprintf(stderr,
 421             "    test-binary --output=text --output=xml --file=output.xml\n\n");
 422     fprintf(stderr, "Other parameters:\n");
 423     fprintf(stderr,
 424             "  --regexp='regexp'\t Only selected tests"
 425             " which names match regexp run\n\n");
 426     fprintf(stderr, "  --start=<test id>\tStart from concrete test id");
 427     fprintf(stderr, "  --group=<group name>\t Run tests only from one group\n");
 428     fprintf(stderr, "  --runignored\t Run also ignored tests\n");
 429     fprintf(stderr, "  --list\t Show a list of Test IDs\n");
 430     fprintf(stderr, "  --listgroups\t Show a list of Test Group names \n");
 431     fprintf(stderr, "  --only-from-xml=<xml file>\t Run only testcases specified in XML file \n"
 432                     "       XML name is taken from attribute id=\"part1_part2\" as whole.\n"
 433                     "       If part1 is not found (no _) then it is implicitily "
 434                            "set according to suite part1 from binary tests\n");
 435     fprintf(
 436         stderr,
 437         "  --listingroup=<group name>\t Show a list of Test IDS in one group\n");
 438     fprintf(stderr, "  --allowchildlogs\t Allow to print logs from child process on screen.\n");
 439     fprintf(stderr, "       When active child process will be able to print logs on stdout and stderr.\n");
 440     fprintf(stderr, "       Both descriptors will be closed after test.\n");
 441     fprintf(stderr, "  --help\t This help\n\n");
 442     std::for_each(m_collectors.begin(),
 443                   m_collectors.end(),
 444                   [] (const TestResultsCollectors::value_type & collector)
 445                   {
 446                       fprintf(stderr,
 447                               "Output %s has specific args:\n",
 448                               collector.first.c_str());
 449                       fprintf(stderr,
 450                               "%s\n",
 451                               collector.second->
 452                                   CollectorSpecificHelp().c_str());
 453                   });
 454     fprintf(stderr, "For bug reporting, please write to:\n");
 455     fprintf(stderr, "<p.dobrowolsk@samsung.com>\n");
 456 }
 457
 458 int TestRunner::ExecTestRunner(int argc, char *argv[])
 459 {
 460     std::vector<std::string> args;
 461     for (int i = 0; i < argc; ++i) {
 462         args.push_back(argv[i]);
 463     }
 464     return ExecTestRunner(args);
 465 }
 466
 467 void TestRunner::MarkAssertion()
 468 {
 469     ++m_totalAssertions;
 470 }
 471
 472 int TestRunner::ExecTestRunner(const ArgsList& value)
 473 {
 474     m_runIgnored = false;
 475     ArgsList args = value;
 476     // Parse command line
 477     if (args.size() == 1) {
 478         InvalidArgs();
 479         Usage();
 480         return -1;
 481     }
 482
 483     args.erase(args.begin());
 484
 485     bool showHelp = false;
 486     bool justList = false;
 487     std::vector<std::string> xmlFiles;
 488
 489     TestResultsCollectorBasePtr currentCollector;
 490
 491     // Parse each argument
 492     FOREACH(it, args)
 493     {
 494         std::string arg = *it;
 495         const std::string regexp = "--regexp=";
 496         const std::string output = "--output=";
 497         const std::string groupId = "--group=";
 498         const std::string runIgnored = "--runignored";
 499         const std::string listCmd = "--list";
 500         const std::string startCmd = "--start=";
 501         const std::string listGroupsCmd = "--listgroups";
 502         const std::string listInGroup = "--listingroup=";
 503         const std::string allowChildLogs = "--allowchildlogs";
 504         const std::string onlyFromXML = "--only-from-xml=";
 505
 506         if (currentCollector) {
 507             if (currentCollector->ParseCollectorSpecificArg(arg)) {
 508                 continue;
 509             }
 510         }
 511
 512         if (arg.find(startCmd) == 0) {
 513             arg.erase(0, startCmd.length());
 514             FOREACH(group, m_testGroups) {
 515                 FOREACH(tc, group->second) {
 516                     if (tc->name == arg) {
 517                         m_startTestId = arg;
 518                         break;
 519                     }
 520                 }
 521                 if (!m_startTestId.empty()) {
 522                     break;
 523                 }
 524             }
 525             if (!m_startTestId.empty()) {
 526                 continue;
 527             }
 528             InvalidArgs();
 529             fprintf(stderr, "Start test id has not been found\n");
 530             Usage();
 531             return 0;
 532         } else if (arg.find(groupId) == 0) {
 533             arg.erase(0, groupId.length());
 534             TestCaseGroupMap::iterator found = m_testGroups.find(arg);
 535             if (found != m_testGroups.end()) {
 536                 std::string name = found->first;
 537                 TestCaseStructList newList = found->second;
 538                 m_testGroups.clear();
 539                 m_testGroups[name] = newList;
 540             } else {
 541                 fprintf(stderr, "Group %s not found\n", arg.c_str());
 542                 InvalidArgs();
 543                 Usage();
 544                 return -1;
 545             }
 546         } else if (arg == runIgnored) {
 547             m_runIgnored = true;
 548         } else if (arg == listCmd) {
 549             justList = true;
 550         } else if (arg == listGroupsCmd) {
 551             FOREACH(group, m_testGroups) {
 552                 printf("GR:%s\n", group->first.c_str());
 553             }
 554             return 0;
 555         } else if (arg.find(listInGroup) == 0) {
 556             arg.erase(0, listInGroup.length());
 557             FOREACH(test, m_testGroups[arg]) {
 558                 printf("ID:%s\n", test->name.c_str());
 559             }
 560             return 0;
 561         } else if (arg.find(allowChildLogs) == 0) {
 562             arg.erase(0, allowChildLogs.length());
 563             m_allowChildLogs = true;
 564         } else if (arg == "--help") {
 565             showHelp = true;
 566         } else if (arg.find(output) == 0) {
 567             arg.erase(0, output.length());
 568             if (m_collectors.find(arg) != m_collectors.end()) {
 569                 InvalidArgs(
 570                     "Multiple outputs of the same type are not supported!");
 571                 Usage();
 572                 return -1;
 573             }
 574             currentCollector.reset(TestResultsCollectorBase::Create(arg));
 575             if (!currentCollector) {
 576                 InvalidArgs("Unsupported output type!");
 577                 Usage();
 578                 return -1;
 579             }
 580             m_collectors[arg] = currentCollector;
 581         } else if (arg.find(regexp) == 0) {
 582             arg.erase(0, regexp.length());
 583             if (arg.length() == 0) {
 584                 InvalidArgs();
 585                 Usage();
 586                 return -1;
 587             }
 588
 589             if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
 590                 arg.erase(0);
 591                 arg.erase(arg.length() - 1);
 592             }
 593
 594             if (arg.length() == 0) {
 595                 InvalidArgs();
 596                 Usage();
 597                 return -1;
 598             }
 599
 600             pcrecpp::RE re(arg.c_str());
 601             FOREACH(group, m_testGroups) {
 602                 TestCaseStructList newList;
 603                 FOREACH(iterator, group->second)
 604                 {
 605                     if (re.PartialMatch(iterator->name)) {
 606                         newList.push_back(*iterator);
 607                     }
 608                 }
 609                 group->second = newList;
 610             }
 611         } else if(arg.find(onlyFromXML) == 0) {
 612             arg.erase(0, onlyFromXML.length());
 613             if (arg.length() == 0) {
 614                 InvalidArgs();
 615                 Usage();
 616                 return -1;
 617             }
 618
 619             if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
 620                 arg.erase(0);
 621                 arg.erase(arg.length() - 1);
 622             }
 623
 624             if (arg.length() == 0) {
 625                 InvalidArgs();
 626                 Usage();
 627                 return -1;
 628             }
 629
 630             xmlFiles.push_back(arg);
 631         } else {
 632             InvalidArgs();
 633             Usage();
 634             return -1;
 635         }
 636     }
 637
 638     if(!xmlFiles.empty())
 639     {
 640         if(!filterGroupsByXmls(xmlFiles))
 641         {
 642             fprintf(stderr, "XML file is not correct\n");
 643             return 0;
 644         }
 645     }
 646
 647     if(justList)
 648     {
 649         FOREACH(group, m_testGroups) {
 650             FOREACH(test, group->second) {
 651                 printf("ID:%s:%s\n", group->first.c_str(), test->name.c_str());
 652             }
 653         }
 654         return 0;
 655     }
 656
 657     currentCollector.reset();
 658
 659     // Show help
 660     if (showHelp) {
 661         Usage();
 662         return 0;
 663     }
 664
 665     if (m_collectors.empty()) {
 666         TestResultsCollectorBasePtr collector(
 667             TestResultsCollectorBase::Create("text"));
 668         m_collectors["text"] = collector;
 669     }
 670
 671     for (auto it = m_collectors.begin(); it != m_collectors.end(); ++it) {
 672         if (!it->second->Configure()) {
 673             fprintf(stderr, "Could not configure selected output");
 674             return 0;
 675         }
 676     }
 677
 678     // Run tests
 679     RunTests();
 680
 681     return 0;
 682 }
 683
 684 bool TestRunner::getRunIgnored() const
 685 {
 686     return m_runIgnored;
 687 }
 688
 689 void TestRunner::Terminate()
 690 {
 691     m_terminate = true;
 692 }
 693
 694 bool TestRunner::GetAllowChildLogs()
 695 {
 696     return m_allowChildLogs;
 697 }
 698
 699 }
 700 } // namespace DPL