modules/test/src/test_runner.cpp

   1 /*
   2  * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.cpp
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @version     1.0
  21  * @brief       This file is the implementation file of test runner
  22  */
  23 #include <dpl/test/test_runner.h>
  24 #include <dpl/test/test_results_collector.h>
  25 #include <dpl/exception.h>
  26 #include <dpl/scoped_free.h>
  27 #include <dpl/foreach.h>
  28 #include <dpl/log/log.h>
  29 #include <dpl/colors.h>
  30 #include <pcrecpp.h>
  31 #include <algorithm>
  32 #include <cstdio>
  33 #include <memory.h>
  34 #include <libgen.h>
  35 #include <cstring>
  36 #include <cstdlib>
  37 #include <dpl/utils/wrt_global_settings.h>
  38
  39 #include <dpl/singleton_impl.h>
  40 IMPLEMENT_SINGLETON(DPL::Test::TestRunner)
  41
  42 namespace DPL
  43 {
  44 namespace Test
  45 {
  46
  47 namespace // anonymous
  48 {
  49
  50 std::string BaseName(std::string aPath)
  51 {
  52     ScopedFree<char> path(strdup(aPath.c_str()));
  53     if (NULL == path.Get())
  54     {
  55        throw std::bad_alloc();
  56     }
  57     char* baseName = basename(path.Get());
  58     std::string retValue = baseName;
  59     return retValue;
  60 }
  61
  62 } // namespace anonymous
  63
  64 //! \brief Failed test message creator
  65 //!
  66 //! \param[in] aTest string for tested expression
  67 //! \param[in] aFile source file name
  68 //! \param[in] aLine source file line
  69 //! \param[in] aMessage error message
  70 TestRunner::TestFailed::TestFailed(const char* aTest,
  71                                    const char* aFile,
  72                                    int aLine,
  73                                    const std::string &aMessage)
  74 {
  75     std::ostringstream assertMsg;
  76     assertMsg << "[" << BaseName(aFile) << ":" << aLine
  77               << "] Assertion failed ("
  78               << aTest << ") " << aMessage;
  79     m_message = assertMsg.str();
  80 }
  81
  82 void TestRunner::RegisterTest(const char *testName, TestCase proc)
  83 {
  84     m_testGroups[m_currentGroup].push_back(TestCaseStruct(testName, proc));
  85 }
  86
  87 void TestRunner::InitGroup(const char* name)
  88 {
  89     m_currentGroup = name;
  90 }
  91
  92
  93 TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase)
  94 {
  95     try
  96     {
  97         testCase.proc();
  98     }
  99     catch (const TestFailed &e)
 100     {
 101         // Simple test failure
 102         CollectResult(testCase.name,
 103                       "",
 104                       TestResultsCollectorBase::FailStatus::FAILED,
 105                       e.GetMessage());
 106         return FAILED;
 107     }
 108     catch (const Ignored &e)
 109     {
 110         if (m_runIgnored) {
 111             // Simple test have to be implemented
 112             CollectResult(testCase.name,
 113                           "",
 114                           TestResultsCollectorBase::FailStatus::IGNORED,
 115                           e.GetMessage());
 116         }
 117
 118         return IGNORED;
 119     }
 120     catch (const DPL::Exception &e)
 121     {
 122         // DPL exception failure
 123         CollectResult(testCase.name,
 124                       "",
 125                       TestResultsCollectorBase::FailStatus::INTERNAL,
 126                       "DPL exception:" + e.GetMessage());
 127
 128         return FAILED;
 129     }
 130     catch (const std::exception &)
 131     {
 132         // std exception failure
 133         CollectResult(testCase.name,
 134                       "",
 135                       TestResultsCollectorBase::FailStatus::INTERNAL,
 136                       "std exception");
 137
 138         return FAILED;
 139     }
 140     catch (...)
 141     {
 142         // Unknown exception failure
 143         CollectResult(testCase.name,
 144                       "",
 145                       TestResultsCollectorBase::FailStatus::INTERNAL,
 146                       "unknown exception");
 147
 148         return FAILED;
 149     }
 150
 151     CollectResult(testCase.name,
 152                   "",
 153                   TestResultsCollectorBase::FailStatus::NONE);
 154
 155     // Everything OK
 156     return PASS;
 157 }
 158
 159 void TestRunner::RunTests()
 160 {
 161     using namespace DPL::Colors::Text;
 162
 163     Banner();
 164     std::for_each(m_collectors.begin(),
 165                   m_collectors.end(),
 166                   [](const TestResultsCollectors::value_type& collector)
 167                   {
 168                       collector.second->Start();
 169                   });
 170
 171     fprintf(stderr, "%s%s%s\n", GREEN_BEGIN, "Running tests...", GREEN_END);
 172     FOREACH(group, m_testGroups) {
 173         TestCaseStructList list = group->second;
 174         if (!list.empty()) {
 175             std::for_each(
 176                     m_collectors.begin(),
 177                     m_collectors.end(),
 178                     [&group](const TestResultsCollectors::value_type& collector)
 179                     {
 180                         collector.second->
 181                                 CollectCurrentTestGroupName(group->first);
 182                     });
 183             list.sort();
 184
 185             for (TestCaseStructList::const_iterator iterator = list.begin();
 186                  iterator != list.end();
 187                  ++iterator)
 188             {
 189                 TestCaseStruct test = *iterator;
 190                 if (m_startTestId == test.name)
 191                     m_startTestId = "";
 192
 193                 if (m_startTestId.empty()) {
 194                     RunTestCase(test);
 195                 }
 196             }
 197         }
 198     }
 199
 200     std::for_each(m_collectors.begin(),
 201                   m_collectors.end(),
 202                   [](const TestResultsCollectors::value_type& collector)
 203                   {
 204                       collector.second->Finish();
 205                   });
 206
 207     // Finished
 208     fprintf(stderr, "%s%s%s\n\n", GREEN_BEGIN, "Finished", GREEN_END);
 209 }
 210
 211 void TestRunner::CollectResult(
 212         const std::string& id,
 213         const std::string& description,
 214         const TestResultsCollectorBase::FailStatus::Type status,
 215         const std::string& reason)
 216 {
 217     std::for_each(m_collectors.begin(),
 218                   m_collectors.end(),
 219                   [&](const TestResultsCollectors::value_type& collector)
 220                   {
 221                       collector.second->CollectResult(id,
 222                                                       description,
 223                                                       status,
 224                                                       reason);
 225                   });
 226 }
 227
 228 void TestRunner::Banner()
 229 {
 230     using namespace DPL::Colors::Text;
 231     fprintf(stderr,
 232             "%s%s%s\n",
 233             BOLD_GREEN_BEGIN,
 234             "DPL tests runner",
 235             BOLD_GREEN_END);
 236     fprintf(stderr,
 237             "%s%s%s%s\n\n",
 238             GREEN_BEGIN,
 239             "Build: ",
 240             __TIMESTAMP__,
 241             GREEN_END);
 242 }
 243
 244 void TestRunner::InvalidArgs(const std::string& message)
 245 {
 246     using namespace DPL::Colors::Text;
 247     fprintf(stderr,
 248             "%s%s%s\n",
 249             BOLD_RED_BEGIN,
 250             message.c_str(),
 251             BOLD_RED_END);
 252 }
 253
 254 void TestRunner::Usage()
 255 {
 256     fprintf(stderr, "Usage: runner [options]\n\n");
 257     fprintf(stderr, "Output type:\n");
 258     fprintf(stderr, "  --output=<output type> --output=<output type> ...\n");
 259     fprintf(stderr, "\n  possible output types:\n");
 260     FOREACH (type, TestResultsCollectorBase::GetCollectorsNames()) {
 261         fprintf(stderr, "    --output=%s\n", type->c_str());
 262     }
 263     fprintf(stderr, "\n  example:\n");
 264     fprintf(stderr, "    test-binary --output=text --output=xml --file=output.xml\n\n");
 265     fprintf(stderr, "Other parameters:\n");
 266     fprintf(stderr,
 267             "  --regexp='regexp'\t Only selected tests"
 268             " which names match regexp run\n\n");
 269     fprintf(stderr, "  --start=<test id>\tStart from concrete test id");
 270     fprintf(stderr, "  --group=<group name>\t Run tests only from one group\n");
 271     fprintf(stderr, "  --runignored\t Run also ignored tests\n");
 272     fprintf(stderr, "  --list\t Show a list of Test IDs\n");
 273     fprintf(stderr, "  --listgroups\t Show a list of Test Group names \n");
 274     fprintf(stderr, "  --listingroup=<group name>\t Show a list of Test IDS in one group\n");
 275     fprintf(stderr, "  --help\t This help\n\n");
 276     std::for_each(m_collectors.begin(),
 277                   m_collectors.end(),
 278                   [](const TestResultsCollectors::value_type& collector)
 279                   {
 280                       fprintf(stderr,
 281                               "Output %s has specific args:\n",
 282                               collector.first.c_str());
 283                       fprintf(stderr,
 284                               "%s\n",
 285                               collector.second->
 286                                       CollectorSpecificHelp().c_str());
 287                   });
 288     fprintf(stderr, "For bug reporting, please write to:\n");
 289     fprintf(stderr, "<p.dobrowolsk@samsung.com>\n");
 290 }
 291
 292 int TestRunner::ExecTestRunner(int argc, char *argv[])
 293 {
 294     std::vector<std::string> args;
 295     for (int i = 0; i < argc; ++i)
 296     {
 297         args.push_back(argv[i]);
 298     }
 299     return ExecTestRunner(args);
 300 }
 301
 302 void TestRunner::MarkAssertion()
 303 {
 304     ++m_totalAssertions;
 305 }
 306
 307 int TestRunner::ExecTestRunner(const ArgsList& value)
 308 {
 309     m_runIgnored = false;
 310     ArgsList args = value;
 311     // Parse command line
 312     if (args.size() == 1)
 313     {
 314         InvalidArgs();
 315         Usage();
 316         return -1;
 317     }
 318
 319     args.erase(args.begin());
 320
 321     bool showHelp = false;
 322
 323     TestResultsCollectorBasePtr currentCollector;
 324
 325     // Parse each argument
 326     FOREACH(it, args)
 327     {
 328         std::string arg = *it;
 329         const std::string regexp = "--regexp=";
 330         const std::string output = "--output=";
 331         const std::string groupId = "--group=";
 332         const std::string runIgnored = "--runignored";
 333         const std::string listCmd = "--list";
 334         const std::string startCmd = "--start=";
 335         const std::string listGroupsCmd = "--listgroups";
 336         const std::string listInGroup = "--listingroup=";
 337
 338         if (currentCollector)
 339         {
 340             if (currentCollector->ParseCollectorSpecificArg(arg))
 341             {
 342                 continue;
 343             }
 344         }
 345
 346         if (arg.find(startCmd) == 0)
 347         {
 348             arg.erase(0, startCmd.length());
 349             FOREACH(group, m_testGroups) {
 350                 FOREACH(tc, group->second) {
 351                     if (tc->name == arg) {
 352                         m_startTestId = arg;
 353                         break;
 354                     }
 355                 }
 356                 if (!m_startTestId.empty()) break;
 357             }
 358             if (!m_startTestId.empty()) continue;
 359             InvalidArgs();
 360             fprintf(stderr, "Start test id has not been found\n");
 361             Usage();
 362             return 0;
 363         }
 364         else if (arg.find(groupId) == 0)
 365         {
 366             arg.erase(0, groupId.length());
 367             TestCaseGroupMap::iterator found = m_testGroups.find(arg);
 368             if (found != m_testGroups.end()) {
 369                 std::string name = found->first;
 370                 TestCaseStructList newList = found->second;
 371                 m_testGroups.clear();
 372                 m_testGroups[name] = newList;
 373             } else {
 374                 fprintf(stderr, "Group %s not found\n", arg.c_str());
 375                 InvalidArgs();
 376                 Usage();
 377                 return -1;
 378             }
 379         }
 380         else if (arg == runIgnored)
 381         {
 382             m_runIgnored = true;
 383         }
 384         else if (arg == listCmd)
 385         {
 386             FOREACH(group, m_testGroups) {
 387                 FOREACH(test, group->second) {
 388                     printf("ID:%s:%s\n", group->first.c_str(), test->name.c_str());
 389                 }
 390             }
 391             return 0;
 392         }
 393         else if (arg == listGroupsCmd)
 394         {
 395             FOREACH(group, m_testGroups) {
 396                 printf("GR:%s\n", group->first.c_str());
 397             }
 398             return 0;
 399         }
 400         else if (arg.find(listInGroup) == 0)
 401         {
 402             arg.erase(0, listInGroup.length());
 403             FOREACH(test, m_testGroups[arg]) {
 404                 printf("ID:%s\n", test->name.c_str());
 405             }
 406             return 0;
 407         }
 408         else if (arg == "--help")
 409             showHelp = true;
 410         else if (arg.find(output) == 0)
 411         {
 412             arg.erase(0, output.length());
 413             if (m_collectors.find(arg) != m_collectors.end())
 414             {
 415                 InvalidArgs("Multiple outputs of the same type are not supported!");
 416                 Usage();
 417                 return -1;
 418             }
 419             currentCollector.Reset(TestResultsCollectorBase::Create(arg));
 420             if (!currentCollector) {
 421                 InvalidArgs("Unsupported output type!");
 422                 Usage();
 423                 return -1;
 424             }
 425             m_collectors[arg] = currentCollector;
 426         }
 427         else if (arg.find(regexp) == 0)
 428         {
 429             arg.erase(0, regexp.length());
 430             if (arg.length() == 0)
 431             {
 432                 InvalidArgs();
 433                 Usage();
 434                 return -1;
 435             }
 436
 437             if (arg[0] == '\'' && arg[arg.length() - 1] == '\'')
 438             {
 439                 arg.erase(0);
 440                 arg.erase(arg.length() - 1);
 441             }
 442
 443             if (arg.length() == 0)
 444             {
 445                 InvalidArgs();
 446                 Usage();
 447                 return -1;
 448             }
 449
 450             pcrecpp::RE re(arg.c_str());
 451             FOREACH(group, m_testGroups) {
 452                 TestCaseStructList newList;
 453                 FOREACH(iterator, group->second)
 454                 {
 455                     if (re.PartialMatch(iterator->name))
 456                     {
 457                         newList.push_back(*iterator);
 458                     }
 459                 }
 460                 group->second = newList;
 461             }
 462         }
 463         else
 464         {
 465             InvalidArgs();
 466             Usage();
 467             return -1;
 468         }
 469     }
 470
 471     currentCollector.Reset();
 472
 473     // Show help
 474     if (showHelp)
 475     {
 476         Usage();
 477         return 0;
 478     }
 479
 480     if (m_collectors.empty())
 481     {
 482         TestResultsCollectorBasePtr collector(
 483                 TestResultsCollectorBase::Create("text"));
 484         m_collectors["text"] = collector;
 485     }
 486
 487     for (auto it = m_collectors.begin(); it != m_collectors.end(); ++it)
 488     {
 489         if (!it->second->Configure())
 490         {
 491             fprintf(stderr, "Could not configure selected output");
 492             return 0;
 493         }
 494     }
 495
 496     // Run tests
 497     RunTests();
 498
 499     return 0;
 500 }
 501
 502 bool TestRunner::getRunIgnored() const
 503 {
 504     return m_runIgnored;
 505 }
 506
 507 }
 508 } // namespace DPL