modules/test/src/test_runner.cpp

   1 /*
   2  * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.cpp
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @version     1.0
  21  * @brief       This file is the implementation file of test runner
  22  */
  23 #include <stddef.h>
  24 #include <dpl/test/test_runner.h>
  25 #include <dpl/test/test_results_collector.h>
  26 #include <dpl/exception.h>
  27 #include <dpl/scoped_free.h>
  28 #include <dpl/foreach.h>
  29 #include <dpl/log/log.h>
  30 #include <dpl/colors.h>
  31 #include <pcrecpp.h>
  32 #include <algorithm>
  33 #include <cstdio>
  34 #include <memory.h>
  35 #include <libgen.h>
  36 #include <cstring>
  37 #include <cstdlib>
  38 #include <dpl/utils/wrt_global_settings.h>
  39
  40 #include <dpl/singleton_impl.h>
  41 IMPLEMENT_SINGLETON(DPL::Test::TestRunner)
  42
  43 namespace DPL
  44 {
  45 namespace Test
  46 {
  47
  48 namespace // anonymous
  49 {
  50
  51 std::string BaseName(std::string aPath)
  52 {
  53     ScopedFree<char> path(strdup(aPath.c_str()));
  54     if (NULL == path.Get())
  55     {
  56        throw std::bad_alloc();
  57     }
  58     char* baseName = basename(path.Get());
  59     std::string retValue = baseName;
  60     return retValue;
  61 }
  62
  63 } // namespace anonymous
  64
  65 //! \brief Failed test message creator
  66 //!
  67 //! \param[in] aTest string for tested expression
  68 //! \param[in] aFile source file name
  69 //! \param[in] aLine source file line
  70 //! \param[in] aMessage error message
  71 TestRunner::TestFailed::TestFailed(const char* aTest,
  72                                    const char* aFile,
  73                                    int aLine,
  74                                    const std::string &aMessage)
  75 {
  76     std::ostringstream assertMsg;
  77     assertMsg << "[" << BaseName(aFile) << ":" << aLine
  78               << "] Assertion failed ("
  79               << aTest << ") " << aMessage;
  80     m_message = assertMsg.str();
  81 }
  82
  83 TestRunner::TestFailed::TestFailed(const std::string &message)
  84 {
  85     m_message = message;
  86 }
  87
  88 void TestRunner::RegisterTest(const char *testName, TestCase proc)
  89 {
  90     m_testGroups[m_currentGroup].push_back(TestCaseStruct(testName, proc));
  91 }
  92
  93 void TestRunner::InitGroup(const char* name)
  94 {
  95     m_currentGroup = name;
  96 }
  97
  98
  99 TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase)
 100 {
 101     try
 102     {
 103         testCase.proc();
 104     }
 105     catch (const TestFailed &e)
 106     {
 107         // Simple test failure
 108         CollectResult(testCase.name,
 109                       "",
 110                       TestResultsCollectorBase::FailStatus::FAILED,
 111                       e.GetMessage());
 112         return FAILED;
 113     }
 114     catch (const Ignored &e)
 115     {
 116         if (m_runIgnored) {
 117             // Simple test have to be implemented
 118             CollectResult(testCase.name,
 119                           "",
 120                           TestResultsCollectorBase::FailStatus::IGNORED,
 121                           e.GetMessage());
 122         }
 123
 124         return IGNORED;
 125     }
 126     catch (const DPL::Exception &e)
 127     {
 128         // DPL exception failure
 129         CollectResult(testCase.name,
 130                       "",
 131                       TestResultsCollectorBase::FailStatus::INTERNAL,
 132                       "DPL exception:" + e.GetMessage());
 133
 134         return FAILED;
 135     }
 136     catch (const std::exception &)
 137     {
 138         // std exception failure
 139         CollectResult(testCase.name,
 140                       "",
 141                       TestResultsCollectorBase::FailStatus::INTERNAL,
 142                       "std exception");
 143
 144         return FAILED;
 145     }
 146     catch (...)
 147     {
 148         // Unknown exception failure
 149         CollectResult(testCase.name,
 150                       "",
 151                       TestResultsCollectorBase::FailStatus::INTERNAL,
 152                       "unknown exception");
 153
 154         return FAILED;
 155     }
 156
 157     CollectResult(testCase.name,
 158                   "",
 159                   TestResultsCollectorBase::FailStatus::NONE);
 160
 161     // Everything OK
 162     return PASS;
 163 }
 164
 165 void TestRunner::RunTests()
 166 {
 167     using namespace DPL::Colors::Text;
 168
 169     Banner();
 170     std::for_each(m_collectors.begin(),
 171                   m_collectors.end(),
 172                   [](const TestResultsCollectors::value_type& collector)
 173                   {
 174                       collector.second->Start();
 175                   });
 176
 177     fprintf(stderr, "%s%s%s\n", GREEN_BEGIN, "Running tests...", GREEN_END);
 178     FOREACH(group, m_testGroups) {
 179         TestCaseStructList list = group->second;
 180         if (!list.empty()) {
 181             std::for_each(
 182                     m_collectors.begin(),
 183                     m_collectors.end(),
 184                     [&group](const TestResultsCollectors::value_type& collector)
 185                     {
 186                         collector.second->
 187                                 CollectCurrentTestGroupName(group->first);
 188                     });
 189             list.sort();
 190
 191             for (TestCaseStructList::const_iterator iterator = list.begin();
 192                  iterator != list.end();
 193                  ++iterator)
 194             {
 195                 TestCaseStruct test = *iterator;
 196                 if (m_startTestId == test.name)
 197                     m_startTestId = "";
 198
 199                 if (m_startTestId.empty()) {
 200                     RunTestCase(test);
 201                 }
 202                 if (m_terminate == true) {
 203                     // Terminate quietly without any logs
 204                     return;
 205                 }
 206             }
 207         }
 208     }
 209
 210     std::for_each(m_collectors.begin(),
 211                   m_collectors.end(),
 212                   [](const TestResultsCollectors::value_type& collector)
 213                   {
 214                       collector.second->Finish();
 215                   });
 216
 217     // Finished
 218     fprintf(stderr, "%s%s%s\n\n", GREEN_BEGIN, "Finished", GREEN_END);
 219 }
 220
 221 void TestRunner::CollectResult(
 222         const std::string& id,
 223         const std::string& description,
 224         const TestResultsCollectorBase::FailStatus::Type status,
 225         const std::string& reason)
 226 {
 227     std::for_each(m_collectors.begin(),
 228                   m_collectors.end(),
 229                   [&](const TestResultsCollectors::value_type& collector)
 230                   {
 231                       collector.second->CollectResult(id,
 232                                                       description,
 233                                                       status,
 234                                                       reason);
 235                   });
 236 }
 237
 238 void TestRunner::Banner()
 239 {
 240     using namespace DPL::Colors::Text;
 241     fprintf(stderr,
 242             "%s%s%s\n",
 243             BOLD_GREEN_BEGIN,
 244             "DPL tests runner",
 245             BOLD_GREEN_END);
 246     fprintf(stderr,
 247             "%s%s%s%s\n\n",
 248             GREEN_BEGIN,
 249             "Build: ",
 250             __TIMESTAMP__,
 251             GREEN_END);
 252 }
 253
 254 void TestRunner::InvalidArgs(const std::string& message)
 255 {
 256     using namespace DPL::Colors::Text;
 257     fprintf(stderr,
 258             "%s%s%s\n",
 259             BOLD_RED_BEGIN,
 260             message.c_str(),
 261             BOLD_RED_END);
 262 }
 263
 264 void TestRunner::Usage()
 265 {
 266     fprintf(stderr, "Usage: runner [options]\n\n");
 267     fprintf(stderr, "Output type:\n");
 268     fprintf(stderr, "  --output=<output type> --output=<output type> ...\n");
 269     fprintf(stderr, "\n  possible output types:\n");
 270     FOREACH (type, TestResultsCollectorBase::GetCollectorsNames()) {
 271         fprintf(stderr, "    --output=%s\n", type->c_str());
 272     }
 273     fprintf(stderr, "\n  example:\n");
 274     fprintf(stderr, "    test-binary --output=text --output=xml --file=output.xml\n\n");
 275     fprintf(stderr, "Other parameters:\n");
 276     fprintf(stderr,
 277             "  --regexp='regexp'\t Only selected tests"
 278             " which names match regexp run\n\n");
 279     fprintf(stderr, "  --start=<test id>\tStart from concrete test id");
 280     fprintf(stderr, "  --group=<group name>\t Run tests only from one group\n");
 281     fprintf(stderr, "  --runignored\t Run also ignored tests\n");
 282     fprintf(stderr, "  --list\t Show a list of Test IDs\n");
 283     fprintf(stderr, "  --listgroups\t Show a list of Test Group names \n");
 284     fprintf(stderr, "  --listingroup=<group name>\t Show a list of Test IDS in one group\n");
 285     fprintf(stderr, "  --help\t This help\n\n");
 286     std::for_each(m_collectors.begin(),
 287                   m_collectors.end(),
 288                   [](const TestResultsCollectors::value_type& collector)
 289                   {
 290                       fprintf(stderr,
 291                               "Output %s has specific args:\n",
 292                               collector.first.c_str());
 293                       fprintf(stderr,
 294                               "%s\n",
 295                               collector.second->
 296                                       CollectorSpecificHelp().c_str());
 297                   });
 298     fprintf(stderr, "For bug reporting, please write to:\n");
 299     fprintf(stderr, "<p.dobrowolsk@samsung.com>\n");
 300 }
 301
 302 int TestRunner::ExecTestRunner(int argc, char *argv[])
 303 {
 304     std::vector<std::string> args;
 305     for (int i = 0; i < argc; ++i)
 306     {
 307         args.push_back(argv[i]);
 308     }
 309     return ExecTestRunner(args);
 310 }
 311
 312 void TestRunner::MarkAssertion()
 313 {
 314     ++m_totalAssertions;
 315 }
 316
 317 int TestRunner::ExecTestRunner(const ArgsList& value)
 318 {
 319     m_runIgnored = false;
 320     ArgsList args = value;
 321     // Parse command line
 322     if (args.size() == 1)
 323     {
 324         InvalidArgs();
 325         Usage();
 326         return -1;
 327     }
 328
 329     args.erase(args.begin());
 330
 331     bool showHelp = false;
 332
 333     TestResultsCollectorBasePtr currentCollector;
 334
 335     // Parse each argument
 336     FOREACH(it, args)
 337     {
 338         std::string arg = *it;
 339         const std::string regexp = "--regexp=";
 340         const std::string output = "--output=";
 341         const std::string groupId = "--group=";
 342         const std::string runIgnored = "--runignored";
 343         const std::string listCmd = "--list";
 344         const std::string startCmd = "--start=";
 345         const std::string listGroupsCmd = "--listgroups";
 346         const std::string listInGroup = "--listingroup=";
 347
 348         if (currentCollector)
 349         {
 350             if (currentCollector->ParseCollectorSpecificArg(arg))
 351             {
 352                 continue;
 353             }
 354         }
 355
 356         if (arg.find(startCmd) == 0)
 357         {
 358             arg.erase(0, startCmd.length());
 359             FOREACH(group, m_testGroups) {
 360                 FOREACH(tc, group->second) {
 361                     if (tc->name == arg) {
 362                         m_startTestId = arg;
 363                         break;
 364                     }
 365                 }
 366                 if (!m_startTestId.empty()) break;
 367             }
 368             if (!m_startTestId.empty()) continue;
 369             InvalidArgs();
 370             fprintf(stderr, "Start test id has not been found\n");
 371             Usage();
 372             return 0;
 373         }
 374         else if (arg.find(groupId) == 0)
 375         {
 376             arg.erase(0, groupId.length());
 377             TestCaseGroupMap::iterator found = m_testGroups.find(arg);
 378             if (found != m_testGroups.end()) {
 379                 std::string name = found->first;
 380                 TestCaseStructList newList = found->second;
 381                 m_testGroups.clear();
 382                 m_testGroups[name] = newList;
 383             } else {
 384                 fprintf(stderr, "Group %s not found\n", arg.c_str());
 385                 InvalidArgs();
 386                 Usage();
 387                 return -1;
 388             }
 389         }
 390         else if (arg == runIgnored)
 391         {
 392             m_runIgnored = true;
 393         }
 394         else if (arg == listCmd)
 395         {
 396             FOREACH(group, m_testGroups) {
 397                 FOREACH(test, group->second) {
 398                     printf("ID:%s:%s\n", group->first.c_str(), test->name.c_str());
 399                 }
 400             }
 401             return 0;
 402         }
 403         else if (arg == listGroupsCmd)
 404         {
 405             FOREACH(group, m_testGroups) {
 406                 printf("GR:%s\n", group->first.c_str());
 407             }
 408             return 0;
 409         }
 410         else if (arg.find(listInGroup) == 0)
 411         {
 412             arg.erase(0, listInGroup.length());
 413             FOREACH(test, m_testGroups[arg]) {
 414                 printf("ID:%s\n", test->name.c_str());
 415             }
 416             return 0;
 417         }
 418         else if (arg == "--help")
 419             showHelp = true;
 420         else if (arg.find(output) == 0)
 421         {
 422             arg.erase(0, output.length());
 423             if (m_collectors.find(arg) != m_collectors.end())
 424             {
 425                 InvalidArgs("Multiple outputs of the same type are not supported!");
 426                 Usage();
 427                 return -1;
 428             }
 429             currentCollector.reset(TestResultsCollectorBase::Create(arg));
 430             if (!currentCollector) {
 431                 InvalidArgs("Unsupported output type!");
 432                 Usage();
 433                 return -1;
 434             }
 435             m_collectors[arg] = currentCollector;
 436         }
 437         else if (arg.find(regexp) == 0)
 438         {
 439             arg.erase(0, regexp.length());
 440             if (arg.length() == 0)
 441             {
 442                 InvalidArgs();
 443                 Usage();
 444                 return -1;
 445             }
 446
 447             if (arg[0] == '\'' && arg[arg.length() - 1] == '\'')
 448             {
 449                 arg.erase(0);
 450                 arg.erase(arg.length() - 1);
 451             }
 452
 453             if (arg.length() == 0)
 454             {
 455                 InvalidArgs();
 456                 Usage();
 457                 return -1;
 458             }
 459
 460             pcrecpp::RE re(arg.c_str());
 461             FOREACH(group, m_testGroups) {
 462                 TestCaseStructList newList;
 463                 FOREACH(iterator, group->second)
 464                 {
 465                     if (re.PartialMatch(iterator->name))
 466                     {
 467                         newList.push_back(*iterator);
 468                     }
 469                 }
 470                 group->second = newList;
 471             }
 472         }
 473         else
 474         {
 475             InvalidArgs();
 476             Usage();
 477             return -1;
 478         }
 479     }
 480
 481     currentCollector.reset();
 482
 483     // Show help
 484     if (showHelp)
 485     {
 486         Usage();
 487         return 0;
 488     }
 489
 490     if (m_collectors.empty())
 491     {
 492         TestResultsCollectorBasePtr collector(
 493                 TestResultsCollectorBase::Create("text"));
 494         m_collectors["text"] = collector;
 495     }
 496
 497     for (auto it = m_collectors.begin(); it != m_collectors.end(); ++it)
 498     {
 499         if (!it->second->Configure())
 500         {
 501             fprintf(stderr, "Could not configure selected output");
 502             return 0;
 503         }
 504     }
 505
 506     // Run tests
 507     RunTests();
 508
 509     return 0;
 510 }
 511
 512 bool TestRunner::getRunIgnored() const
 513 {
 514     return m_runIgnored;
 515 }
 516
 517 void TestRunner::terminate()
 518 {
 519     m_terminate = true;
 520 }
 521
 522 }
 523 } // namespace DPL