modules/test/src/test_runner.cpp

   1 /*
   2  * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.cpp
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @version     1.0
  21  * @brief       This file is the implementation file of test runner
  22  */
  23 #include <stddef.h>
  24 #include <dpl/test/test_runner.h>
  25 #include <dpl/test/test_results_collector.h>
  26 #include <dpl/exception.h>
  27 #include <dpl/scoped_free.h>
  28 #include <dpl/foreach.h>
  29 #include <dpl/log/log.h>
  30 #include <dpl/colors.h>
  31 #include <pcrecpp.h>
  32 #include <algorithm>
  33 #include <cstdio>
  34 #include <memory.h>
  35 #include <libgen.h>
  36 #include <cstring>
  37 #include <cstdlib>
  38 #include <dpl/utils/wrt_global_settings.h>
  39
  40 #include <dpl/singleton_impl.h>
  41 IMPLEMENT_SINGLETON(DPL::Test::TestRunner)
  42
  43 namespace DPL
  44 {
  45 namespace Test
  46 {
  47
  48 namespace // anonymous
  49 {
  50
  51 std::string BaseName(std::string aPath)
  52 {
  53     ScopedFree<char> path(strdup(aPath.c_str()));
  54     if (NULL == path.Get())
  55     {
  56        throw std::bad_alloc();
  57     }
  58     char* baseName = basename(path.Get());
  59     std::string retValue = baseName;
  60     return retValue;
  61 }
  62
  63 } // namespace anonymous
  64
  65 //! \brief Failed test message creator
  66 //!
  67 //! \param[in] aTest string for tested expression
  68 //! \param[in] aFile source file name
  69 //! \param[in] aLine source file line
  70 //! \param[in] aMessage error message
  71 TestRunner::TestFailed::TestFailed(const char* aTest,
  72                                    const char* aFile,
  73                                    int aLine,
  74                                    const std::string &aMessage)
  75 {
  76     std::ostringstream assertMsg;
  77     assertMsg << "[" << BaseName(aFile) << ":" << aLine
  78               << "] Assertion failed ("
  79               << aTest << ") " << aMessage;
  80     m_message = assertMsg.str();
  81 }
  82
  83 void TestRunner::RegisterTest(const char *testName, TestCase proc)
  84 {
  85     m_testGroups[m_currentGroup].push_back(TestCaseStruct(testName, proc));
  86 }
  87
  88 void TestRunner::InitGroup(const char* name)
  89 {
  90     m_currentGroup = name;
  91 }
  92
  93
  94 TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase)
  95 {
  96     try
  97     {
  98         testCase.proc();
  99     }
 100     catch (const TestFailed &e)
 101     {
 102         // Simple test failure
 103         CollectResult(testCase.name,
 104                       "",
 105                       TestResultsCollectorBase::FailStatus::FAILED,
 106                       e.GetMessage());
 107         return FAILED;
 108     }
 109     catch (const Ignored &e)
 110     {
 111         if (m_runIgnored) {
 112             // Simple test have to be implemented
 113             CollectResult(testCase.name,
 114                           "",
 115                           TestResultsCollectorBase::FailStatus::IGNORED,
 116                           e.GetMessage());
 117         }
 118
 119         return IGNORED;
 120     }
 121     catch (const DPL::Exception &e)
 122     {
 123         // DPL exception failure
 124         CollectResult(testCase.name,
 125                       "",
 126                       TestResultsCollectorBase::FailStatus::INTERNAL,
 127                       "DPL exception:" + e.GetMessage());
 128
 129         return FAILED;
 130     }
 131     catch (const std::exception &)
 132     {
 133         // std exception failure
 134         CollectResult(testCase.name,
 135                       "",
 136                       TestResultsCollectorBase::FailStatus::INTERNAL,
 137                       "std exception");
 138
 139         return FAILED;
 140     }
 141     catch (...)
 142     {
 143         // Unknown exception failure
 144         CollectResult(testCase.name,
 145                       "",
 146                       TestResultsCollectorBase::FailStatus::INTERNAL,
 147                       "unknown exception");
 148
 149         return FAILED;
 150     }
 151
 152     CollectResult(testCase.name,
 153                   "",
 154                   TestResultsCollectorBase::FailStatus::NONE);
 155
 156     // Everything OK
 157     return PASS;
 158 }
 159
 160 void TestRunner::RunTests()
 161 {
 162     using namespace DPL::Colors::Text;
 163
 164     Banner();
 165     std::for_each(m_collectors.begin(),
 166                   m_collectors.end(),
 167                   [](const TestResultsCollectors::value_type& collector)
 168                   {
 169                       collector.second->Start();
 170                   });
 171
 172     fprintf(stderr, "%s%s%s\n", GREEN_BEGIN, "Running tests...", GREEN_END);
 173     FOREACH(group, m_testGroups) {
 174         TestCaseStructList list = group->second;
 175         if (!list.empty()) {
 176             std::for_each(
 177                     m_collectors.begin(),
 178                     m_collectors.end(),
 179                     [&group](const TestResultsCollectors::value_type& collector)
 180                     {
 181                         collector.second->
 182                                 CollectCurrentTestGroupName(group->first);
 183                     });
 184             list.sort();
 185
 186             for (TestCaseStructList::const_iterator iterator = list.begin();
 187                  iterator != list.end();
 188                  ++iterator)
 189             {
 190                 TestCaseStruct test = *iterator;
 191                 if (m_startTestId == test.name)
 192                     m_startTestId = "";
 193
 194                 if (m_startTestId.empty()) {
 195                     RunTestCase(test);
 196                 }
 197             }
 198         }
 199     }
 200
 201     std::for_each(m_collectors.begin(),
 202                   m_collectors.end(),
 203                   [](const TestResultsCollectors::value_type& collector)
 204                   {
 205                       collector.second->Finish();
 206                   });
 207
 208     // Finished
 209     fprintf(stderr, "%s%s%s\n\n", GREEN_BEGIN, "Finished", GREEN_END);
 210 }
 211
 212 void TestRunner::CollectResult(
 213         const std::string& id,
 214         const std::string& description,
 215         const TestResultsCollectorBase::FailStatus::Type status,
 216         const std::string& reason)
 217 {
 218     std::for_each(m_collectors.begin(),
 219                   m_collectors.end(),
 220                   [&](const TestResultsCollectors::value_type& collector)
 221                   {
 222                       collector.second->CollectResult(id,
 223                                                       description,
 224                                                       status,
 225                                                       reason);
 226                   });
 227 }
 228
 229 void TestRunner::Banner()
 230 {
 231     using namespace DPL::Colors::Text;
 232     fprintf(stderr,
 233             "%s%s%s\n",
 234             BOLD_GREEN_BEGIN,
 235             "DPL tests runner",
 236             BOLD_GREEN_END);
 237     fprintf(stderr,
 238             "%s%s%s%s\n\n",
 239             GREEN_BEGIN,
 240             "Build: ",
 241             __TIMESTAMP__,
 242             GREEN_END);
 243 }
 244
 245 void TestRunner::InvalidArgs(const std::string& message)
 246 {
 247     using namespace DPL::Colors::Text;
 248     fprintf(stderr,
 249             "%s%s%s\n",
 250             BOLD_RED_BEGIN,
 251             message.c_str(),
 252             BOLD_RED_END);
 253 }
 254
 255 void TestRunner::Usage()
 256 {
 257     fprintf(stderr, "Usage: runner [options]\n\n");
 258     fprintf(stderr, "Output type:\n");
 259     fprintf(stderr, "  --output=<output type> --output=<output type> ...\n");
 260     fprintf(stderr, "\n  possible output types:\n");
 261     FOREACH (type, TestResultsCollectorBase::GetCollectorsNames()) {
 262         fprintf(stderr, "    --output=%s\n", type->c_str());
 263     }
 264     fprintf(stderr, "\n  example:\n");
 265     fprintf(stderr, "    test-binary --output=text --output=xml --file=output.xml\n\n");
 266     fprintf(stderr, "Other parameters:\n");
 267     fprintf(stderr,
 268             "  --regexp='regexp'\t Only selected tests"
 269             " which names match regexp run\n\n");
 270     fprintf(stderr, "  --start=<test id>\tStart from concrete test id");
 271     fprintf(stderr, "  --group=<group name>\t Run tests only from one group\n");
 272     fprintf(stderr, "  --runignored\t Run also ignored tests\n");
 273     fprintf(stderr, "  --list\t Show a list of Test IDs\n");
 274     fprintf(stderr, "  --listgroups\t Show a list of Test Group names \n");
 275     fprintf(stderr, "  --listingroup=<group name>\t Show a list of Test IDS in one group\n");
 276     fprintf(stderr, "  --help\t This help\n\n");
 277     std::for_each(m_collectors.begin(),
 278                   m_collectors.end(),
 279                   [](const TestResultsCollectors::value_type& collector)
 280                   {
 281                       fprintf(stderr,
 282                               "Output %s has specific args:\n",
 283                               collector.first.c_str());
 284                       fprintf(stderr,
 285                               "%s\n",
 286                               collector.second->
 287                                       CollectorSpecificHelp().c_str());
 288                   });
 289     fprintf(stderr, "For bug reporting, please write to:\n");
 290     fprintf(stderr, "<p.dobrowolsk@samsung.com>\n");
 291 }
 292
 293 int TestRunner::ExecTestRunner(int argc, char *argv[])
 294 {
 295     std::vector<std::string> args;
 296     for (int i = 0; i < argc; ++i)
 297     {
 298         args.push_back(argv[i]);
 299     }
 300     return ExecTestRunner(args);
 301 }
 302
 303 void TestRunner::MarkAssertion()
 304 {
 305     ++m_totalAssertions;
 306 }
 307
 308 int TestRunner::ExecTestRunner(const ArgsList& value)
 309 {
 310     m_runIgnored = false;
 311     ArgsList args = value;
 312     // Parse command line
 313     if (args.size() == 1)
 314     {
 315         InvalidArgs();
 316         Usage();
 317         return -1;
 318     }
 319
 320     args.erase(args.begin());
 321
 322     bool showHelp = false;
 323
 324     TestResultsCollectorBasePtr currentCollector;
 325
 326     // Parse each argument
 327     FOREACH(it, args)
 328     {
 329         std::string arg = *it;
 330         const std::string regexp = "--regexp=";
 331         const std::string output = "--output=";
 332         const std::string groupId = "--group=";
 333         const std::string runIgnored = "--runignored";
 334         const std::string listCmd = "--list";
 335         const std::string startCmd = "--start=";
 336         const std::string listGroupsCmd = "--listgroups";
 337         const std::string listInGroup = "--listingroup=";
 338
 339         if (currentCollector)
 340         {
 341             if (currentCollector->ParseCollectorSpecificArg(arg))
 342             {
 343                 continue;
 344             }
 345         }
 346
 347         if (arg.find(startCmd) == 0)
 348         {
 349             arg.erase(0, startCmd.length());
 350             FOREACH(group, m_testGroups) {
 351                 FOREACH(tc, group->second) {
 352                     if (tc->name == arg) {
 353                         m_startTestId = arg;
 354                         break;
 355                     }
 356                 }
 357                 if (!m_startTestId.empty()) break;
 358             }
 359             if (!m_startTestId.empty()) continue;
 360             InvalidArgs();
 361             fprintf(stderr, "Start test id has not been found\n");
 362             Usage();
 363             return 0;
 364         }
 365         else if (arg.find(groupId) == 0)
 366         {
 367             arg.erase(0, groupId.length());
 368             TestCaseGroupMap::iterator found = m_testGroups.find(arg);
 369             if (found != m_testGroups.end()) {
 370                 std::string name = found->first;
 371                 TestCaseStructList newList = found->second;
 372                 m_testGroups.clear();
 373                 m_testGroups[name] = newList;
 374             } else {
 375                 fprintf(stderr, "Group %s not found\n", arg.c_str());
 376                 InvalidArgs();
 377                 Usage();
 378                 return -1;
 379             }
 380         }
 381         else if (arg == runIgnored)
 382         {
 383             m_runIgnored = true;
 384         }
 385         else if (arg == listCmd)
 386         {
 387             FOREACH(group, m_testGroups) {
 388                 FOREACH(test, group->second) {
 389                     printf("ID:%s:%s\n", group->first.c_str(), test->name.c_str());
 390                 }
 391             }
 392             return 0;
 393         }
 394         else if (arg == listGroupsCmd)
 395         {
 396             FOREACH(group, m_testGroups) {
 397                 printf("GR:%s\n", group->first.c_str());
 398             }
 399             return 0;
 400         }
 401         else if (arg.find(listInGroup) == 0)
 402         {
 403             arg.erase(0, listInGroup.length());
 404             FOREACH(test, m_testGroups[arg]) {
 405                 printf("ID:%s\n", test->name.c_str());
 406             }
 407             return 0;
 408         }
 409         else if (arg == "--help")
 410             showHelp = true;
 411         else if (arg.find(output) == 0)
 412         {
 413             arg.erase(0, output.length());
 414             if (m_collectors.find(arg) != m_collectors.end())
 415             {
 416                 InvalidArgs("Multiple outputs of the same type are not supported!");
 417                 Usage();
 418                 return -1;
 419             }
 420             currentCollector.reset(TestResultsCollectorBase::Create(arg));
 421             if (!currentCollector) {
 422                 InvalidArgs("Unsupported output type!");
 423                 Usage();
 424                 return -1;
 425             }
 426             m_collectors[arg] = currentCollector;
 427         }
 428         else if (arg.find(regexp) == 0)
 429         {
 430             arg.erase(0, regexp.length());
 431             if (arg.length() == 0)
 432             {
 433                 InvalidArgs();
 434                 Usage();
 435                 return -1;
 436             }
 437
 438             if (arg[0] == '\'' && arg[arg.length() - 1] == '\'')
 439             {
 440                 arg.erase(0);
 441                 arg.erase(arg.length() - 1);
 442             }
 443
 444             if (arg.length() == 0)
 445             {
 446                 InvalidArgs();
 447                 Usage();
 448                 return -1;
 449             }
 450
 451             pcrecpp::RE re(arg.c_str());
 452             FOREACH(group, m_testGroups) {
 453                 TestCaseStructList newList;
 454                 FOREACH(iterator, group->second)
 455                 {
 456                     if (re.PartialMatch(iterator->name))
 457                     {
 458                         newList.push_back(*iterator);
 459                     }
 460                 }
 461                 group->second = newList;
 462             }
 463         }
 464         else
 465         {
 466             InvalidArgs();
 467             Usage();
 468             return -1;
 469         }
 470     }
 471
 472     currentCollector.reset();
 473
 474     // Show help
 475     if (showHelp)
 476     {
 477         Usage();
 478         return 0;
 479     }
 480
 481     if (m_collectors.empty())
 482     {
 483         TestResultsCollectorBasePtr collector(
 484                 TestResultsCollectorBase::Create("text"));
 485         m_collectors["text"] = collector;
 486     }
 487
 488     for (auto it = m_collectors.begin(); it != m_collectors.end(); ++it)
 489     {
 490         if (!it->second->Configure())
 491         {
 492             fprintf(stderr, "Could not configure selected output");
 493             return 0;
 494         }
 495     }
 496
 497     // Run tests
 498     RunTests();
 499
 500     return 0;
 501 }
 502
 503 bool TestRunner::getRunIgnored() const
 504 {
 505     return m_runIgnored;
 506 }
 507
 508 }
 509 } // namespace DPL