modules/test/src/test_runner.cpp

   1 /*
   2  * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.cpp
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @version     1.0
  21  * @brief       This file is the implementation file of test runner
  22  */
  23 #include <stddef.h>
  24 #include <dpl/test/test_runner.h>
  25 #include <dpl/test/test_results_collector.h>
  26 #include <dpl/exception.h>
  27 #include <dpl/scoped_free.h>
  28 #include <dpl/foreach.h>
  29 #include <dpl/log/log.h>
  30 #include <dpl/colors.h>
  31 #include <pcrecpp.h>
  32 #include <algorithm>
  33 #include <cstdio>
  34 #include <memory.h>
  35 #include <libgen.h>
  36 #include <cstring>
  37 #include <cstdlib>
  38 #include <dpl/utils/wrt_global_settings.h>
  39
  40 #include <dpl/singleton_impl.h>
  41 IMPLEMENT_SINGLETON(DPL::Test::TestRunner)
  42
  43 namespace DPL {
  44 namespace Test {
  45 namespace // anonymous
  46 {
  47 std::string BaseName(std::string aPath)
  48 {
  49     ScopedFree<char> path(strdup(aPath.c_str()));
  50     if (NULL == path.Get()) {
  51         throw std::bad_alloc();
  52     }
  53     char* baseName = basename(path.Get());
  54     std::string retValue = baseName;
  55     return retValue;
  56 }
  57 } // namespace anonymous
  58
  59 //! \brief Failed test message creator
  60 //!
  61 //! \param[in] aTest string for tested expression
  62 //! \param[in] aFile source file name
  63 //! \param[in] aLine source file line
  64 //! \param[in] aMessage error message
  65 TestRunner::TestFailed::TestFailed(const char* aTest,
  66                                    const char* aFile,
  67                                    int aLine,
  68                                    const std::string &aMessage)
  69 {
  70     std::ostringstream assertMsg;
  71     assertMsg << "[" << BaseName(aFile) << ":" << aLine
  72               << "] Assertion failed ("
  73               << aTest << ") " << aMessage;
  74     m_message = assertMsg.str();
  75 }
  76
  77 TestRunner::TestFailed::TestFailed(const std::string &message)
  78 {
  79     m_message = message;
  80 }
  81
  82 void TestRunner::RegisterTest(const char *testName, TestCase proc)
  83 {
  84     m_testGroups[m_currentGroup].push_back(TestCaseStruct(testName, proc));
  85 }
  86
  87 void TestRunner::InitGroup(const char* name)
  88 {
  89     m_currentGroup = name;
  90 }
  91
  92 TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase)
  93 {
  94     try {
  95         testCase.proc();
  96     } catch (const TestFailed &e) {
  97         // Simple test failure
  98         CollectResult(testCase.name,
  99                       "",
 100                       TestResultsCollectorBase::FailStatus::FAILED,
 101                       e.GetMessage());
 102         return FAILED;
 103     } catch (const Ignored &e) {
 104         if (m_runIgnored) {
 105             // Simple test have to be implemented
 106             CollectResult(testCase.name,
 107                           "",
 108                           TestResultsCollectorBase::FailStatus::IGNORED,
 109                           e.GetMessage());
 110         }
 111
 112         return IGNORED;
 113     } catch (const DPL::Exception &e) {
 114         // DPL exception failure
 115         CollectResult(testCase.name,
 116                       "",
 117                       TestResultsCollectorBase::FailStatus::INTERNAL,
 118                       "DPL exception:" + e.GetMessage());
 119
 120         return FAILED;
 121     } catch (const std::exception &) {
 122         // std exception failure
 123         CollectResult(testCase.name,
 124                       "",
 125                       TestResultsCollectorBase::FailStatus::INTERNAL,
 126                       "std exception");
 127
 128         return FAILED;
 129     } catch (...) {
 130         // Unknown exception failure
 131         CollectResult(testCase.name,
 132                       "",
 133                       TestResultsCollectorBase::FailStatus::INTERNAL,
 134                       "unknown exception");
 135
 136         return FAILED;
 137     }
 138
 139     CollectResult(testCase.name,
 140                   "",
 141                   TestResultsCollectorBase::FailStatus::NONE);
 142
 143     // Everything OK
 144     return PASS;
 145 }
 146
 147 void TestRunner::RunTests()
 148 {
 149     using namespace DPL::Colors::Text;
 150
 151     Banner();
 152     std::for_each(m_collectors.begin(),
 153                   m_collectors.end(),
 154                   [] (const TestResultsCollectors::value_type & collector)
 155                   {
 156                       collector.second->Start();
 157                   });
 158
 159     fprintf(stderr, "%s%s%s\n", GREEN_BEGIN, "Running tests...", GREEN_END);
 160     FOREACH(group, m_testGroups) {
 161         TestCaseStructList list = group->second;
 162         if (!list.empty()) {
 163             std::for_each(
 164                 m_collectors.begin(),
 165                 m_collectors.end(),
 166                 [&group](const TestResultsCollectors::value_type & collector)
 167                 {
 168                     collector.second->
 169                         CollectCurrentTestGroupName(group->first);
 170                 });
 171             list.sort();
 172
 173             for (TestCaseStructList::const_iterator iterator = list.begin();
 174                  iterator != list.end();
 175                  ++iterator)
 176             {
 177                 TestCaseStruct test = *iterator;
 178                 if (m_startTestId == test.name) {
 179                     m_startTestId = "";
 180                 }
 181
 182                 if (m_startTestId.empty()) {
 183                     RunTestCase(test);
 184                 }
 185                 if (m_terminate == true) {
 186                     // Terminate quietly without any logs
 187                     return;
 188                 }
 189             }
 190         }
 191     }
 192
 193     std::for_each(m_collectors.begin(),
 194                   m_collectors.end(),
 195                   [] (const TestResultsCollectors::value_type & collector)
 196                   {
 197                       collector.second->Finish();
 198                   });
 199
 200     // Finished
 201     fprintf(stderr, "%s%s%s\n\n", GREEN_BEGIN, "Finished", GREEN_END);
 202 }
 203
 204 void TestRunner::CollectResult(
 205     const std::string& id,
 206     const std::string& description,
 207     const TestResultsCollectorBase::FailStatus::Type status,
 208     const std::string& reason)
 209 {
 210     std::for_each(m_collectors.begin(),
 211                   m_collectors.end(),
 212                   [&](const TestResultsCollectors::value_type & collector)
 213                   {
 214                       collector.second->CollectResult(id,
 215                                                       description,
 216                                                       status,
 217                                                       reason);
 218                   });
 219 }
 220
 221 void TestRunner::Banner()
 222 {
 223     using namespace DPL::Colors::Text;
 224     fprintf(stderr,
 225             "%s%s%s\n",
 226             BOLD_GREEN_BEGIN,
 227             "DPL tests runner",
 228             BOLD_GREEN_END);
 229     fprintf(stderr,
 230             "%s%s%s%s\n\n",
 231             GREEN_BEGIN,
 232             "Build: ",
 233             __TIMESTAMP__,
 234             GREEN_END);
 235 }
 236
 237 void TestRunner::InvalidArgs(const std::string& message)
 238 {
 239     using namespace DPL::Colors::Text;
 240     fprintf(stderr,
 241             "%s%s%s\n",
 242             BOLD_RED_BEGIN,
 243             message.c_str(),
 244             BOLD_RED_END);
 245 }
 246
 247 void TestRunner::Usage()
 248 {
 249     fprintf(stderr, "Usage: runner [options]\n\n");
 250     fprintf(stderr, "Output type:\n");
 251     fprintf(stderr, "  --output=<output type> --output=<output type> ...\n");
 252     fprintf(stderr, "\n  possible output types:\n");
 253     FOREACH(type, TestResultsCollectorBase::GetCollectorsNames()) {
 254         fprintf(stderr, "    --output=%s\n", type->c_str());
 255     }
 256     fprintf(stderr, "\n  example:\n");
 257     fprintf(stderr,
 258             "    test-binary --output=text --output=xml --file=output.xml\n\n");
 259     fprintf(stderr, "Other parameters:\n");
 260     fprintf(stderr,
 261             "  --regexp='regexp'\t Only selected tests"
 262             " which names match regexp run\n\n");
 263     fprintf(stderr, "  --start=<test id>\tStart from concrete test id");
 264     fprintf(stderr, "  --group=<group name>\t Run tests only from one group\n");
 265     fprintf(stderr, "  --runignored\t Run also ignored tests\n");
 266     fprintf(stderr, "  --list\t Show a list of Test IDs\n");
 267     fprintf(stderr, "  --listgroups\t Show a list of Test Group names \n");
 268     fprintf(
 269         stderr,
 270         "  --listingroup=<group name>\t Show a list of Test IDS in one group\n");
 271     fprintf(stderr, "  --help\t This help\n\n");
 272     std::for_each(m_collectors.begin(),
 273                   m_collectors.end(),
 274                   [] (const TestResultsCollectors::value_type & collector)
 275                   {
 276                       fprintf(stderr,
 277                               "Output %s has specific args:\n",
 278                               collector.first.c_str());
 279                       fprintf(stderr,
 280                               "%s\n",
 281                               collector.second->
 282                                   CollectorSpecificHelp().c_str());
 283                   });
 284     fprintf(stderr, "For bug reporting, please write to:\n");
 285     fprintf(stderr, "<p.dobrowolsk@samsung.com>\n");
 286 }
 287
 288 int TestRunner::ExecTestRunner(int argc, char *argv[])
 289 {
 290     std::vector<std::string> args;
 291     for (int i = 0; i < argc; ++i) {
 292         args.push_back(argv[i]);
 293     }
 294     return ExecTestRunner(args);
 295 }
 296
 297 void TestRunner::MarkAssertion()
 298 {
 299     ++m_totalAssertions;
 300 }
 301
 302 int TestRunner::ExecTestRunner(const ArgsList& value)
 303 {
 304     m_runIgnored = false;
 305     ArgsList args = value;
 306     // Parse command line
 307     if (args.size() == 1) {
 308         InvalidArgs();
 309         Usage();
 310         return -1;
 311     }
 312
 313     args.erase(args.begin());
 314
 315     bool showHelp = false;
 316
 317     TestResultsCollectorBasePtr currentCollector;
 318
 319     // Parse each argument
 320     FOREACH(it, args)
 321     {
 322         std::string arg = *it;
 323         const std::string regexp = "--regexp=";
 324         const std::string output = "--output=";
 325         const std::string groupId = "--group=";
 326         const std::string runIgnored = "--runignored";
 327         const std::string listCmd = "--list";
 328         const std::string startCmd = "--start=";
 329         const std::string listGroupsCmd = "--listgroups";
 330         const std::string listInGroup = "--listingroup=";
 331
 332         if (currentCollector) {
 333             if (currentCollector->ParseCollectorSpecificArg(arg)) {
 334                 continue;
 335             }
 336         }
 337
 338         if (arg.find(startCmd) == 0) {
 339             arg.erase(0, startCmd.length());
 340             FOREACH(group, m_testGroups) {
 341                 FOREACH(tc, group->second) {
 342                     if (tc->name == arg) {
 343                         m_startTestId = arg;
 344                         break;
 345                     }
 346                 }
 347                 if (!m_startTestId.empty()) {
 348                     break;
 349                 }
 350             }
 351             if (!m_startTestId.empty()) {
 352                 continue;
 353             }
 354             InvalidArgs();
 355             fprintf(stderr, "Start test id has not been found\n");
 356             Usage();
 357             return 0;
 358         } else if (arg.find(groupId) == 0) {
 359             arg.erase(0, groupId.length());
 360             TestCaseGroupMap::iterator found = m_testGroups.find(arg);
 361             if (found != m_testGroups.end()) {
 362                 std::string name = found->first;
 363                 TestCaseStructList newList = found->second;
 364                 m_testGroups.clear();
 365                 m_testGroups[name] = newList;
 366             } else {
 367                 fprintf(stderr, "Group %s not found\n", arg.c_str());
 368                 InvalidArgs();
 369                 Usage();
 370                 return -1;
 371             }
 372         } else if (arg == runIgnored) {
 373             m_runIgnored = true;
 374         } else if (arg == listCmd) {
 375             FOREACH(group, m_testGroups) {
 376                 FOREACH(test, group->second) {
 377                     printf("ID:%s:%s\n", group->first.c_str(), test->name.c_str());
 378                 }
 379             }
 380             return 0;
 381         } else if (arg == listGroupsCmd) {
 382             FOREACH(group, m_testGroups) {
 383                 printf("GR:%s\n", group->first.c_str());
 384             }
 385             return 0;
 386         } else if (arg.find(listInGroup) == 0) {
 387             arg.erase(0, listInGroup.length());
 388             FOREACH(test, m_testGroups[arg]) {
 389                 printf("ID:%s\n", test->name.c_str());
 390             }
 391             return 0;
 392         } else if (arg == "--help") {
 393             showHelp = true;
 394         } else if (arg.find(output) == 0) {
 395             arg.erase(0, output.length());
 396             if (m_collectors.find(arg) != m_collectors.end()) {
 397                 InvalidArgs(
 398                     "Multiple outputs of the same type are not supported!");
 399                 Usage();
 400                 return -1;
 401             }
 402             currentCollector.reset(TestResultsCollectorBase::Create(arg));
 403             if (!currentCollector) {
 404                 InvalidArgs("Unsupported output type!");
 405                 Usage();
 406                 return -1;
 407             }
 408             m_collectors[arg] = currentCollector;
 409         } else if (arg.find(regexp) == 0) {
 410             arg.erase(0, regexp.length());
 411             if (arg.length() == 0) {
 412                 InvalidArgs();
 413                 Usage();
 414                 return -1;
 415             }
 416
 417             if (arg[0] == '\'' && arg[arg.length() - 1] == '\'') {
 418                 arg.erase(0);
 419                 arg.erase(arg.length() - 1);
 420             }
 421
 422             if (arg.length() == 0) {
 423                 InvalidArgs();
 424                 Usage();
 425                 return -1;
 426             }
 427
 428             pcrecpp::RE re(arg.c_str());
 429             FOREACH(group, m_testGroups) {
 430                 TestCaseStructList newList;
 431                 FOREACH(iterator, group->second)
 432                 {
 433                     if (re.PartialMatch(iterator->name)) {
 434                         newList.push_back(*iterator);
 435                     }
 436                 }
 437                 group->second = newList;
 438             }
 439         } else {
 440             InvalidArgs();
 441             Usage();
 442             return -1;
 443         }
 444     }
 445
 446     currentCollector.reset();
 447
 448     // Show help
 449     if (showHelp) {
 450         Usage();
 451         return 0;
 452     }
 453
 454     if (m_collectors.empty()) {
 455         TestResultsCollectorBasePtr collector(
 456             TestResultsCollectorBase::Create("text"));
 457         m_collectors["text"] = collector;
 458     }
 459
 460     for (auto it = m_collectors.begin(); it != m_collectors.end(); ++it) {
 461         if (!it->second->Configure()) {
 462             fprintf(stderr, "Could not configure selected output");
 463             return 0;
 464         }
 465     }
 466
 467     // Run tests
 468     RunTests();
 469
 470     return 0;
 471 }
 472
 473 bool TestRunner::getRunIgnored() const
 474 {
 475     return m_runIgnored;
 476 }
 477
 478 void TestRunner::terminate()
 479 {
 480     m_terminate = true;
 481 }
 482 }
 483 } // namespace DPL