modules/test/src/test_runner.cpp

   1 /*
   2  * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.cpp
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @version     1.0
  21  * @brief       This file is the implementation file of test runner
  22  */
  23 #include <dpl/test/test_runner.h>
  24 #include <dpl/test/test_results_collector.h>
  25 #include <dpl/exception.h>
  26 #include <dpl/scoped_free.h>
  27 #include <dpl/foreach.h>
  28 #include <dpl/log/log.h>
  29 #include <dpl/colors.h>
  30 #include <pcrecpp.h>
  31 #include <algorithm>
  32 #include <cstdio>
  33 #include <memory.h>
  34 #include <libgen.h>
  35 #include <cstring>
  36
  37 #include <dpl/singleton_impl.h>
  38 IMPLEMENT_SINGLETON(DPL::Test::TestRunner)
  39
  40 namespace DPL
  41 {
  42 namespace Test
  43 {
  44
  45 namespace // anonymous
  46 {
  47
  48 std::string BaseName(std::string aPath)
  49 {
  50     ScopedFree<char> path(strdup(aPath.c_str()));
  51     if (NULL == path.Get())
  52     {
  53        throw std::bad_alloc();
  54     }
  55     char* baseName = basename(path.Get());
  56     std::string retValue = baseName;
  57     return retValue;
  58 }
  59
  60 } // namespace anonymous
  61
  62 //! \brief Failed test message creator
  63 //!
  64 //! \param[in] aTest string for tested expression
  65 //! \param[in] aFile source file name
  66 //! \param[in] aLine source file line
  67 //! \param[in] aMessage error message
  68 TestRunner::TestFailed::TestFailed(const char* aTest,
  69                                    const char* aFile,
  70                                    int aLine,
  71                                    const std::string &aMessage)
  72 {
  73     std::ostringstream assertMsg;
  74     assertMsg << "[" << BaseName(aFile) << ":" << aLine
  75               << "] Assertion failed ("
  76               << aTest << ") " << aMessage;
  77     m_message = assertMsg.str();
  78 }
  79
  80 void TestRunner::RegisterTest(const char *testName, TestCase proc)
  81 {
  82     m_testGroups[m_currentGroup].push_back(TestCaseStruct(testName, proc));
  83 }
  84
  85 void TestRunner::InitGroup(const char* name)
  86 {
  87     m_currentGroup = name;
  88 }
  89
  90
  91 TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase)
  92 {
  93     try
  94     {
  95         testCase.proc();
  96     }
  97     catch (const TestFailed &e)
  98     {
  99         // Simple test failure
 100         m_collector->CollectResult(testCase.name,
 101                                    "",
 102                                    TestResultsCollectorBase::FailStatus::FAILED,
 103                                    e.GetMessage());
 104         return FAILED;
 105     }
 106     catch (const Ignored &e)
 107     {
 108         // Simple test have to be implemented
 109         m_collector->CollectResult(testCase.name,
 110                                    "",
 111                                    TestResultsCollectorBase::FailStatus::IGNORED,
 112                                    e.GetMessage());
 113
 114         return IGNORED;
 115     }
 116     catch (const ToDo &e)
 117     {
 118         // Simple test have to be implemented
 119         m_collector->CollectResult(testCase.name,
 120                                    "",
 121                                    TestResultsCollectorBase::FailStatus::TODO,
 122                                    e.GetMessage());
 123
 124         return TODO;
 125     }
 126     catch (const DPL::Exception &e)
 127     {
 128         // DPL exception failure
 129         m_collector->CollectResult(testCase.name,
 130                                    "",
 131                                    TestResultsCollectorBase::FailStatus::INTERNAL,
 132                                    "DPL exception:" +
 133                                    e.GetMessage());
 134
 135         return FAILED;
 136     }
 137     catch (const std::exception &)
 138     {
 139         // std exception failure
 140         m_collector->CollectResult(testCase.name,
 141                                    "",
 142                                    TestResultsCollectorBase::FailStatus::INTERNAL,
 143                                    "std exception");
 144
 145         return FAILED;
 146     }
 147     catch (...)
 148     {
 149         // Unknown exception failure
 150         m_collector->CollectResult(testCase.name,
 151                                    "",
 152                                    TestResultsCollectorBase::FailStatus::INTERNAL,
 153                                    "unknown exception");
 154
 155         return FAILED;
 156     }
 157
 158     m_collector->CollectResult(testCase.name,
 159                                "",
 160                                TestResultsCollectorBase::FailStatus::NONE);
 161     // Everything OK
 162     return PASS;
 163 }
 164
 165 void TestRunner::RunTests()
 166 {
 167     using namespace DPL::Colors::Text;
 168
 169     Banner();
 170     m_collector->Start();
 171
 172     fprintf(stderr, "%s%s%s\n", GREEN_BEGIN, "Running tests...", GREEN_END);
 173     FOREACH(group, m_testGroups) {
 174         TestCaseStructList list = group->second;
 175         if (!list.empty()) {
 176             m_collector->CollectCurrentTestGroupName(group->first);
 177             list.sort();
 178
 179             for (TestCaseStructList::const_iterator iterator = list.begin();
 180                  iterator != list.end();
 181                  ++iterator)
 182             {
 183                 TestCaseStruct test = *iterator;
 184                 if (m_startTestId == test.name)
 185                     m_startTestId = "";
 186
 187                 if (m_startTestId.empty()) {
 188                     RunTestCase(test);
 189                 }
 190             }
 191         }
 192     }
 193
 194     m_collector->Finish();
 195     // Finished
 196     fprintf(stderr, "%s%s%s\n\n", GREEN_BEGIN, "Finished", GREEN_END);
 197 }
 198
 199 void TestRunner::Banner()
 200 {
 201     using namespace DPL::Colors::Text;
 202     fprintf(stderr,
 203             "%s%s%s\n",
 204             BOLD_GREEN_BEGIN,
 205             "DPL tests runner",
 206             BOLD_GREEN_END);
 207     fprintf(stderr,
 208             "%s%s%s%s\n\n",
 209             GREEN_BEGIN,
 210             "Build: ",
 211             __TIMESTAMP__,
 212             GREEN_END);
 213 }
 214
 215 void TestRunner::InvalidArgs()
 216 {
 217     using namespace DPL::Colors::Text;
 218     fprintf(stderr,
 219             "%s%s%s\n",
 220             BOLD_RED_BEGIN,
 221             "Invalid parameters!",
 222             BOLD_RED_END);
 223 }
 224
 225 void TestRunner::Usage()
 226 {
 227     fprintf(stderr, "Usage: runner [options]\n\n");
 228     fprintf(stderr, "Output type:\n");
 229     fprintf(stderr, "  --output=<output type>\n");
 230     fprintf(stderr, "  possible output types:\n");
 231     FOREACH (type, TestResultsCollectorBase::GetCollectorsNames()) {
 232         fprintf(stderr, "  --output=%s\n", type->c_str());
 233     }
 234     fprintf(stderr, "Other parameters:\n");
 235     fprintf(stderr,
 236             "  --regexp='regexp'\t Only selected tests"
 237             " which names match regexp run\n\n");
 238     fprintf(stderr, "  --start=<test id>\tStart from concrete test id");
 239     fprintf(stderr, "  --group=<group name>\t Run tests only from one group\n");
 240     fprintf(stderr, "  --list\t Show a list of Test IDs\n");
 241     fprintf(stderr, "  --listgroups\t Show a list of Test Group names \n");
 242     fprintf(stderr, "  --listingroup=<group name>\t Show a list of Test IDS in one group\n");
 243     fprintf(stderr, "  --help\t This help\n\n");
 244     if (m_collector) {
 245         fprintf(stderr, "Output %s has specific args:\n", m_collectorName.c_str());
 246         fprintf(stderr, "%s\n", m_collector->CollectorSpecificHelp().c_str());
 247     }
 248     fprintf(stderr, "For bug reporting, please write to:\n");
 249     fprintf(stderr, "<p.dobrowolsk@samsung.com>\n");
 250 }
 251
 252 int TestRunner::ExecTestRunner(int argc, char *argv[])
 253 {
 254     std::vector<std::string> args;
 255     for (int i = 0; i < argc; ++i)
 256     {
 257         args.push_back(argv[i]);
 258     }
 259     return ExecTestRunner(args);
 260 }
 261
 262 void TestRunner::MarkAssertion()
 263 {
 264     ++m_totalAssertions;
 265 }
 266
 267 int TestRunner::ExecTestRunner(const ArgsList& value)
 268 {
 269     ArgsList args = value;
 270     // Parse command line
 271     if (args.size() == 1)
 272     {
 273         InvalidArgs();
 274         Usage();
 275         return -1;
 276     }
 277
 278     args.erase(args.begin());
 279
 280     bool showHelp = false;
 281
 282     // Parse each argument
 283     FOREACH(it, args)
 284     {
 285         std::string arg = *it;
 286         const std::string regexp = "--regexp=";
 287         const std::string output = "--output=";
 288         const std::string groupId = "--group=";
 289         const std::string listCmd = "--list";
 290         const std::string startCmd = "--start=";
 291         const std::string listGroupsCmd = "--listgroups";
 292         const std::string listInGroup = "--listingroup=";
 293
 294         if (m_collector && m_collector->ParseCollectorSpecificArg(arg)) continue;
 295         else if (arg.find(startCmd) == 0)
 296         {
 297             arg.erase(0, startCmd.length());
 298             FOREACH(group, m_testGroups) {
 299                 FOREACH(tc, group->second) {
 300                     if (tc->name == arg) {
 301                         m_startTestId = arg;
 302                         break;
 303                     }
 304                 }
 305                 if (!m_startTestId.empty()) break;
 306             }
 307             if (!m_startTestId.empty()) continue;
 308             InvalidArgs();
 309             fprintf(stderr, "Start test id has not been found\n");
 310             Usage();
 311             return 0;
 312         }
 313         else if (arg.find(groupId) == 0)
 314         {
 315             arg.erase(0, groupId.length());
 316             TestCaseGroupMap::iterator found = m_testGroups.find(arg);
 317             if (found != m_testGroups.end()) {
 318                 std::string name = found->first;
 319                 TestCaseStructList newList = found->second;
 320                 m_testGroups.clear();
 321                 m_testGroups[name] = newList;
 322             } else {
 323                 fprintf(stderr, "Group %s not found\n", arg.c_str());
 324                 InvalidArgs();
 325                 Usage();
 326                 return -1;
 327             }
 328         }
 329         else if (arg == listCmd)
 330         {
 331             FOREACH(group, m_testGroups) {
 332                 FOREACH(test, group->second) {
 333                     printf("ID:%s:%s\n", group->first.c_str(), test->name.c_str());
 334                 }
 335             }
 336             return 0;
 337         }
 338         else if (arg == listGroupsCmd)
 339         {
 340             FOREACH(group, m_testGroups) {
 341                 printf("GR:%s\n", group->first.c_str());
 342             }
 343             return 0;
 344         }
 345         else if (arg.find(listInGroup) == 0)
 346         {
 347             arg.erase(0, listInGroup.length());
 348             FOREACH(test, m_testGroups[arg]) {
 349                 printf("ID:%s\n", test->name.c_str());
 350             }
 351             return 0;
 352         }
 353         else if (arg == "--help")
 354             showHelp = true;
 355         else if (arg.find(output) == 0)
 356         {
 357             arg.erase(0, output.length());
 358             m_collector.Reset(TestResultsCollectorBase::Create(arg));
 359             if (!m_collector) {
 360                 InvalidArgs();
 361                 Usage();
 362                 return -1;
 363             } else {
 364                 m_collectorName = arg;
 365             }
 366         }
 367         else if (arg.find(regexp) == 0)
 368         {
 369             arg.erase(0, regexp.length());
 370             if (arg.length() == 0)
 371             {
 372                 InvalidArgs();
 373                 Usage();
 374                 return -1;
 375             }
 376
 377             if (arg[0] == '\'' && arg[arg.length() - 1] == '\'')
 378             {
 379                 arg.erase(0);
 380                 arg.erase(arg.length() - 1);
 381             }
 382
 383             if (arg.length() == 0)
 384             {
 385                 InvalidArgs();
 386                 Usage();
 387                 return -1;
 388             }
 389
 390             pcrecpp::RE re(arg.c_str());
 391             FOREACH(group, m_testGroups) {
 392                 TestCaseStructList newList;
 393                 FOREACH(iterator, group->second)
 394                 {
 395                     if (re.PartialMatch(iterator->name))
 396                     {
 397                         newList.push_back(*iterator);
 398                     }
 399                 }
 400                 group->second = newList;
 401             }
 402         }
 403         else
 404         {
 405             InvalidArgs();
 406             Usage();
 407             return -1;
 408         }
 409     }
 410
 411     // Show help
 412     if (showHelp)
 413     {
 414         Usage();
 415         return 0;
 416     }
 417
 418     if (!m_collector)
 419         m_collector.Reset(TestResultsCollectorBase::Create("text"));
 420
 421     if (!m_collector->Configure()) {
 422         fprintf(stderr, "Could not configure selected output");
 423         return 0;
 424     }
 425
 426     // Run tests
 427     RunTests();
 428
 429     return 0;
 430 }
 431
 432 }
 433 } // namespace DPL