modules/test/src/test_runner.cpp

   1 /*
   2  * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_runner.cpp
  18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
  19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  20  * @version     1.0
  21  * @brief       This file is the implementation file of test runner
  22  */
  23 #include <dpl/test/test_runner.h>
  24 #include <dpl/test/test_results_collector.h>
  25 #include <dpl/exception.h>
  26 #include <dpl/scoped_free.h>
  27 #include <dpl/foreach.h>
  28 #include <dpl/log/log.h>
  29 #include <dpl/colors.h>
  30 #include <pcrecpp.h>
  31 #include <algorithm>
  32 #include <cstdio>
  33 #include <memory.h>
  34 #include <libgen.h>
  35 #include <cstring>
  36 #include <cstdlib>
  37 #include <dpl/utils/wrt_global_settings.h>
  38
  39 #include <dpl/singleton_impl.h>
  40 IMPLEMENT_SINGLETON(DPL::Test::TestRunner)
  41
  42 namespace DPL
  43 {
  44 namespace Test
  45 {
  46
  47 namespace // anonymous
  48 {
  49
  50 std::string BaseName(std::string aPath)
  51 {
  52     ScopedFree<char> path(strdup(aPath.c_str()));
  53     if (NULL == path.Get())
  54     {
  55        throw std::bad_alloc();
  56     }
  57     char* baseName = basename(path.Get());
  58     std::string retValue = baseName;
  59     return retValue;
  60 }
  61
  62 } // namespace anonymous
  63
  64 //! \brief Failed test message creator
  65 //!
  66 //! \param[in] aTest string for tested expression
  67 //! \param[in] aFile source file name
  68 //! \param[in] aLine source file line
  69 //! \param[in] aMessage error message
  70 TestRunner::TestFailed::TestFailed(const char* aTest,
  71                                    const char* aFile,
  72                                    int aLine,
  73                                    const std::string &aMessage)
  74 {
  75     std::ostringstream assertMsg;
  76     assertMsg << "[" << BaseName(aFile) << ":" << aLine
  77               << "] Assertion failed ("
  78               << aTest << ") " << aMessage;
  79     m_message = assertMsg.str();
  80 }
  81
  82 void TestRunner::RegisterTest(const char *testName, TestCase proc)
  83 {
  84     m_testGroups[m_currentGroup].push_back(TestCaseStruct(testName, proc));
  85 }
  86
  87 void TestRunner::InitGroup(const char* name)
  88 {
  89     m_currentGroup = name;
  90 }
  91
  92
  93 TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase)
  94 {
  95     try
  96     {
  97         testCase.proc();
  98     }
  99     catch (const TestFailed &e)
 100     {
 101         // Simple test failure
 102         m_collector->CollectResult(testCase.name,
 103                                    "",
 104                                    TestResultsCollectorBase::FailStatus::FAILED,
 105                                    e.GetMessage());
 106         return FAILED;
 107     }
 108     catch (const Ignored &e)
 109     {
 110         // Simple test have to be implemented
 111         m_collector->CollectResult(testCase.name,
 112                                    "",
 113                                    TestResultsCollectorBase::FailStatus::IGNORED,
 114                                    e.GetMessage());
 115
 116         return IGNORED;
 117     }
 118     catch (const ToDo &e)
 119     {
 120         // Simple test have to be implemented
 121         m_collector->CollectResult(testCase.name,
 122                                    "",
 123                                    TestResultsCollectorBase::FailStatus::TODO,
 124                                    e.GetMessage());
 125
 126         return TODO;
 127     }
 128     catch (const DPL::Exception &e)
 129     {
 130         // DPL exception failure
 131         m_collector->CollectResult(testCase.name,
 132                                    "",
 133                                    TestResultsCollectorBase::FailStatus::INTERNAL,
 134                                    "DPL exception:" +
 135                                    e.GetMessage());
 136
 137         return FAILED;
 138     }
 139     catch (const std::exception &)
 140     {
 141         // std exception failure
 142         m_collector->CollectResult(testCase.name,
 143                                    "",
 144                                    TestResultsCollectorBase::FailStatus::INTERNAL,
 145                                    "std exception");
 146
 147         return FAILED;
 148     }
 149     catch (...)
 150     {
 151         // Unknown exception failure
 152         m_collector->CollectResult(testCase.name,
 153                                    "",
 154                                    TestResultsCollectorBase::FailStatus::INTERNAL,
 155                                    "unknown exception");
 156
 157         return FAILED;
 158     }
 159
 160     m_collector->CollectResult(testCase.name,
 161                                "",
 162                                TestResultsCollectorBase::FailStatus::NONE);
 163     // Everything OK
 164     return PASS;
 165 }
 166
 167 void TestRunner::RunTests()
 168 {
 169     using namespace DPL::Colors::Text;
 170
 171     Banner();
 172     m_collector->Start();
 173
 174     fprintf(stderr, "%s%s%s\n", GREEN_BEGIN, "Running tests...", GREEN_END);
 175     FOREACH(group, m_testGroups) {
 176         TestCaseStructList list = group->second;
 177         if (!list.empty()) {
 178             m_collector->CollectCurrentTestGroupName(group->first);
 179             list.sort();
 180
 181             for (TestCaseStructList::const_iterator iterator = list.begin();
 182                  iterator != list.end();
 183                  ++iterator)
 184             {
 185                 TestCaseStruct test = *iterator;
 186                 if (m_startTestId == test.name)
 187                     m_startTestId = "";
 188
 189                 if (m_startTestId.empty()) {
 190                     RunTestCase(test);
 191                 }
 192             }
 193         }
 194     }
 195
 196     m_collector->Finish();
 197     // Finished
 198     fprintf(stderr, "%s%s%s\n\n", GREEN_BEGIN, "Finished", GREEN_END);
 199 }
 200
 201 void TestRunner::Banner()
 202 {
 203     using namespace DPL::Colors::Text;
 204     fprintf(stderr,
 205             "%s%s%s\n",
 206             BOLD_GREEN_BEGIN,
 207             "DPL tests runner",
 208             BOLD_GREEN_END);
 209     fprintf(stderr,
 210             "%s%s%s%s\n\n",
 211             GREEN_BEGIN,
 212             "Build: ",
 213             __TIMESTAMP__,
 214             GREEN_END);
 215 }
 216
 217 void TestRunner::InvalidArgs()
 218 {
 219     using namespace DPL::Colors::Text;
 220     fprintf(stderr,
 221             "%s%s%s\n",
 222             BOLD_RED_BEGIN,
 223             "Invalid parameters!",
 224             BOLD_RED_END);
 225 }
 226
 227 void TestRunner::Usage()
 228 {
 229     fprintf(stderr, "Usage: runner [options]\n\n");
 230     fprintf(stderr, "Output type:\n");
 231     fprintf(stderr, "  --output=<output type>\n");
 232     fprintf(stderr, "  possible output types:\n");
 233     FOREACH (type, TestResultsCollectorBase::GetCollectorsNames()) {
 234         fprintf(stderr, "  --output=%s\n", type->c_str());
 235     }
 236     fprintf(stderr, "Other parameters:\n");
 237     fprintf(stderr,
 238             "  --regexp='regexp'\t Only selected tests"
 239             " which names match regexp run\n\n");
 240     fprintf(stderr, "  --start=<test id>\tStart from concrete test id");
 241     fprintf(stderr, "  --group=<group name>\t Run tests only from one group\n");
 242     fprintf(stderr, "  --list\t Show a list of Test IDs\n");
 243     fprintf(stderr, "  --listgroups\t Show a list of Test Group names \n");
 244     fprintf(stderr, "  --listingroup=<group name>\t Show a list of Test IDS in one group\n");
 245     fprintf(stderr, "  --help\t This help\n\n");
 246     if (m_collector) {
 247         fprintf(stderr, "Output %s has specific args:\n", m_collectorName.c_str());
 248         fprintf(stderr, "%s\n", m_collector->CollectorSpecificHelp().c_str());
 249     }
 250     fprintf(stderr, "For bug reporting, please write to:\n");
 251     fprintf(stderr, "<p.dobrowolsk@samsung.com>\n");
 252 }
 253
 254 int TestRunner::ExecTestRunner(int argc, char *argv[])
 255 {
 256     std::vector<std::string> args;
 257     for (int i = 0; i < argc; ++i)
 258     {
 259         args.push_back(argv[i]);
 260     }
 261     return ExecTestRunner(args);
 262 }
 263
 264 void TestRunner::MarkAssertion()
 265 {
 266     ++m_totalAssertions;
 267 }
 268
 269 int TestRunner::ExecTestRunner(const ArgsList& value)
 270 {
 271     ArgsList args = value;
 272     // Parse command line
 273     if (args.size() == 1)
 274     {
 275         InvalidArgs();
 276         Usage();
 277         return -1;
 278     }
 279
 280     args.erase(args.begin());
 281
 282     bool showHelp = false;
 283
 284     // Parse each argument
 285     FOREACH(it, args)
 286     {
 287         std::string arg = *it;
 288         const std::string regexp = "--regexp=";
 289         const std::string output = "--output=";
 290         const std::string groupId = "--group=";
 291         const std::string listCmd = "--list";
 292         const std::string startCmd = "--start=";
 293         const std::string listGroupsCmd = "--listgroups";
 294         const std::string listInGroup = "--listingroup=";
 295
 296         if (m_collector && m_collector->ParseCollectorSpecificArg(arg)) continue;
 297         else if (arg.find(startCmd) == 0)
 298         {
 299             arg.erase(0, startCmd.length());
 300             FOREACH(group, m_testGroups) {
 301                 FOREACH(tc, group->second) {
 302                     if (tc->name == arg) {
 303                         m_startTestId = arg;
 304                         break;
 305                     }
 306                 }
 307                 if (!m_startTestId.empty()) break;
 308             }
 309             if (!m_startTestId.empty()) continue;
 310             InvalidArgs();
 311             fprintf(stderr, "Start test id has not been found\n");
 312             Usage();
 313             return 0;
 314         }
 315         else if (arg.find(groupId) == 0)
 316         {
 317             arg.erase(0, groupId.length());
 318             TestCaseGroupMap::iterator found = m_testGroups.find(arg);
 319             if (found != m_testGroups.end()) {
 320                 std::string name = found->first;
 321                 TestCaseStructList newList = found->second;
 322                 m_testGroups.clear();
 323                 m_testGroups[name] = newList;
 324             } else {
 325                 fprintf(stderr, "Group %s not found\n", arg.c_str());
 326                 InvalidArgs();
 327                 Usage();
 328                 return -1;
 329             }
 330         }
 331         else if (arg == listCmd)
 332         {
 333             FOREACH(group, m_testGroups) {
 334                 FOREACH(test, group->second) {
 335                     printf("ID:%s:%s\n", group->first.c_str(), test->name.c_str());
 336                 }
 337             }
 338             return 0;
 339         }
 340         else if (arg == listGroupsCmd)
 341         {
 342             FOREACH(group, m_testGroups) {
 343                 printf("GR:%s\n", group->first.c_str());
 344             }
 345             return 0;
 346         }
 347         else if (arg.find(listInGroup) == 0)
 348         {
 349             arg.erase(0, listInGroup.length());
 350             FOREACH(test, m_testGroups[arg]) {
 351                 printf("ID:%s\n", test->name.c_str());
 352             }
 353             return 0;
 354         }
 355         else if (arg == "--help")
 356             showHelp = true;
 357         else if (arg.find(output) == 0)
 358         {
 359             arg.erase(0, output.length());
 360             m_collector.Reset(TestResultsCollectorBase::Create(arg));
 361             if (!m_collector) {
 362                 InvalidArgs();
 363                 Usage();
 364                 return -1;
 365             } else {
 366                 m_collectorName = arg;
 367             }
 368         }
 369         else if (arg.find(regexp) == 0)
 370         {
 371             arg.erase(0, regexp.length());
 372             if (arg.length() == 0)
 373             {
 374                 InvalidArgs();
 375                 Usage();
 376                 return -1;
 377             }
 378
 379             if (arg[0] == '\'' && arg[arg.length() - 1] == '\'')
 380             {
 381                 arg.erase(0);
 382                 arg.erase(arg.length() - 1);
 383             }
 384
 385             if (arg.length() == 0)
 386             {
 387                 InvalidArgs();
 388                 Usage();
 389                 return -1;
 390             }
 391
 392             pcrecpp::RE re(arg.c_str());
 393             FOREACH(group, m_testGroups) {
 394                 TestCaseStructList newList;
 395                 FOREACH(iterator, group->second)
 396                 {
 397                     if (re.PartialMatch(iterator->name))
 398                     {
 399                         newList.push_back(*iterator);
 400                     }
 401                 }
 402                 group->second = newList;
 403             }
 404         }
 405         else
 406         {
 407             InvalidArgs();
 408             Usage();
 409             return -1;
 410         }
 411     }
 412
 413     // Show help
 414     if (showHelp)
 415     {
 416         Usage();
 417         return 0;
 418     }
 419
 420     if (!m_collector)
 421         m_collector.Reset(TestResultsCollectorBase::Create("text"));
 422
 423     if (!m_collector->Configure()) {
 424         fprintf(stderr, "Could not configure selected output");
 425         return 0;
 426     }
 427
 428     // Run tests
 429     RunTests();
 430
 431     return 0;
 432 }
 433
 434 }
 435 } // namespace DPL