modules/test/src/test_results_collector.cpp

   1 /*
   2  * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_results_collector.h
  18  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  19  * @version     1.0
  20  * @brief       Implementation file some concrete TestResulstsCollector
  21  */
  22
  23 #include <dpl/test/test_results_collector.h>
  24 #include <dpl/colors.h>
  25 #include <dpl/assert.h>
  26 #include <dpl/foreach.h>
  27 #include <dpl/scoped_fclose.h>
  28
  29 #include <string>
  30 #include <cstdio>
  31 #include <fstream>
  32 #include <sstream>
  33
  34 namespace DPL
  35 {
  36 namespace Test
  37 {
  38
  39 namespace
  40 {
  41 const char *DEFAULT_HTML_FILE_NAME = "index.html";
  42 const char *DEFAULT_TAP_FILE_NAME = "results.tap";
  43
  44 class Statistic
  45 {
  46   public:
  47     Statistic() :
  48         m_failed(0),
  49         m_ignored(0),
  50         m_todo(0),
  51         m_passed(0),
  52         m_count(0)
  53     {
  54     }
  55
  56     void AddTest(TestResultsCollectorBase::FailStatus::Type type)
  57     {
  58         ++m_count;
  59         switch (type) {
  60             case TestResultsCollectorBase::FailStatus::INTERNAL:
  61             case TestResultsCollectorBase::FailStatus::FAILED:   ++m_failed; break;
  62             case TestResultsCollectorBase::FailStatus::IGNORED:  ++m_ignored; break;
  63             case TestResultsCollectorBase::FailStatus::TODO:     ++m_todo; break;
  64             case TestResultsCollectorBase::FailStatus::NONE:     ++m_passed; break;
  65             default:
  66                 Assert(false && "Bad FailStatus");
  67         }
  68     }
  69
  70     size_t GetTotal() const { return m_count; }
  71     size_t GetPassed() const { return m_passed; }
  72     size_t GetSuccesed() const { return m_passed; }
  73     size_t GetFailed() const { return m_failed; }
  74     size_t GetTODO() const { return m_todo; }
  75     size_t GetIgnored() const { return m_ignored; }
  76     float GetPassedOrIgnoredPercend() const
  77     {
  78         float passIgnoredPercent =
  79             100.0f * (static_cast<float>(m_passed)
  80                       + static_cast<float>(m_ignored))
  81             / static_cast<float>(m_count);
  82         return passIgnoredPercent;
  83     }
  84
  85   private:
  86     size_t m_failed;
  87     size_t m_ignored;
  88     size_t m_todo;
  89     size_t m_passed;
  90     size_t m_count;
  91 };
  92
  93 class ConsoleCollector
  94     : public TestResultsCollectorBase
  95 {
  96   public:
  97     static TestResultsCollectorBase* Constructor();
  98
  99   private:
 100     ConsoleCollector() {}
 101
 102     virtual void CollectCurrentTestGroupName(const std::string& name)
 103     {
 104         printf("Starting group %s\n", name.c_str());
 105         m_currentGroup = name;
 106     }
 107
 108     virtual void Finish()
 109     {
 110         using namespace DPL::Colors::Text;
 111
 112         // Show result
 113         FOREACH(group, m_groupsStats) {
 114             PrintStats(group->first, group->second);
 115         }
 116         PrintStats("All tests together", m_stats);
 117     }
 118
 119     virtual void CollectResult(const std::string& id,
 120                                const std::string& /*description*/,
 121                                const FailStatus::Type status = FailStatus::NONE,
 122                                const std::string& reason = "")
 123     {
 124         using namespace DPL::Colors::Text;
 125         std::string tmp = "'" + id + "' ...";
 126
 127         printf("Running test case %-60s", tmp.c_str());
 128         switch(status) {
 129             case TestResultsCollectorBase::FailStatus::NONE:
 130                 printf("[%s%s%s]\n", BOLD_GREEN_BEGIN, "   OK   ", BOLD_GREEN_END); break;
 131             case TestResultsCollectorBase::FailStatus::FAILED:
 132                 PrintfErrorMessage(  " FAILED ", reason, true); break;
 133             case TestResultsCollectorBase::FailStatus::IGNORED:
 134                 PrintfIgnoredMessage("Ignored ", reason, true); break;
 135             case TestResultsCollectorBase::FailStatus::TODO:
 136                 PrintfTODOMessage(   "  TODO  ", reason, true); break;
 137             case TestResultsCollectorBase::FailStatus::INTERNAL:
 138                 PrintfErrorMessage(  "INTERNAL", reason, true); break;
 139             default:
 140                 Assert(false && "Bad status");
 141         }
 142         m_stats.AddTest(status);
 143         m_groupsStats[m_currentGroup].AddTest(status);
 144     }
 145
 146     void PrintfErrorMessage(const char* type,
 147                             const std::string& message,
 148                             bool verbosity)
 149     {
 150         using namespace DPL::Colors::Text;
 151         if (verbosity) {
 152             printf("[%s%s%s] %s%s%s\n",
 153                    BOLD_RED_BEGIN,
 154                    type,
 155                    BOLD_RED_END,
 156                    BOLD_YELLOW_BEGIN,
 157                    message.c_str(),
 158                    BOLD_YELLOW_END);
 159         } else {
 160             printf("[%s%s%s]\n",
 161                     BOLD_RED_BEGIN,
 162                     type,
 163                     BOLD_RED_END);
 164         }
 165     }
 166
 167     void PrintfTODOMessage(const char* type,
 168                            const std::string& message,
 169                            bool verbosity)
 170     {
 171         using namespace DPL::Colors::Text;
 172         if (verbosity) {
 173             printf("[%s%s%s] %s%s%s\n",
 174                    BOLD_WHITE_BEGIN,
 175                    type,
 176                    BOLD_WHITE_END,
 177                    BOLD_GOLD_BEGIN,
 178                    message.c_str(),
 179                    BOLD_GOLD_END);
 180         } else {
 181             printf("[%s%s%s]\n",
 182                    BOLD_WHITE_BEGIN,
 183                    type,
 184                    BOLD_WHITE_END);
 185         }
 186     }
 187
 188     void PrintfIgnoredMessage(const char* type,
 189                               const std::string& message,
 190                               bool verbosity)
 191     {
 192         using namespace DPL::Colors::Text;
 193         if (verbosity) {
 194             printf("[%s%s%s] %s%s%s\n",
 195                    CYAN_BEGIN,
 196                    type,
 197                    CYAN_END,
 198                    BOLD_GOLD_BEGIN,
 199                    message.c_str(),
 200                    BOLD_GOLD_END);
 201         } else {
 202             printf("[%s%s%s]\n",
 203                    CYAN_BEGIN ,
 204                    type,
 205                    CYAN_END);
 206         }
 207     }
 208
 209     void PrintStats(const std::string& title, const Statistic& stats)
 210     {
 211         using namespace DPL::Colors::Text;
 212         printf("\n%sResults [%s]: %s\n", BOLD_GREEN_BEGIN, title.c_str(), BOLD_GREEN_END);
 213         printf("%s%s%3d%s\n", CYAN_BEGIN,     "Total tests:            ", stats.GetTotal(), CYAN_END);
 214         printf("%s%s%3d%s\n", CYAN_BEGIN,     "Succeeded or ignored:   ", stats.GetPassed() + stats.GetIgnored(), CYAN_END);
 215         printf("  %s%s%3d%s\n", CYAN_BEGIN,     "Succeeded:            ", stats.GetPassed(), CYAN_END);
 216         printf("  %s%s%3d%s\n", CYAN_BEGIN,     "Ignored:              ", stats.GetIgnored(), CYAN_END);
 217         printf("%s%s%3d%s\n", CYAN_BEGIN,     "Failed:                 ", stats.GetFailed(), CYAN_END);
 218         printf("%s%s%3d%s\n", CYAN_BEGIN,     "Todo:                   ", stats.GetTODO(), CYAN_END);
 219         printf("%s%s%3.0f%%%s\n", CYAN_BEGIN, "Succeeded or ignored %: ", stats.GetPassedOrIgnoredPercend(), CYAN_END);
 220     }
 221
 222     Statistic m_stats;
 223     std::map<std::string, Statistic> m_groupsStats;
 224     std::string m_currentGroup;
 225 };
 226
 227
 228 TestResultsCollectorBase* ConsoleCollector::Constructor()
 229 {
 230     return new ConsoleCollector();
 231 }
 232
 233
 234 class HtmlCollector
 235     : public TestResultsCollectorBase
 236 {
 237   public:
 238     static TestResultsCollectorBase* Constructor();
 239
 240   private:
 241     HtmlCollector() : m_filename(DEFAULT_HTML_FILE_NAME) {}
 242
 243     virtual void CollectCurrentTestGroupName(const std::string& name)
 244     {
 245         fprintf(m_fp.Get(),"<b>Starting group %s", name.c_str());
 246         m_currentGroup = name;
 247     }
 248
 249     virtual bool Configure()
 250     {
 251         m_fp.Reset(fopen (m_filename.c_str(), "w"));
 252         if (!m_fp) {
 253             LogPedantic("Could not open file " << m_filename << " for writing");
 254             return false;
 255         }
 256         return true;
 257     }
 258     virtual std::string CollectorSpecificHelp() const
 259     {
 260         return "--file=<filename> - name of file for output\n"
 261                "                    default - index.html\n";
 262     }
 263
 264     virtual void Start()
 265     {
 266         Assert(!!m_fp && "File handle must not be null");
 267         fprintf(m_fp.Get(),
 268                 "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0"
 269                 "Transitional//EN\" "
 270                 "\"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\""
 271                 ">\n");
 272         fprintf(m_fp.Get(),
 273                 "<html xmlns=\"http://www.w3.org/1999/xhtml\" "
 274                 "lang=\"en\" dir=\"ltr\">\n");
 275         fprintf(m_fp.Get(), "<body style=\"background-color: black;\">\n");
 276         fprintf(m_fp.Get(), "<pre>\n");
 277         fprintf(m_fp.Get(), "<font color=\"white\">\n");
 278     }
 279
 280     virtual void Finish()
 281     {
 282         using namespace DPL::Colors::Html;
 283         // Show result
 284         FOREACH(group, m_groupsStats) {
 285             PrintStats(group->first, group->second);
 286         }
 287         PrintStats("All tests together", m_stats);
 288         fprintf(m_fp.Get(), "</font>\n");
 289         fprintf(m_fp.Get(), "</pre>\n");
 290         fprintf(m_fp.Get(), "</body>\n");
 291         fprintf(m_fp.Get(), "</html>\n");
 292     }
 293
 294     virtual bool ParseCollectorSpecificArg(const std::string& arg)
 295     {
 296         const std::string argname = "--file=";
 297         if (0 == arg.find(argname)) {
 298             m_filename = arg.substr(argname.size());
 299             return true;
 300         } else {
 301             return false;
 302         }
 303     }
 304
 305     virtual void CollectResult(const std::string& id,
 306                                const std::string& /*description*/,
 307                                const FailStatus::Type status = FailStatus::NONE,
 308                                const std::string& reason = "")
 309     {
 310         using namespace DPL::Colors::Html;
 311         std::string tmp = "'" + id + "' ...";
 312
 313         fprintf(m_fp.Get(), "Running test case %-100s", tmp.c_str());
 314         switch(status) {
 315             case TestResultsCollectorBase::FailStatus::NONE:
 316                 fprintf(m_fp.Get(), "[%s%s%s]\n", BOLD_GREEN_BEGIN, "   OK   ", BOLD_GREEN_END); break;
 317             case TestResultsCollectorBase::FailStatus::FAILED:
 318                 PrintfErrorMessage(  " FAILED ", reason, true); break;
 319             case TestResultsCollectorBase::FailStatus::IGNORED:
 320                 PrintfIgnoredMessage("Ignored ", reason, true); break;
 321             case TestResultsCollectorBase::FailStatus::TODO:
 322                 PrintfTODOMessage(   "  TODO  ", reason, true); break;
 323             case TestResultsCollectorBase::FailStatus::INTERNAL:
 324                 PrintfErrorMessage(  "INTERNAL", reason, true); break;
 325             default:
 326                 Assert(false && "Bad status");
 327         }
 328         m_groupsStats[m_currentGroup].AddTest(status);
 329         m_stats.AddTest(status);
 330     }
 331
 332     void PrintfErrorMessage(const char* type,
 333                             const std::string& message,
 334                             bool verbosity)
 335     {
 336         using namespace DPL::Colors::Html;
 337         if (verbosity) {
 338             fprintf(m_fp.Get(),
 339                     "[%s%s%s] %s%s%s\n",
 340                     BOLD_RED_BEGIN,
 341                     type,
 342                     BOLD_RED_END,
 343                     BOLD_YELLOW_BEGIN,
 344                     message.c_str(),
 345                     BOLD_YELLOW_END);
 346         } else {
 347             fprintf(m_fp.Get(),
 348                     "[%s%s%s]\n",
 349                     BOLD_RED_BEGIN,
 350                     type,
 351                     BOLD_RED_END);
 352         }
 353     }
 354
 355     void PrintfTODOMessage(const char* type,
 356                            const std::string& message,
 357                            bool verbosity)
 358     {
 359         using namespace DPL::Colors::Html;
 360         if (verbosity) {
 361             fprintf(m_fp.Get(),
 362                     "[%s%s%s] %s%s%s\n",
 363                     BOLD_WHITE_BEGIN,
 364                     type,
 365                     BOLD_WHITE_END,
 366                     BOLD_GOLD_BEGIN,
 367                     message.c_str(),
 368                     BOLD_GOLD_END);
 369         } else {
 370             fprintf(m_fp.Get(),
 371                     "[%s%s%s]\n",
 372                     BOLD_WHITE_BEGIN,
 373                     type,
 374                     BOLD_WHITE_END);
 375         }
 376     }
 377
 378     void PrintfIgnoredMessage(const char* type,
 379                               const std::string& message,
 380                               bool verbosity)
 381     {
 382         using namespace DPL::Colors::Html;
 383
 384         if (verbosity) {
 385             fprintf(m_fp.Get(),
 386                     "[%s%s%s] %s%s%s\n",
 387                     CYAN_BEGIN,
 388                     type,
 389                     CYAN_END,
 390                     BOLD_GOLD_BEGIN,
 391                     message.c_str(),
 392                     BOLD_GOLD_END);
 393         } else {
 394             fprintf(m_fp.Get(),
 395                     "[%s%s%s]\n",
 396                     CYAN_BEGIN ,
 397                     type,
 398                     CYAN_END);
 399         }
 400     }
 401
 402     void PrintStats(const std::string& name, const Statistic& stats)
 403     {
 404         using namespace DPL::Colors::Html;
 405         fprintf(m_fp.Get(), "\n%sResults [%s]:%s\n", BOLD_GREEN_BEGIN, name.c_str(), BOLD_GREEN_END);
 406         fprintf(m_fp.Get(), "%s%s%3d%s\n", CYAN_BEGIN,     "Total tests:            ", stats.GetTotal(), CYAN_END);
 407         fprintf(m_fp.Get(), "%s%s%3d%s\n", CYAN_BEGIN,     "Succeeded or ignored:   ", stats.GetPassed() + stats.GetIgnored(), CYAN_END);
 408         fprintf(m_fp.Get(), "  %s%s%3d%s\n", CYAN_BEGIN,     "Succeeded:            ", stats.GetPassed(), CYAN_END);
 409         fprintf(m_fp.Get(), "  %s%s%3d%s\n", CYAN_BEGIN,     "Ignored:              ", stats.GetIgnored(), CYAN_END);
 410         fprintf(m_fp.Get(), "%s%s%3d%s\n", CYAN_BEGIN,     "Failed:                 ", stats.GetFailed(), CYAN_END);
 411         fprintf(m_fp.Get(), "%s%s%3d%s\n", CYAN_BEGIN,     "Todo:                   ", stats.GetTODO(), CYAN_END);
 412         fprintf(m_fp.Get(), "%s%s%3.0f%%%s\n", CYAN_BEGIN, "Succeeded or ignored %: ", stats.GetPassedOrIgnoredPercend(), CYAN_END);
 413     }
 414
 415     std::string m_filename;
 416     ScopedFClose m_fp;
 417     Statistic m_stats;
 418     std::string m_currentGroup;
 419     std::map<std::string, Statistic> m_groupsStats;
 420 };
 421
 422 TestResultsCollectorBase* HtmlCollector::Constructor()
 423 {
 424     return new HtmlCollector();
 425 }
 426
 427 class CSVCollector
 428     : public TestResultsCollectorBase
 429 {
 430   public:
 431     static TestResultsCollectorBase* Constructor();
 432
 433   private:
 434     CSVCollector() {}
 435
 436     virtual void Start() {
 437         printf("GROUP;ID;RESULT;REASON\n");
 438     }
 439
 440     virtual void CollectCurrentTestGroupName(const std::string& name)
 441     {
 442         m_currentGroup = name;
 443     }
 444
 445     virtual void CollectResult(const std::string& id,
 446                                const std::string& /*description*/,
 447                                const FailStatus::Type status = FailStatus::NONE,
 448                                const std::string& reason = "")
 449     {
 450         std::string statusMsg = "";
 451         switch(status) {
 452             case TestResultsCollectorBase::FailStatus::NONE: statusMsg = "OK"; break;
 453             case TestResultsCollectorBase::FailStatus::FAILED: statusMsg = "FAILED"; break;
 454             case TestResultsCollectorBase::FailStatus::IGNORED: statusMsg = "IGNORED"; break;
 455             case TestResultsCollectorBase::FailStatus::TODO:  statusMsg = "TODO"; break;
 456             case TestResultsCollectorBase::FailStatus::INTERNAL: statusMsg = "FAILED"; break;
 457             default:
 458                 Assert(false && "Bad status");
 459         }
 460         printf("%s;%s;%s;%s\n",
 461                m_currentGroup.c_str(),
 462                id.c_str(),
 463                statusMsg.c_str(),
 464                reason.c_str());
 465     }
 466
 467     std::string m_currentGroup;
 468 };
 469
 470
 471 TestResultsCollectorBase* CSVCollector::Constructor()
 472 {
 473     return new CSVCollector();
 474 }
 475
 476 }
 477
 478 class TAPCollector
 479     : public TestResultsCollectorBase
 480 {
 481   public:
 482     static TestResultsCollectorBase* Constructor();
 483
 484   private:
 485     TAPCollector() : m_filename(DEFAULT_TAP_FILE_NAME)  {}
 486
 487     virtual bool Configure()
 488     {
 489         m_output.open(m_filename.c_str(), std::ios_base::trunc);
 490         if (m_output.fail()) {
 491             LogError("Can't open output file: " << m_filename);
 492             return false;
 493         }
 494         return true;
 495     }
 496     virtual std::string CollectorSpecificHelp() const
 497     {
 498         std::string retVal = "--file=<filename> - name of file for output\n"
 499                              "                    default - ";
 500         retVal += DEFAULT_TAP_FILE_NAME;
 501         retVal += "\n";
 502         return retVal;
 503     }
 504
 505     virtual void Start()
 506     {
 507         Assert(m_output.good() && "Output file must be opened.");
 508         m_output << "TAP version 13" << std::endl;
 509         m_testIndex = 0;
 510     }
 511
 512     virtual void Finish()
 513     {
 514         m_output << "1.." << m_testIndex << std::endl;
 515         m_output << m_collectedData.rdbuf();
 516         m_output.close();
 517     }
 518
 519     virtual bool ParseCollectorSpecificArg(const std::string& arg)
 520     {
 521         const std::string argname = "--file=";
 522         if (0 == arg.find(argname)) {
 523             m_filename = arg.substr(argname.size());
 524             return true;
 525         } else {
 526             return false;
 527         }
 528     }
 529
 530
 531
 532     virtual void CollectResult(const std::string& id,
 533                                const std::string& description,
 534                                const FailStatus::Type status = FailStatus::NONE,
 535                                const std::string& reason = "")
 536     {
 537         m_testIndex++;
 538         switch(status) {
 539             case TestResultsCollectorBase::FailStatus::NONE:
 540                 LogBasicTAP(true, id, description);
 541                 endTAPLine();
 542                 break;
 543             case TestResultsCollectorBase::FailStatus::FAILED:
 544                 LogBasicTAP(false, id, description);
 545                 endTAPLine();
 546                 break;
 547             case TestResultsCollectorBase::FailStatus::IGNORED:
 548                 LogBasicTAP(true, id, description);
 549                 m_collectedData << " # skip " << reason;
 550                 endTAPLine();
 551                 break;
 552             case TestResultsCollectorBase::FailStatus::TODO:
 553                 LogBasicTAP(false, id, description);
 554                 m_collectedData << " # TODO " << reason;
 555                 endTAPLine();
 556                 break;
 557             case TestResultsCollectorBase::FailStatus::INTERNAL:
 558                 LogBasicTAP(true, id, description);
 559                 endTAPLine();
 560                 m_collectedData << "    ---" << std::endl;
 561                 m_collectedData << "    message: " << reason << std::endl;
 562                 m_collectedData << "    severity: Internal" << std::endl;
 563                 m_collectedData << "    ..." << std::endl;
 564                 break;
 565             default:
 566                 Assert(false && "Bad status");
 567         }
 568     }
 569
 570     void LogBasicTAP(bool isOK, const std::string& id,
 571             const std::string& description)
 572     {
 573         if (!isOK) {
 574             m_collectedData << "not ";
 575         }
 576         m_collectedData << "ok " << m_testIndex << " [" <<
 577                             id << "] " << description;
 578     }
 579
 580     void endTAPLine()
 581     {
 582         m_collectedData << std::endl;
 583     }
 584
 585
 586     std::string m_filename;
 587     std::stringstream m_collectedData;
 588     std::ofstream m_output;
 589     int m_testIndex;
 590 };
 591
 592
 593 TestResultsCollectorBase* TAPCollector::Constructor()
 594 {
 595     return new TAPCollector();
 596 }
 597
 598
 599 void TestResultsCollectorBase::RegisterCollectorConstructor(
 600     const std::string& name,
 601     TestResultsCollectorBase::CollectorConstructorFunc func)
 602 {
 603     Assert(m_constructorsMap.find(name) == m_constructorsMap.end());
 604     m_constructorsMap[name] = func;
 605 }
 606
 607 TestResultsCollectorBase* TestResultsCollectorBase::Create(
 608     const std::string& name)
 609 {
 610     ConstructorsMap::iterator found = m_constructorsMap.find(name);
 611     if (found != m_constructorsMap.end())
 612         return found->second();
 613     else
 614         return NULL;
 615 }
 616
 617 std::vector<std::string> TestResultsCollectorBase::GetCollectorsNames()
 618 {
 619     std::vector<std::string> list;
 620     FOREACH(it, m_constructorsMap)
 621     {
 622         list.push_back(it->first);
 623     }
 624     return list;
 625 }
 626
 627 TestResultsCollectorBase::ConstructorsMap TestResultsCollectorBase::m_constructorsMap;
 628
 629 namespace
 630 {
 631 static int RegisterCollectorConstructors();
 632 static const int RegisterHelperVariable = RegisterCollectorConstructors();
 633 int RegisterCollectorConstructors()
 634 {
 635     (void)RegisterHelperVariable;
 636
 637     TestResultsCollectorBase::RegisterCollectorConstructor(
 638         "text",
 639         &ConsoleCollector::Constructor);
 640     TestResultsCollectorBase::RegisterCollectorConstructor(
 641         "html",
 642         &HtmlCollector::Constructor);
 643     TestResultsCollectorBase::RegisterCollectorConstructor(
 644         "csv",
 645         &CSVCollector::Constructor);
 646     TestResultsCollectorBase::RegisterCollectorConstructor(
 647         "tap",
 648         &TAPCollector::Constructor);
 649
 650     return 0;
 651 }
 652
 653 }
 654
 655 }
 656 }