modules/test/src/test_results_collector.cpp

   1 /*
   2  * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  *    Licensed under the Apache License, Version 2.0 (the "License");
   5  *    you may not use this file except in compliance with the License.
   6  *    You may obtain a copy of the License at
   7  *
   8  *        http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  *    Unless required by applicable law or agreed to in writing, software
  11  *    distributed under the License is distributed on an "AS IS" BASIS,
  12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  *    See the License for the specific language governing permissions and
  14  *    limitations under the License.
  15  */
  16 /*
  17  * @file        test_results_collector.h
  18  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
  19  * @version     1.0
  20  * @brief       Implementation file some concrete TestResulstsCollector
  21  */
  22
  23 #include <dpl/test/test_results_collector.h>
  24 #include <dpl/colors.h>
  25 #include <dpl/assert.h>
  26 #include <dpl/foreach.h>
  27 #include <dpl/scoped_fclose.h>
  28
  29 #include <string>
  30 #include <cstdio>
  31
  32 namespace DPL
  33 {
  34 namespace Test
  35 {
  36
  37 namespace
  38 {
  39 const char *DEFAULT_HTML_FILE_NAME = "index.html";
  40
  41 class Statistic
  42 {
  43   public:
  44     Statistic() :
  45         m_failed(0),
  46         m_ignored(0),
  47         m_todo(0),
  48         m_passed(0),
  49         m_count(0)
  50     {
  51     }
  52
  53     void AddTest(TestResultsCollectorBase::FailStatus::Type type)
  54     {
  55         ++m_count;
  56         switch (type) {
  57             case TestResultsCollectorBase::FailStatus::INTERNAL:
  58             case TestResultsCollectorBase::FailStatus::FAILED:   ++m_failed; break;
  59             case TestResultsCollectorBase::FailStatus::IGNORED:  ++m_ignored; break;
  60             case TestResultsCollectorBase::FailStatus::TODO:     ++m_todo; break;
  61             case TestResultsCollectorBase::FailStatus::NONE:     ++m_passed; break;
  62             default:
  63                 Assert(false && "Bad FailStatus");
  64         }
  65     }
  66
  67     size_t GetTotal() const { return m_count; }
  68     size_t GetPassed() const { return m_passed; }
  69     size_t GetSuccesed() const { return m_passed; }
  70     size_t GetFailed() const { return m_failed; }
  71     size_t GetTODO() const { return m_todo; }
  72     size_t GetIgnored() const { return m_ignored; }
  73     float GetPassedOrIgnoredPercend() const
  74     {
  75         float passIgnoredPercent =
  76             100.0f * (static_cast<float>(m_passed)
  77                       + static_cast<float>(m_ignored))
  78             / static_cast<float>(m_count);
  79         return passIgnoredPercent;
  80     }
  81
  82   private:
  83     size_t m_failed;
  84     size_t m_ignored;
  85     size_t m_todo;
  86     size_t m_passed;
  87     size_t m_count;
  88 };
  89
  90 class ConsoleCollector
  91     : public TestResultsCollectorBase
  92 {
  93   public:
  94     static TestResultsCollectorBase* Constructor();
  95
  96   private:
  97     ConsoleCollector() {}
  98
  99     virtual void CollectCurrentTestGroupName(const std::string& name)
 100     {
 101         printf("Starting group %s\n", name.c_str());
 102         m_currentGroup = name;
 103     }
 104
 105     virtual void Finish()
 106     {
 107         using namespace DPL::Colors::Text;
 108
 109         // Show result
 110         FOREACH(group, m_groupsStats) {
 111             PrintStats(group->first, group->second);
 112         }
 113         PrintStats("All tests together", m_stats);
 114     }
 115
 116     virtual void CollectResult(const std::string& id,
 117                                const std::string& /*description*/,
 118                                const FailStatus::Type status = FailStatus::NONE,
 119                                const std::string& reason = "")
 120     {
 121         using namespace DPL::Colors::Text;
 122         std::string tmp = "'" + id + "' ...";
 123
 124         printf("Running test case %-60s", tmp.c_str());
 125         switch(status) {
 126             case TestResultsCollectorBase::FailStatus::NONE:
 127                 printf("[%s%s%s]\n", BOLD_GREEN_BEGIN, "   OK   ", BOLD_GREEN_END); break;
 128             case TestResultsCollectorBase::FailStatus::FAILED:
 129                 PrintfErrorMessage(  " FAILED ", reason, true); break;
 130             case TestResultsCollectorBase::FailStatus::IGNORED:
 131                 PrintfIgnoredMessage("Ignored ", reason, true); break;
 132             case TestResultsCollectorBase::FailStatus::TODO:
 133                 PrintfTODOMessage(   "  TODO  ", reason, true); break;
 134             case TestResultsCollectorBase::FailStatus::INTERNAL:
 135                 PrintfErrorMessage(  "INTERNAL", reason, true); break;
 136             default:
 137                 Assert(false && "Bad status");
 138         }
 139         m_stats.AddTest(status);
 140         m_groupsStats[m_currentGroup].AddTest(status);
 141     }
 142
 143     void PrintfErrorMessage(const char* type,
 144                             const std::string& message,
 145                             bool verbosity)
 146     {
 147         using namespace DPL::Colors::Text;
 148         if (verbosity) {
 149             printf("[%s%s%s] %s%s%s\n",
 150                    BOLD_RED_BEGIN,
 151                    type,
 152                    BOLD_RED_END,
 153                    BOLD_YELLOW_BEGIN,
 154                    message.c_str(),
 155                    BOLD_YELLOW_END);
 156         } else {
 157             printf("[%s%s%s]\n",
 158                     BOLD_RED_BEGIN,
 159                     type,
 160                     BOLD_RED_END);
 161         }
 162     }
 163
 164     void PrintfTODOMessage(const char* type,
 165                            const std::string& message,
 166                            bool verbosity)
 167     {
 168         using namespace DPL::Colors::Text;
 169         if (verbosity) {
 170             printf("[%s%s%s] %s%s%s\n",
 171                    BOLD_WHITE_BEGIN,
 172                    type,
 173                    BOLD_WHITE_END,
 174                    BOLD_GOLD_BEGIN,
 175                    message.c_str(),
 176                    BOLD_GOLD_END);
 177         } else {
 178             printf("[%s%s%s]\n",
 179                    BOLD_WHITE_BEGIN,
 180                    type,
 181                    BOLD_WHITE_END);
 182         }
 183     }
 184
 185     void PrintfIgnoredMessage(const char* type,
 186                               const std::string& message,
 187                               bool verbosity)
 188     {
 189         using namespace DPL::Colors::Text;
 190         if (verbosity) {
 191             printf("[%s%s%s] %s%s%s\n",
 192                    CYAN_BEGIN,
 193                    type,
 194                    CYAN_END,
 195                    BOLD_GOLD_BEGIN,
 196                    message.c_str(),
 197                    BOLD_GOLD_END);
 198         } else {
 199             printf("[%s%s%s]\n",
 200                    CYAN_BEGIN ,
 201                    type,
 202                    CYAN_END);
 203         }
 204     }
 205
 206     void PrintStats(const std::string& title, const Statistic& stats)
 207     {
 208         using namespace DPL::Colors::Text;
 209         printf("\n%sResults [%s]: %s\n", BOLD_GREEN_BEGIN, title.c_str(), BOLD_GREEN_END);
 210         printf("%s%s%3d%s\n", CYAN_BEGIN,     "Total tests:            ", stats.GetTotal(), CYAN_END);
 211         printf("%s%s%3d%s\n", CYAN_BEGIN,     "Succeeded or ignored:   ", stats.GetPassed() + stats.GetIgnored(), CYAN_END);
 212         printf("  %s%s%3d%s\n", CYAN_BEGIN,     "Succeeded:            ", stats.GetPassed(), CYAN_END);
 213         printf("  %s%s%3d%s\n", CYAN_BEGIN,     "Ignored:              ", stats.GetIgnored(), CYAN_END);
 214         printf("%s%s%3d%s\n", CYAN_BEGIN,     "Failed:                 ", stats.GetFailed(), CYAN_END);
 215         printf("%s%s%3d%s\n", CYAN_BEGIN,     "Todo:                   ", stats.GetTODO(), CYAN_END);
 216         printf("%s%s%3.0f%%%s\n", CYAN_BEGIN, "Succeeded or ignored %: ", stats.GetPassedOrIgnoredPercend(), CYAN_END);
 217     }
 218
 219     Statistic m_stats;
 220     std::map<std::string, Statistic> m_groupsStats;
 221     std::string m_currentGroup;
 222 };
 223
 224
 225 TestResultsCollectorBase* ConsoleCollector::Constructor()
 226 {
 227     return new ConsoleCollector();
 228 }
 229
 230
 231 class HtmlCollector
 232     : public TestResultsCollectorBase
 233 {
 234   public:
 235     static TestResultsCollectorBase* Constructor();
 236
 237   private:
 238     HtmlCollector() : m_filename(DEFAULT_HTML_FILE_NAME) {}
 239
 240     virtual void CollectCurrentTestGroupName(const std::string& name)
 241     {
 242         fprintf(m_fp.Get(),"<b>Starting group %s", name.c_str());
 243         m_currentGroup = name;
 244     }
 245
 246     virtual bool Configure()
 247     {
 248         m_fp.Reset(fopen (m_filename.c_str(), "w"));
 249         if (!m_fp) {
 250             LogPedantic("Could not open file " << m_filename << " for writing");
 251             return false;
 252         }
 253         return true;
 254     }
 255     virtual std::string CollectorSpecificHelp() const
 256     {
 257         return "--file=<filename> - name of file for output\n"
 258                "                    default - index.html\n";
 259     }
 260
 261     virtual void Start()
 262     {
 263         Assert(!!m_fp && "File handle must not be null");
 264         fprintf(m_fp.Get(),
 265                 "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0"
 266                 "Transitional//EN\" "
 267                 "\"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\""
 268                 ">\n");
 269         fprintf(m_fp.Get(),
 270                 "<html xmlns=\"http://www.w3.org/1999/xhtml\" "
 271                 "lang=\"en\" dir=\"ltr\">\n");
 272         fprintf(m_fp.Get(), "<body style=\"background-color: black;\">\n");
 273         fprintf(m_fp.Get(), "<pre>\n");
 274         fprintf(m_fp.Get(), "<font color=\"white\">\n");
 275     }
 276
 277     virtual void Finish()
 278     {
 279         using namespace DPL::Colors::Html;
 280         // Show result
 281         FOREACH(group, m_groupsStats) {
 282             PrintStats(group->first, group->second);
 283         }
 284         PrintStats("All tests together", m_stats);
 285         fprintf(m_fp.Get(), "</font>\n");
 286         fprintf(m_fp.Get(), "</pre>\n");
 287         fprintf(m_fp.Get(), "</body>\n");
 288         fprintf(m_fp.Get(), "</html>\n");
 289     }
 290
 291     virtual bool ParseCollectorSpecificArg(const std::string& arg)
 292     {
 293         const std::string argname = "--file=";
 294         if (0 == arg.find(argname)) {
 295             m_filename = arg.substr(argname.size());
 296             return true;
 297         } else {
 298             return false;
 299         }
 300     }
 301
 302     virtual void CollectResult(const std::string& id,
 303                                const std::string& /*description*/,
 304                                const FailStatus::Type status = FailStatus::NONE,
 305                                const std::string& reason = "")
 306     {
 307         using namespace DPL::Colors::Html;
 308         std::string tmp = "'" + id + "' ...";
 309
 310         fprintf(m_fp.Get(), "Running test case %-100s", tmp.c_str());
 311         switch(status) {
 312             case TestResultsCollectorBase::FailStatus::NONE:
 313                 fprintf(m_fp.Get(), "[%s%s%s]\n", BOLD_GREEN_BEGIN, "   OK   ", BOLD_GREEN_END); break;
 314             case TestResultsCollectorBase::FailStatus::FAILED:
 315                 PrintfErrorMessage(  " FAILED ", reason, true); break;
 316             case TestResultsCollectorBase::FailStatus::IGNORED:
 317                 PrintfIgnoredMessage("Ignored ", reason, true); break;
 318             case TestResultsCollectorBase::FailStatus::TODO:
 319                 PrintfTODOMessage(   "  TODO  ", reason, true); break;
 320             case TestResultsCollectorBase::FailStatus::INTERNAL:
 321                 PrintfErrorMessage(  "INTERNAL", reason, true); break;
 322             default:
 323                 Assert(false && "Bad status");
 324         }
 325         m_groupsStats[m_currentGroup].AddTest(status);
 326         m_stats.AddTest(status);
 327     }
 328
 329     void PrintfErrorMessage(const char* type,
 330                             const std::string& message,
 331                             bool verbosity)
 332     {
 333         using namespace DPL::Colors::Html;
 334         if (verbosity) {
 335             fprintf(m_fp.Get(),
 336                     "[%s%s%s] %s%s%s\n",
 337                     BOLD_RED_BEGIN,
 338                     type,
 339                     BOLD_RED_END,
 340                     BOLD_YELLOW_BEGIN,
 341                     message.c_str(),
 342                     BOLD_YELLOW_END);
 343         } else {
 344             fprintf(m_fp.Get(),
 345                     "[%s%s%s]\n",
 346                     BOLD_RED_BEGIN,
 347                     type,
 348                     BOLD_RED_END);
 349         }
 350     }
 351
 352     void PrintfTODOMessage(const char* type,
 353                            const std::string& message,
 354                            bool verbosity)
 355     {
 356         using namespace DPL::Colors::Html;
 357         if (verbosity) {
 358             fprintf(m_fp.Get(),
 359                     "[%s%s%s] %s%s%s\n",
 360                     BOLD_WHITE_BEGIN,
 361                     type,
 362                     BOLD_WHITE_END,
 363                     BOLD_GOLD_BEGIN,
 364                     message.c_str(),
 365                     BOLD_GOLD_END);
 366         } else {
 367             fprintf(m_fp.Get(),
 368                     "[%s%s%s]\n",
 369                     BOLD_WHITE_BEGIN,
 370                     type,
 371                     BOLD_WHITE_END);
 372         }
 373     }
 374
 375     void PrintfIgnoredMessage(const char* type,
 376                               const std::string& message,
 377                               bool verbosity)
 378     {
 379         using namespace DPL::Colors::Html;
 380
 381         if (verbosity) {
 382             fprintf(m_fp.Get(),
 383                     "[%s%s%s] %s%s%s\n",
 384                     CYAN_BEGIN,
 385                     type,
 386                     CYAN_END,
 387                     BOLD_GOLD_BEGIN,
 388                     message.c_str(),
 389                     BOLD_GOLD_END);
 390         } else {
 391             fprintf(m_fp.Get(),
 392                     "[%s%s%s]\n",
 393                     CYAN_BEGIN ,
 394                     type,
 395                     CYAN_END);
 396         }
 397     }
 398
 399     void PrintStats(const std::string& name, const Statistic& stats)
 400     {
 401         using namespace DPL::Colors::Html;
 402         fprintf(m_fp.Get(), "\n%sResults [%s]:%s\n", BOLD_GREEN_BEGIN, name.c_str(), BOLD_GREEN_END);
 403         fprintf(m_fp.Get(), "%s%s%3d%s\n", CYAN_BEGIN,     "Total tests:            ", stats.GetTotal(), CYAN_END);
 404         fprintf(m_fp.Get(), "%s%s%3d%s\n", CYAN_BEGIN,     "Succeeded or ignored:   ", stats.GetPassed() + stats.GetIgnored(), CYAN_END);
 405         fprintf(m_fp.Get(), "  %s%s%3d%s\n", CYAN_BEGIN,     "Succeeded:            ", stats.GetPassed(), CYAN_END);
 406         fprintf(m_fp.Get(), "  %s%s%3d%s\n", CYAN_BEGIN,     "Ignored:              ", stats.GetIgnored(), CYAN_END);
 407         fprintf(m_fp.Get(), "%s%s%3d%s\n", CYAN_BEGIN,     "Failed:                 ", stats.GetFailed(), CYAN_END);
 408         fprintf(m_fp.Get(), "%s%s%3d%s\n", CYAN_BEGIN,     "Todo:                   ", stats.GetTODO(), CYAN_END);
 409         fprintf(m_fp.Get(), "%s%s%3.0f%%%s\n", CYAN_BEGIN, "Succeeded or ignored %: ", stats.GetPassedOrIgnoredPercend(), CYAN_END);
 410     }
 411
 412     std::string m_filename;
 413     ScopedFClose m_fp;
 414     Statistic m_stats;
 415     std::string m_currentGroup;
 416     std::map<std::string, Statistic> m_groupsStats;
 417 };
 418
 419 TestResultsCollectorBase* HtmlCollector::Constructor()
 420 {
 421     return new HtmlCollector();
 422 }
 423
 424 class CSVCollector
 425     : public TestResultsCollectorBase
 426 {
 427   public:
 428     static TestResultsCollectorBase* Constructor();
 429
 430   private:
 431     CSVCollector() {}
 432
 433     virtual void Start() {
 434         printf("GROUP;ID;RESULT;REASON\n");
 435     }
 436
 437     virtual void CollectCurrentTestGroupName(const std::string& name)
 438     {
 439         m_currentGroup = name;
 440     }
 441
 442     virtual void CollectResult(const std::string& id,
 443                                const std::string& /*description*/,
 444                                const FailStatus::Type status = FailStatus::NONE,
 445                                const std::string& reason = "")
 446     {
 447         std::string statusMsg = "";
 448         switch(status) {
 449             case TestResultsCollectorBase::FailStatus::NONE: statusMsg = "OK"; break;
 450             case TestResultsCollectorBase::FailStatus::FAILED: statusMsg = "FAILED"; break;
 451             case TestResultsCollectorBase::FailStatus::IGNORED: statusMsg = "IGNORED"; break;
 452             case TestResultsCollectorBase::FailStatus::TODO:  statusMsg = "TODO"; break;
 453             case TestResultsCollectorBase::FailStatus::INTERNAL: statusMsg = "FAILED"; break;
 454             default:
 455                 Assert(false && "Bad status");
 456         }
 457         printf("%s;%s;%s;%s\n",
 458                m_currentGroup.c_str(),
 459                id.c_str(),
 460                statusMsg.c_str(),
 461                reason.c_str());
 462     }
 463
 464     std::string m_currentGroup;
 465 };
 466
 467
 468 TestResultsCollectorBase* CSVCollector::Constructor()
 469 {
 470     return new CSVCollector();
 471 }
 472
 473 }
 474
 475 void TestResultsCollectorBase::RegisterCollectorConstructor(
 476     const std::string& name,
 477     TestResultsCollectorBase::CollectorConstructorFunc func)
 478 {
 479     Assert(m_constructorsMap.find(name) == m_constructorsMap.end());
 480     m_constructorsMap[name] = func;
 481 }
 482
 483 TestResultsCollectorBase* TestResultsCollectorBase::Create(
 484     const std::string& name)
 485 {
 486     ConstructorsMap::iterator found = m_constructorsMap.find(name);
 487     if (found != m_constructorsMap.end())
 488         return found->second();
 489     else
 490         return NULL;
 491 }
 492
 493 std::vector<std::string> TestResultsCollectorBase::GetCollectorsNames()
 494 {
 495     std::vector<std::string> list;
 496     FOREACH(it, m_constructorsMap)
 497     {
 498         list.push_back(it->first);
 499     }
 500     return list;
 501 }
 502
 503 TestResultsCollectorBase::ConstructorsMap TestResultsCollectorBase::m_constructorsMap;
 504
 505 namespace
 506 {
 507 static int RegisterCollectorConstructors();
 508 static const int RegisterHelperVariable = RegisterCollectorConstructors();
 509 int RegisterCollectorConstructors()
 510 {
 511     (void)RegisterHelperVariable;
 512
 513     TestResultsCollectorBase::RegisterCollectorConstructor(
 514         "text",
 515         &ConsoleCollector::Constructor);
 516     TestResultsCollectorBase::RegisterCollectorConstructor(
 517         "html",
 518         &HtmlCollector::Constructor);
 519     TestResultsCollectorBase::RegisterCollectorConstructor(
 520         "csv",
 521         &CSVCollector::Constructor);
 522
 523     return 0;
 524 }
 525
 526 }
 527
 528 }
 529 }