tizen beta release
[framework/web/wrt-commons.git] / modules / test / src / test_runner.cpp
1 /*
2  * Copyright (c) 2011 Samsung Electronics Co., Ltd All Rights Reserved
3  *
4  *    Licensed under the Apache License, Version 2.0 (the "License");
5  *    you may not use this file except in compliance with the License.
6  *    You may obtain a copy of the License at
7  *
8  *        http://www.apache.org/licenses/LICENSE-2.0
9  *
10  *    Unless required by applicable law or agreed to in writing, software
11  *    distributed under the License is distributed on an "AS IS" BASIS,
12  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  *    See the License for the specific language governing permissions and
14  *    limitations under the License.
15  */
16 /*
17  * @file        test_runner.cpp
18  * @author      Przemyslaw Dobrowolski (p.dobrowolsk@samsung.com)
19  * @author      Lukasz Wrzosek (l.wrzosek@samsung.com)
20  * @version     1.0
21  * @brief       This file is the implementation file of test runner
22  */
23 #include <dpl/test/test_runner.h>
24 #include <dpl/test/test_results_collector.h>
25 #include <dpl/exception.h>
26 #include <dpl/scoped_free.h>
27 #include <dpl/foreach.h>
28 #include <dpl/log/log.h>
29 #include <dpl/colors.h>
30 #include <pcrecpp.h>
31 #include <algorithm>
32 #include <cstdio>
33 #include <memory.h>
34 #include <libgen.h>
35 #include <cstring>
36
37 #include <dpl/singleton_impl.h>
38 IMPLEMENT_SINGLETON(DPL::Test::TestRunner)
39
40 namespace DPL
41 {
42 namespace Test
43 {
44
45 namespace // anonymous
46 {
47
48 std::string BaseName(std::string aPath)
49 {
50     ScopedFree<char> path(strdup(aPath.c_str()));
51     if (NULL == path.Get())
52     {
53        throw std::bad_alloc();
54     }
55     char* baseName = basename(path.Get());
56     std::string retValue = baseName;
57     return retValue;
58 }
59
60 } // namespace anonymous
61
62 //! \brief Failed test message creator
63 //!
64 //! \param[in] aTest string for tested expression
65 //! \param[in] aFile source file name
66 //! \param[in] aLine source file line
67 //! \param[in] aMessage error message
68 TestRunner::TestFailed::TestFailed(const char* aTest,
69                                    const char* aFile,
70                                    int aLine,
71                                    const std::string &aMessage)
72 {
73     std::ostringstream assertMsg;
74     assertMsg << "[" << BaseName(aFile) << ":" << aLine
75               << "] Assertion failed ("
76               << aTest << ") " << aMessage;
77     m_message = assertMsg.str();
78 }
79
80 void TestRunner::RegisterTest(const char *testName, TestCase proc)
81 {
82     m_testGroups[m_currentGroup].push_back(TestCaseStruct(testName, proc));
83 }
84
85 void TestRunner::InitGroup(const char* name)
86 {
87     m_currentGroup = name;
88 }
89
90
91 TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase)
92 {
93     try
94     {
95         testCase.proc();
96     }
97     catch (const TestFailed &e)
98     {
99         // Simple test failure
100         m_collector->CollectResult(testCase.name,
101                                    "",
102                                    TestResultsCollectorBase::FailStatus::FAILED,
103                                    e.GetMessage());
104         return FAILED;
105     }
106     catch (const Ignored &e)
107     {
108         // Simple test have to be implemented
109         m_collector->CollectResult(testCase.name,
110                                    "",
111                                    TestResultsCollectorBase::FailStatus::IGNORED,
112                                    e.GetMessage());
113
114         return IGNORED;
115     }
116     catch (const ToDo &e)
117     {
118         // Simple test have to be implemented
119         m_collector->CollectResult(testCase.name,
120                                    "",
121                                    TestResultsCollectorBase::FailStatus::TODO,
122                                    e.GetMessage());
123
124         return TODO;
125     }
126     catch (const DPL::Exception &e)
127     {
128         // DPL exception failure
129         m_collector->CollectResult(testCase.name,
130                                    "",
131                                    TestResultsCollectorBase::FailStatus::INTERNAL,
132                                    "DPL exception:" +
133                                    e.GetMessage());
134
135         return FAILED;
136     }
137     catch (const std::exception &)
138     {
139         // std exception failure
140         m_collector->CollectResult(testCase.name,
141                                    "",
142                                    TestResultsCollectorBase::FailStatus::INTERNAL,
143                                    "std exception");
144
145         return FAILED;
146     }
147     catch (...)
148     {
149         // Unknown exception failure
150         m_collector->CollectResult(testCase.name,
151                                    "",
152                                    TestResultsCollectorBase::FailStatus::INTERNAL,
153                                    "unknown exception");
154
155         return FAILED;
156     }
157
158     m_collector->CollectResult(testCase.name,
159                                "",
160                                TestResultsCollectorBase::FailStatus::NONE);
161     // Everything OK
162     return PASS;
163 }
164
165 void TestRunner::RunTests()
166 {
167     using namespace DPL::Colors::Text;
168
169     Banner();
170     m_collector->Start();
171
172     fprintf(stderr, "%s%s%s\n", GREEN_BEGIN, "Running tests...", GREEN_END);
173     FOREACH(group, m_testGroups) {
174         TestCaseStructList list = group->second;
175         if (!list.empty()) {
176             m_collector->CollectCurrentTestGroupName(group->first);
177             list.sort();
178
179             for (TestCaseStructList::const_iterator iterator = list.begin();
180                  iterator != list.end();
181                  ++iterator)
182             {
183                 TestCaseStruct test = *iterator;
184                 if (m_startTestId == test.name)
185                     m_startTestId = "";
186
187                 if (m_startTestId.empty()) {
188                     RunTestCase(test);
189                 }
190             }
191         }
192     }
193
194     m_collector->Finish();
195     // Finished
196     fprintf(stderr, "%s%s%s\n\n", GREEN_BEGIN, "Finished", GREEN_END);
197 }
198
199 void TestRunner::Banner()
200 {
201     using namespace DPL::Colors::Text;
202     fprintf(stderr,
203             "%s%s%s\n",
204             BOLD_GREEN_BEGIN,
205             "DPL tests runner",
206             BOLD_GREEN_END);
207     fprintf(stderr,
208             "%s%s%s%s\n\n",
209             GREEN_BEGIN,
210             "Build: ",
211             __TIMESTAMP__,
212             GREEN_END);
213 }
214
215 void TestRunner::InvalidArgs()
216 {
217     using namespace DPL::Colors::Text;
218     fprintf(stderr,
219             "%s%s%s\n",
220             BOLD_RED_BEGIN,
221             "Invalid parameters!",
222             BOLD_RED_END);
223 }
224
225 void TestRunner::Usage()
226 {
227     fprintf(stderr, "Usage: runner [options]\n\n");
228     fprintf(stderr, "Output type:\n");
229     fprintf(stderr, "  --output=<output type>\n");
230     fprintf(stderr, "  possible output types:\n");
231     FOREACH (type, TestResultsCollectorBase::GetCollectorsNames()) {
232         fprintf(stderr, "  --output=%s\n", type->c_str());
233     }
234     fprintf(stderr, "Other parameters:\n");
235     fprintf(stderr,
236             "  --regexp='regexp'\t Only selected tests"
237             " which names match regexp run\n\n");
238     fprintf(stderr, "  --start=<test id>\tStart from concrete test id");
239     fprintf(stderr, "  --group=<group name>\t Run tests only from one group\n");
240     fprintf(stderr, "  --list\t Show a list of Test IDs\n");
241     fprintf(stderr, "  --listgroups\t Show a list of Test Group names \n");
242     fprintf(stderr, "  --listingroup=<group name>\t Show a list of Test IDS in one group\n");
243     fprintf(stderr, "  --help\t This help\n\n");
244     if (m_collector) {
245         fprintf(stderr, "Output %s has specific args:\n", m_collectorName.c_str());
246         fprintf(stderr, "%s\n", m_collector->CollectorSpecificHelp().c_str());
247     }
248     fprintf(stderr, "For bug reporting, please write to:\n");
249     fprintf(stderr, "<p.dobrowolsk@samsung.com>\n");
250 }
251
252 int TestRunner::ExecTestRunner(int argc, char *argv[])
253 {
254     std::vector<std::string> args;
255     for (int i = 0; i < argc; ++i)
256     {
257         args.push_back(argv[i]);
258     }
259     return ExecTestRunner(args);
260 }
261
262 void TestRunner::MarkAssertion()
263 {
264     ++m_totalAssertions;
265 }
266
267 int TestRunner::ExecTestRunner(const ArgsList& value)
268 {
269     ArgsList args = value;
270     // Parse command line
271     if (args.size() == 1)
272     {
273         InvalidArgs();
274         Usage();
275         return -1;
276     }
277
278     args.erase(args.begin());
279
280     bool showHelp = false;
281
282     // Parse each argument
283     FOREACH(it, args)
284     {
285         std::string arg = *it;
286         const std::string regexp = "--regexp=";
287         const std::string output = "--output=";
288         const std::string groupId = "--group=";
289         const std::string listCmd = "--list";
290         const std::string startCmd = "--start=";
291         const std::string listGroupsCmd = "--listgroups";
292         const std::string listInGroup = "--listingroup=";
293
294         if (m_collector && m_collector->ParseCollectorSpecificArg(arg)) continue;
295         else if (arg.find(startCmd) == 0)
296         {
297             arg.erase(0, startCmd.length());
298             FOREACH(group, m_testGroups) {
299                 FOREACH(tc, group->second) {
300                     if (tc->name == arg) {
301                         m_startTestId = arg;
302                         break;
303                     }
304                 }
305                 if (!m_startTestId.empty()) break;
306             }
307             if (!m_startTestId.empty()) continue;
308             InvalidArgs();
309             fprintf(stderr, "Start test id has not been found\n");
310             Usage();
311             return 0;
312         }
313         else if (arg.find(groupId) == 0)
314         {
315             arg.erase(0, groupId.length());
316             TestCaseGroupMap::iterator found = m_testGroups.find(arg);
317             if (found != m_testGroups.end()) {
318                 std::string name = found->first;
319                 TestCaseStructList newList = found->second;
320                 m_testGroups.clear();
321                 m_testGroups[name] = newList;
322             } else {
323                 fprintf(stderr, "Group %s not found\n", arg.c_str());
324                 InvalidArgs();
325                 Usage();
326                 return -1;
327             }
328         }
329         else if (arg == listCmd)
330         {
331             FOREACH(group, m_testGroups) {
332                 FOREACH(test, group->second) {
333                     printf("ID:%s:%s\n", group->first.c_str(), test->name.c_str());
334                 }
335             }
336             return 0;
337         }
338         else if (arg == listGroupsCmd)
339         {
340             FOREACH(group, m_testGroups) {
341                 printf("GR:%s\n", group->first.c_str());
342             }
343             return 0;
344         }
345         else if (arg.find(listInGroup) == 0)
346         {
347             arg.erase(0, listInGroup.length());
348             FOREACH(test, m_testGroups[arg]) {
349                 printf("ID:%s\n", test->name.c_str());
350             }
351             return 0;
352         }
353         else if (arg == "--help")
354             showHelp = true;
355         else if (arg.find(output) == 0)
356         {
357             arg.erase(0, output.length());
358             m_collector.Reset(TestResultsCollectorBase::Create(arg));
359             if (!m_collector) {
360                 InvalidArgs();
361                 Usage();
362                 return -1;
363             } else {
364                 m_collectorName = arg;
365             }
366         }
367         else if (arg.find(regexp) == 0)
368         {
369             arg.erase(0, regexp.length());
370             if (arg.length() == 0)
371             {
372                 InvalidArgs();
373                 Usage();
374                 return -1;
375             }
376
377             if (arg[0] == '\'' && arg[arg.length() - 1] == '\'')
378             {
379                 arg.erase(0);
380                 arg.erase(arg.length() - 1);
381             }
382
383             if (arg.length() == 0)
384             {
385                 InvalidArgs();
386                 Usage();
387                 return -1;
388             }
389
390             pcrecpp::RE re(arg.c_str());
391             FOREACH(group, m_testGroups) {
392                 TestCaseStructList newList;
393                 FOREACH(iterator, group->second)
394                 {
395                     if (re.PartialMatch(iterator->name))
396                     {
397                         newList.push_back(*iterator);
398                     }
399                 }
400                 group->second = newList;
401             }
402         }
403         else
404         {
405             InvalidArgs();
406             Usage();
407             return -1;
408         }
409     }
410
411     // Show help
412     if (showHelp)
413     {
414         Usage();
415         return 0;
416     }
417
418     if (!m_collector)
419         m_collector.Reset(TestResultsCollectorBase::Create("text"));
420
421     if (!m_collector->Configure()) {
422         fprintf(stderr, "Could not configure selected output");
423         return 0;
424     }
425
426     // Run tests
427     RunTests();
428
429     return 0;
430 }
431
432 }
433 } // namespace DPL