Collect test fail reasons in case of nested assertions

RUNNER_ASSERT_MSG macro (and related) throw DPL::Test::TestRunner::TestFailed
exception. If such an exception is thrown during clean-up caused by another
exception and test interruption, termination of tests program happen leaving
no clues about termination cause.

This patch checks if an exception has been thrown earlier and if so,
it doesn't throw next exception but passes exception message to TestRunner.
All such messages passed with addFailReason() method are gathered inside
TestRunner and used during ResultCollection after finishing test and passing
results to collectors.

This patch introduces simple concatrenation method of joining all fail
reasons into single string, that is passed to Collectors.
When collectors API will be ready to receive multiple reasons of single test
failure, another patch can simply pass whole fail results queue.

Change-Id: I1bf582da6526a4187e7c4a53afaeea976cf4bda1

diff --git a/README b/README
index 9caea47..65b2928 100644 (file)
--- a/README
+++ b/README
@@ -114,6 +114,10 @@ tests-common
 --Assert macros----------------------------------------------------------------
 Used within test registering macros.
 
+First failed assertion throws test failed exception. If another assertions
+fail, information about fail conditions and backtrace is cumulated and
+presented together with already thrown exception message.
+
 dpl-test-framework
   test_runner.h
     RUNNER_ASSERT_MSG

diff --git a/tests/framework/include/dpl/test/test_runner.h b/tests/framework/include/dpl/test/test_runner.h
index 70117e3..df6f54d 100644 (file)
--- a/tests/framework/include/dpl/test/test_runner.h
+++ b/tests/framework/include/dpl/test/test_runner.h
@@ -26,9 +26,11 @@
 
 #include <chrono>
 #include <cstring>
+#include <exception>
 #include <iostream>
 #include <list>
 #include <map>
+#include <queue>
 #include <set>
 #include <sstream>
 #include <string>
@@ -52,6 +54,8 @@ class TestRunner
     std::string m_startTestId;
     bool m_runIgnored;
 
+    std::queue<std::string> m_failReason;
+
   public:
     TestRunner() :
         m_currentTestCase(nullptr)
@@ -68,6 +72,8 @@ class TestRunner
                                              std::chrono::system_clock::duration result,
                                              std::chrono::system_clock::duration resultMax);
 
+    void addFailReason(const std::string &reason);
+
     typedef void (*TestCase)();
 
   private:
@@ -135,6 +141,8 @@ class TestRunner
 
     void RunTests();
 
+    std::string getConcatedFailReason(const std::string &reason);
+
     void CollectResult(const std::string& id,
                        const std::string& description,
                        const TestResultsCollectorBase::FailStatus::Type status
@@ -237,40 +245,46 @@ typedef DPL::Singleton<TestRunner> TestRunnerSingleton;
  * body. Failing assertion indicates failing test.
  */
 
-#define RUNNER_ASSERT_MSG(test, message)                              \
-    do                                                                \
-    {                                                                 \
-        DPL::Test::TestRunnerSingleton::Instance().MarkAssertion();   \
-                                                                      \
-        if (!(test))                                                  \
-        {                                                             \
-            std::ostringstream assertMsg;                             \
-            assertMsg << message << gdbbacktrace();                   \
-            throw DPL::Test::TestRunner::TestFailed(#test,            \
-                                                    __FILE__,         \
-                                                    __LINE__,         \
-                                                    assertMsg.str()); \
-        }                                                             \
+#define RUNNER_ASSERT_MSG(test, message)                                              \
+    do                                                                                \
+    {                                                                                 \
+        DPL::Test::TestRunnerSingleton::Instance().MarkAssertion();                   \
+                                                                                      \
+        if (!(test))                                                                  \
+        {                                                                             \
+            std::ostringstream assertMsg;                                             \
+            assertMsg << message << gdbbacktrace();                                   \
+            DPL::Test::TestRunner::TestFailed e(#test,                                \
+                                                __FILE__,                             \
+                                                __LINE__,                             \
+                                                assertMsg.str());                     \
+            if (!std::uncaught_exception())                                           \
+                throw e;                                                              \
+            DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
+        }                                                                             \
     } while (0)
 
-#define RUNNER_ASSERT_ERRNO_MSG(test, message)                        \
-    do                                                                \
-    {                                                                 \
-        DPL::Test::TestRunnerSingleton::Instance().MarkAssertion();   \
-                                                                      \
-        if (!(test))                                                  \
-        {                                                             \
-            const char *err = strerror(errno);                        \
-            std::ostringstream assertMsg;                             \
-            assertMsg << message;                                     \
-            if (!assertMsg.str().empty())                             \
-                assertMsg << ". ";                                    \
-            assertMsg << err << gdbbacktrace();                       \
-            throw DPL::Test::TestRunner::TestFailed(#test,            \
-                                                    __FILE__,         \
-                                                    __LINE__,         \
-                                                    assertMsg.str()); \
-        }                                                             \
+#define RUNNER_ASSERT_ERRNO_MSG(test, message)                                        \
+    do                                                                                \
+    {                                                                                 \
+        DPL::Test::TestRunnerSingleton::Instance().MarkAssertion();                   \
+                                                                                      \
+        if (!(test))                                                                  \
+        {                                                                             \
+            const char *err = strerror(errno);                                        \
+            std::ostringstream assertMsg;                                             \
+            assertMsg << message;                                                     \
+            if (!assertMsg.str().empty())                                             \
+                assertMsg << ". ";                                                    \
+            assertMsg << err << gdbbacktrace();                                       \
+            DPL::Test::TestRunner::TestFailed e(#test,                                \
+                                                __FILE__,                             \
+                                                __LINE__,                             \
+                                                assertMsg.str());                     \
+            if (!std::uncaught_exception())                                           \
+                throw e;                                                              \
+            DPL::Test::TestRunnerSingleton::Instance().addFailReason(e.GetMessage()); \
+        }                                                                             \
     } while (0)
 
 #define RUNNER_ASSERT_ERRNO(test) \

diff --git a/tests/framework/src/test_runner.cpp b/tests/framework/src/test_runner.cpp
index af76536..52060aa 100644 (file)
--- a/tests/framework/src/test_runner.cpp
+++ b/tests/framework/src/test_runner.cpp
@@ -253,7 +253,7 @@ TestRunner::Status TestRunner::RunTestCase(const TestCaseStruct& testCase)
         CollectResult(testCase.name,
                       "",
                       TestResultsCollectorBase::FailStatus::FAILED,
-                      e.GetMessage());
+                      getConcatedFailReason(e.GetMessage()));
 
         setCurrentTestCase(nullptr);
         return FAILED;
@@ -426,6 +426,21 @@ void TestRunner::setCurrentTestCasePerformanceResult(bool isPerformanceTest,
     testCase->m_performanceMaxTime = resultMax;
 }
 
+void TestRunner::addFailReason(const std::string &reason)
+{
+    m_failReason.push(reason);
+}
+
+std::string TestRunner::getConcatedFailReason(const std::string &reason)
+{
+    std::string ret;
+    while (!m_failReason.empty())
+    {
+        ret += m_failReason.front();
+        m_failReason.pop();
+    }
+    return reason + ret;
+}
 
 void TestRunner::CollectResult(
     const std::string& id,