testlib: Improve the silent logging mode

Previously the silent logging mode suppressed passes, skips and internal
testlib info messages, but did not suppress debugging output, making it
hard to see the fails in a noisy test.  This commit changes silent mode
so that it suppresses all output except test failures and fatal errors,
making silent mode truly useful for seeing just the important test
output.

This commit also adds a selftest to verify the behaviour of silent mode.

Change-Id: I75420aead03682306210746a87e2a3b608b58fc6
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>

diff --git a/src/testlib/qplaintestlogger.cpp b/src/testlib/qplaintestlogger.cpp
index ca86387..2f87e08 100644 (file)
--- a/src/testlib/qplaintestlogger.cpp
+++ b/src/testlib/qplaintestlogger.cpp
@@ -371,8 +371,9 @@ void QPlainTestLogger::leaveTestFunction()
 void QPlainTestLogger::addIncident(IncidentTypes type, const char *description,
                                    const char *file, int line)
 {
-    // suppress PASS in silent mode
-    if (type == QAbstractTestLogger::Pass && QTestLog::verboseLevel() < 0)
+    // suppress PASS and XFAIL in silent mode
+    if ((type == QAbstractTestLogger::Pass || type == QAbstractTestLogger::XFail)
+        && QTestLog::verboseLevel() < 0)
         return;
 
     printMessage(QTest::incidentType2String(type), description, file, line);
@@ -380,15 +381,18 @@ void QPlainTestLogger::addIncident(IncidentTypes type, const char *description,
 
 void QPlainTestLogger::addBenchmarkResult(const QBenchmarkResult &result)
 {
+    // suppress benchmark results in silent mode
+    if (QTestLog::verboseLevel() < 0)
+        return;
+
     printBenchmarkResult(result);
 }
 
 void QPlainTestLogger::addMessage(MessageTypes type, const char *message,
                                   const char *file, int line)
 {
-    // suppress PASS in silent mode
-    if ((type == QAbstractTestLogger::Skip || type == QAbstractTestLogger::Info)
-       && QTestLog::verboseLevel() < 0)
+    // suppress non-fatal messages in silent mode
+    if (type != QAbstractTestLogger::QFatal && QTestLog::verboseLevel() < 0)
         return;
 
     printMessage(QTest::messageType2String(type), message, file, line);

diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index b8678ba..a4f1a39 100644 (file)
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -1192,7 +1192,7 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml)
          "     *** If no output format is specified, -txt is assumed.\n"
          "\n"
          " Detail options:\n"
-         " -silent             : Only outputs warnings and failures\n"
+         " -silent             : Only output failures and fatal errors in plain text output\n"
          " -v1                 : Print enter messages for each testfunction\n"
          " -v2                 : Also print out each QVERIFY/QCOMPARE/QTEST\n"
          " -vs                 : Print every signal emitted\n"

diff --git a/tests/auto/testlib/selftests/expected_silent.txt b/tests/auto/testlib/selftests/expected_silent.txt
new file mode 100644 (file)
index 0000000..39e0bfc
--- /dev/null
+++ b/tests/auto/testlib/selftests/expected_silent.txt
@@ -0,0 +1,9 @@
+Testing tst_Silent
+FAIL!  : tst_Silent::fail() 'false' returned FALSE. (This test should fail)
+   Loc: [/home/jasmcdon/depot/qt5-test/qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp(73)]
+XPASS  : tst_Silent::xpass() 'true' returned TRUE unexpectedly. (This test should XPASS)
+   Loc: [/home/jasmcdon/depot/qt5-test/qtbase/tests/auto/testlib/selftests/silent/tst_silent.cpp(85)]
+QFATAL : tst_Silent::messages() This is a fatal error message that should still appear in silent test output
+FAIL!  : tst_Silent::messages() Received a fatal error.
+   Loc: [Unknown file(0)]
+Totals: 3 passed, 3 failed, 1 skipped

diff --git a/tests/auto/testlib/selftests/selftests.pri b/tests/auto/testlib/selftests/selftests.pri
index 0809c4d..1fc66e6 100644 (file)
--- a/tests/auto/testlib/selftests/selftests.pri
+++ b/tests/auto/testlib/selftests/selftests.pri
@@ -30,6 +30,7 @@ SUBPROGRAMS = \
      printdatatags \
      printdatatagswithglobaltags \
      qexecstringlist \
+     silent \
      singleskip \
      skip \
      skipcleanup \

diff --git a/tests/auto/testlib/selftests/selftests.qrc b/tests/auto/testlib/selftests/selftests.qrc
index baa539a..d05ac2a 100644 (file)
--- a/tests/auto/testlib/selftests/selftests.qrc
+++ b/tests/auto/testlib/selftests/selftests.qrc
@@ -93,6 +93,7 @@
         <file>expected_printdatatags.txt</file>
         <file>expected_printdatatagswithglobaltags.txt</file>
         <file>expected_qexecstringlist.txt</file>
+        <file>expected_silent.txt</file>
         <file>expected_singleskip.lightxml</file>
         <file>expected_singleskip.txt</file>
         <file>expected_singleskip.xml</file>

diff --git a/tests/auto/testlib/selftests/silent/silent.pro b/tests/auto/testlib/selftests/silent/silent.pro
new file mode 100644 (file)
index 0000000..3150f65
--- /dev/null
+++ b/tests/auto/testlib/selftests/silent/silent.pro
@@ -0,0 +1,7 @@
+SOURCES += tst_silent.cpp
+QT = core testlib-private
+
+mac:CONFIG -= app_bundle
+CONFIG -= debug_and_release_target
+
+TARGET = silent

diff --git a/tests/auto/testlib/selftests/silent/tst_silent.cpp b/tests/auto/testlib/selftests/silent/tst_silent.cpp
new file mode 100644 (file)
index 0000000..ec280f9
--- /dev/null
+++ b/tests/auto/testlib/selftests/silent/tst_silent.cpp
@@ -0,0 +1,99 @@
+/****************************************************************************
+**
+** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** GNU Lesser General Public License Usage
+** This file may be used under the terms of the GNU Lesser General Public
+** License version 2.1 as published by the Free Software Foundation and
+** appearing in the file LICENSE.LGPL included in the packaging of this
+** file. Please review the following information to ensure the GNU Lesser
+** General Public License version 2.1 requirements will be met:
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights. These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU General
+** Public License version 3.0 as published by the Free Software Foundation
+** and appearing in the file LICENSE.GPL included in the packaging of this
+** file. Please review the following information to ensure the GNU General
+** Public License version 3.0 requirements will be met:
+** http://www.gnu.org/copyleft/gpl.html.
+**
+** Other Usage
+** Alternatively, this file may be used in accordance with the terms and
+** conditions contained in a signed written agreement between you and Nokia.
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include <QtCore/QCoreApplication>
+#include <QtTest/QtTest>
+#include <private/qtestlog_p.h>
+
+class tst_Silent : public QObject
+{
+    Q_OBJECT
+
+private slots:
+    void pass();
+    void skip();
+    void fail();
+    void xfail();
+    void xpass();
+
+    // This test function must be last, as it calls qFatal().
+    void messages();
+};
+
+void tst_Silent::pass()
+{
+    QVERIFY(true);
+}
+
+void tst_Silent::skip()
+{
+    QSKIP("This test should skip");
+}
+
+void tst_Silent::fail()
+{
+    QVERIFY2(false, "This test should fail");
+}
+
+void tst_Silent::xfail()
+{
+    QEXPECT_FAIL("", "This test should XFAIL", Abort);
+    QVERIFY(false);
+}
+
+void tst_Silent::xpass()
+{
+    QEXPECT_FAIL("", "This test should XPASS", Abort);
+    QVERIFY2(true, "This test should XPASS");
+}
+
+void tst_Silent::messages()
+{
+    qWarning("This is a warning that should not appear in silent test output");
+    QWARN("This is an internal testlib warning that should not appear in silent test output");
+    qDebug("This is a debug message that should not appear in silent test output");
+    qCritical("This is a critical message that should not appear in silent test output");
+    QTestLog::info("This is an internal testlib info message that should not appear in silent test output", __FILE__, __LINE__);
+    qFatal("This is a fatal error message that should still appear in silent test output");
+}
+
+QTEST_MAIN(tst_Silent)
+#include "tst_silent.moc"

diff --git a/tests/auto/testlib/selftests/tst_selftests.cpp b/tests/auto/testlib/selftests/tst_selftests.cpp
index 7367d66..2b90bdb 100644 (file)
--- a/tests/auto/testlib/selftests/tst_selftests.cpp
+++ b/tests/auto/testlib/selftests/tst_selftests.cpp
@@ -354,6 +354,7 @@ void tst_Selftests::runSubTest_data()
         << "printdatatags"
         << "printdatatagswithglobaltags"
         << "qexecstringlist"
+        << "silent"
         << "singleskip"
         << "skip"
         << "skipcleanup"
@@ -409,6 +410,9 @@ void tst_Selftests::runSubTest_data()
             else if (subtest == "printdatatagswithglobaltags") {
                 arguments << "-datatags";
             }
+            else if (subtest == "silent") {
+                arguments << "-silent";
+            }
 
 
             // These tests don't work right unless logging plain text to
@@ -434,6 +438,9 @@ void tst_Selftests::runSubTest_data()
                 if (subtest == "printdatatagswithglobaltags") {
                     continue;
                 }
+                if (subtest == "silent") {
+                    continue;
+                }
                 // `crashes' will not output valid XML on platforms without a crash handler
                 if (subtest == "crashes") {
                     continue;