From 016cd01846539404a0eff1e3823fa2206fe2c9a9 Mon Sep 17 00:00:00 2001 From: Jason McDonald Date: Tue, 18 Oct 2011 14:53:23 +1000 Subject: [PATCH] Remove QTest::SkipMode from qtestlib API. When the SkipAll mode is used, tests only report a SKIP for the first line of test data and subsequent lines are not reported at all. This behaviour makes it impossible for anything post-processing test results to accurately report test pass- and run- rates because they cannot see how many lines of test data were skipped. This commit removes SkipMode. QSKIPs in regular test functions and data functions are treated the same as SkipSingle, so that every skipped line of local or global test data is reported in the test log. QSKIPs elsewhere are treated the same as SkipAll -- skipping in init() causes the next test function to be skipped entirely, and skipping in initTestCase() or initTestCase_data() causes all test functions to be skipped. This commit only changes qtestlib and the selftests. A further commit will change the autotests to remove the SkipMode parameter from QSKIP calls. Note that the change in expected output for the globaldata selftest is deliberate, as the QSKIP in the skipLocal test function has effectively changed from SkipAll to SkipSingle. Task-number: QTBUG-21851, QTBUG-21652 Change-Id: I7b1c53fe7ca9dde032810b789d967e2a402bbe5d Reviewed-by: Rohan McGovern Reviewed-by: Alex --- doc/src/snippets/code/src_qtestlib_qtestcase.cpp | 2 +- src/testlib/qtest_global.h | 1 - src/testlib/qtestcase.cpp | 47 +++++----------------- src/testlib/qtestcase.h | 11 +++-- .../benchlibcallgrind/tst_benchlibcallgrind.cpp | 2 +- .../tst_benchlibtickcounter.cpp | 2 +- .../testlib/selftests/expected_globaldata.lightxml | 12 ++++++ .../auto/testlib/selftests/expected_globaldata.txt | 6 ++- .../auto/testlib/selftests/expected_globaldata.xml | 12 ++++++ .../testlib/selftests/expected_globaldata.xunitxml | 8 +++- .../auto/testlib/selftests/expected_singleskip.txt | 2 +- tests/auto/testlib/selftests/expected_skipinit.txt | 2 +- .../selftests/expected_skipinitdata.lightxml | 2 +- .../testlib/selftests/expected_skipinitdata.txt | 4 +- .../testlib/selftests/expected_skipinitdata.xml | 2 +- .../selftests/expected_skipinitdata.xunitxml | 4 +- .../selftests/globaldata/tst_globaldata.cpp | 6 +-- .../selftests/singleskip/tst_singleskip.cpp | 2 +- tests/auto/testlib/selftests/skip/tst_skip.cpp | 6 +-- .../testlib/selftests/skipinit/tst_skipinit.cpp | 2 +- .../selftests/skipinitdata/tst_skipinitdata.cpp | 2 +- tests/auto/testlib/selftests/xunit/tst_xunit.cpp | 2 +- 22 files changed, 75 insertions(+), 64 deletions(-) diff --git a/doc/src/snippets/code/src_qtestlib_qtestcase.cpp b/doc/src/snippets/code/src_qtestlib_qtestcase.cpp index 5b1f558..75d0dd5 100644 --- a/doc/src/snippets/code/src_qtestlib_qtestcase.cpp +++ b/doc/src/snippets/code/src_qtestlib_qtestcase.cpp @@ -100,7 +100,7 @@ QTEST(QString("hello").toUpper(), "myString"); //! [8] if (!QSqlDatabase::drivers().contains("SQLITE")) - QSKIP("This test requires the SQLITE database driver", SkipAll); + QSKIP("This test requires the SQLITE database driver"); //! [8] diff --git a/src/testlib/qtest_global.h b/src/testlib/qtest_global.h index 974d82a..6789c59 100644 --- a/src/testlib/qtest_global.h +++ b/src/testlib/qtest_global.h @@ -77,7 +77,6 @@ QT_MODULE(Test) namespace QTest { - enum SkipMode { SkipSingle = 1, SkipAll = 2 }; enum TestFailMode { Abort = 1, Continue = 2 }; int Q_TESTLIB_EXPORT qt_snprintf(char *str, int size, const char *format, ...); diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp index 451fa5b..0ff760b 100644 --- a/src/testlib/qtestcase.cpp +++ b/src/testlib/qtestcase.cpp @@ -267,23 +267,24 @@ QT_BEGIN_NAMESPACE \sa QCOMPARE() */ -/*! \macro QSKIP(description, mode) +/*! \macro QSKIP(description) \relates QTest The QSKIP() macro stops execution of the test without adding a failure to the test log. You can use it to skip tests that wouldn't make sense in the current configuration. The text \a description is appended to the test log and should - contain an explanation why the test couldn't be executed. \a mode is a QTest::SkipMode - and describes whether to proceed with the rest of the test data or not. + contain an explanation of why the test couldn't be executed. + + If the test is data-driven, each call to QSKIP() will skip only the current row, + so an unconditional call to QSKIP will produce one skip message in the test log + for each row of test data. \bold {Note:} This macro can only be used in a test function that is invoked by the test framework. Example: \snippet doc/src/snippets/code/src_qtestlib_qtestcase.cpp 8 - - \sa QTest::SkipMode */ /*! \macro QEXPECT_FAIL(dataIndex, comment, mode) @@ -415,22 +416,6 @@ QT_BEGIN_NAMESPACE {Chapter 5: Writing a Benchmark}{Writing a Benchmark} */ - - -/*! \enum QTest::SkipMode - - This enum describes the modes for skipping tests during execution - of the test data. - - \value SkipSingle Skips the current entry in the test table; continues - execution of all the other entries in the table. - - \value SkipAll Skips all the entries in the test table; the test won't - be executed further. - - \sa QSKIP() -*/ - /*! \enum QTest::TestFailMode This enum describes the modes for handling an expected failure of the @@ -1469,20 +1454,12 @@ static bool qInvokeTestMethod(const char *slotName, const char *data=0) QTestResult::setCurrentTestLocation(QTestResult::DataFunc); QTest::qt_snprintf(member, 512, "%s_data()", slot); invokeMethod(QTest::currentTestObject, member); - - // if we encounter a SkipAll in the _data slot, we skip the whole - // testfunction, no matter how much global data exists - if (QTestResult::skipCurrentTest()) { - QTestResult::setCurrentGlobalTestData(0); - break; - } } bool foundFunction = false; if (!QTestResult::skipCurrentTest()) { int curDataIndex = 0; const int dataCount = table.dataCount(); - QTestResult::setSkipCurrentTest(false); // Data tag requested but none available? if (data && !dataCount) { @@ -1498,6 +1475,7 @@ static bool qInvokeTestMethod(const char *slotName, const char *data=0) /* For each entry in the data table, do: */ do { + QTestResult::setSkipCurrentTest(false); if (!data || !qstrcmp(data, table.testData(curDataIndex)->dataTag())) { foundFunction = true; QTestDataSetter s(curDataIndex >= dataCount ? static_cast(0) @@ -1505,9 +1483,6 @@ static bool qInvokeTestMethod(const char *slotName, const char *data=0) qInvokeTestMethodDataEntry(slot); - if (QTestResult::skipCurrentTest()) - // check whether SkipAll was requested - break; if (data) break; } @@ -1954,15 +1929,13 @@ bool QTest::qVerify(bool statement, const char *statementStr, const char *descri return QTestResult::verify(statement, statementStr, description, file, line); } -/*! \fn void QTest::qSkip(const char *message, SkipMode mode, const char *file, int line) +/*! \fn void QTest::qSkip(const char *message, const char *file, int line) \internal */ -void QTest::qSkip(const char *message, QTest::SkipMode mode, - const char *file, int line) +void QTest::qSkip(const char *message, const char *file, int line) { QTestResult::addSkip(message, file, line); - if (mode == QTest::SkipAll) - QTestResult::setSkipCurrentTest(true); + QTestResult::setSkipCurrentTest(true); } /*! \fn bool QTest::qExpectFail(const char *dataIndex, const char *comment, TestFailMode mode, const char *file, int line) diff --git a/src/testlib/qtestcase.h b/src/testlib/qtestcase.h index 3e00035..7ba9fd7 100644 --- a/src/testlib/qtestcase.h +++ b/src/testlib/qtestcase.h @@ -47,6 +47,8 @@ #include #include +#include + QT_BEGIN_HEADER QT_BEGIN_NAMESPACE @@ -110,9 +112,12 @@ do { \ QCOMPARE(__expr, __expected); \ } while (0) -#define QSKIP(statement, mode) \ +#define QSKIP(statement, ...) \ do {\ - QTest::qSkip(statement, QTest::mode, __FILE__, __LINE__);\ + if (strcmp(#__VA_ARGS__, "") != 0)\ + QTest::qWarn("The two argument version of QSKIP is deprecated and will be removed soon. "\ + "Please update this test by removing the second parameter.", __FILE__, __LINE__);\ + QTest::qSkip(statement, __FILE__, __LINE__);\ return;\ } while (0) @@ -162,7 +167,7 @@ namespace QTest Q_TESTLIB_EXPORT bool qVerify(bool statement, const char *statementStr, const char *description, const char *file, int line); Q_TESTLIB_EXPORT void qFail(const char *statementStr, const char *file, int line); - Q_TESTLIB_EXPORT void qSkip(const char *message, SkipMode mode, const char *file, int line); + Q_TESTLIB_EXPORT void qSkip(const char *message, const char *file, int line); Q_TESTLIB_EXPORT bool qExpectFail(const char *dataIndex, const char *comment, TestFailMode mode, const char *file, int line); Q_TESTLIB_EXPORT void qWarn(const char *message, const char *file = 0, int line = 0); diff --git a/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp b/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp index 3505496..1199462 100644 --- a/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp +++ b/tests/auto/testlib/selftests/benchlibcallgrind/tst_benchlibcallgrind.cpp @@ -76,7 +76,7 @@ void tst_BenchlibCallgrind::failInChildProcess() void tst_BenchlibCallgrind::twoHundredMillionInstructions() { #if !defined(__GNUC__) || !defined(__i386) - QSKIP("This test is only defined for gcc and x86.", SkipAll); + QSKIP("This test is only defined for gcc and x86."); #else QBENCHMARK { __asm__ __volatile__( diff --git a/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp b/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp index cc8b3a4..9e274ed 100644 --- a/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp +++ b/tests/auto/testlib/selftests/benchlibtickcounter/tst_benchlibtickcounter.cpp @@ -56,7 +56,7 @@ private slots: void tst_BenchlibTickCounter::threeBillionTicks() { #ifndef HAVE_TICK_COUNTER - QSKIP("Tick counter not available on this platform", SkipAll); + QSKIP("Tick counter not available on this platform"); #else QBENCHMARK { CycleCounterTicks start = getticks(); diff --git a/tests/auto/testlib/selftests/expected_globaldata.lightxml b/tests/auto/testlib/selftests/expected_globaldata.lightxml index b8208aa..7a50f01 100644 --- a/tests/auto/testlib/selftests/expected_globaldata.lightxml +++ b/tests/auto/testlib/selftests/expected_globaldata.lightxml @@ -94,6 +94,18 @@ + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_globaldata.txt b/tests/auto/testlib/selftests/expected_globaldata.txt index efd8272..e3905e2 100644 --- a/tests/auto/testlib/selftests/expected_globaldata.txt +++ b/tests/auto/testlib/selftests/expected_globaldata.txt @@ -25,6 +25,10 @@ QDEBUG : tst_globaldata::skipLocal(1:local 1) init skipLocal local 1 SKIP : tst_globaldata::skipLocal(1:local 1) skipping Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp(149)] QDEBUG : tst_globaldata::skipLocal(1:local 1) cleanup skipLocal local 1 +QDEBUG : tst_globaldata::skipLocal(1:local 2) init skipLocal local 2 +SKIP : tst_globaldata::skipLocal(1:local 2) skipping + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp(149)] +QDEBUG : tst_globaldata::skipLocal(1:local 2) cleanup skipLocal local 2 QDEBUG : tst_globaldata::skipSingle(1:local 1) init skipSingle local 1 QDEBUG : tst_globaldata::skipSingle(1:local 1) global: false local: false QDEBUG : tst_globaldata::skipSingle(1:local 1) cleanup skipSingle local 1 @@ -41,5 +45,5 @@ QDEBUG : tst_globaldata::skipSingle(2:local 2) cleanup skipSingle local 2 PASS : tst_globaldata::skipSingle() QDEBUG : tst_globaldata::cleanupTestCase() cleanupTestCase cleanupTestCase (null) PASS : tst_globaldata::cleanupTestCase() -Totals: 4 passed, 0 failed, 3 skipped +Totals: 4 passed, 0 failed, 4 skipped ********* Finished testing of tst_globaldata ********* diff --git a/tests/auto/testlib/selftests/expected_globaldata.xml b/tests/auto/testlib/selftests/expected_globaldata.xml index e0c6947..9abbeee 100644 --- a/tests/auto/testlib/selftests/expected_globaldata.xml +++ b/tests/auto/testlib/selftests/expected_globaldata.xml @@ -96,6 +96,18 @@ + + + + + + + + + + + + diff --git a/tests/auto/testlib/selftests/expected_globaldata.xunitxml b/tests/auto/testlib/selftests/expected_globaldata.xunitxml index eade597..9e0cd14 100644 --- a/tests/auto/testlib/selftests/expected_globaldata.xunitxml +++ b/tests/auto/testlib/selftests/expected_globaldata.xunitxml @@ -1,5 +1,5 @@ - + @@ -32,6 +32,9 @@ + + + @@ -72,6 +75,9 @@ + + + diff --git a/tests/auto/testlib/selftests/expected_singleskip.txt b/tests/auto/testlib/selftests/expected_singleskip.txt index 1e54c46..b9085b1 100644 --- a/tests/auto/testlib/selftests/expected_singleskip.txt +++ b/tests/auto/testlib/selftests/expected_singleskip.txt @@ -2,7 +2,7 @@ Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@ PASS : tst_SingleSkip::initTestCase() SKIP : tst_SingleSkip::myTest() skipping test - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp(23)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp(56)] PASS : tst_SingleSkip::cleanupTestCase() Totals: 2 passed, 0 failed, 1 skipped ********* Finished testing of tst_SingleSkip ********* diff --git a/tests/auto/testlib/selftests/expected_skipinit.txt b/tests/auto/testlib/selftests/expected_skipinit.txt index a8b4f8e..a86060f 100644 --- a/tests/auto/testlib/selftests/expected_skipinit.txt +++ b/tests/auto/testlib/selftests/expected_skipinit.txt @@ -1,7 +1,7 @@ ********* Start testing of tst_SkipInit ********* Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@ SKIP : tst_SkipInit::initTestCase() Skip inside initTestCase. This should skip all tests in the class. - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp(22)] + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp(55)] PASS : tst_SkipInit::cleanupTestCase() Totals: 1 passed, 0 failed, 1 skipped ********* Finished testing of tst_SkipInit ********* diff --git a/tests/auto/testlib/selftests/expected_skipinitdata.lightxml b/tests/auto/testlib/selftests/expected_skipinitdata.lightxml index e933b71..cd82024 100644 --- a/tests/auto/testlib/selftests/expected_skipinitdata.lightxml +++ b/tests/auto/testlib/selftests/expected_skipinitdata.lightxml @@ -4,6 +4,6 @@ - + diff --git a/tests/auto/testlib/selftests/expected_skipinitdata.txt b/tests/auto/testlib/selftests/expected_skipinitdata.txt index 8640855..a4cbbea 100644 --- a/tests/auto/testlib/selftests/expected_skipinitdata.txt +++ b/tests/auto/testlib/selftests/expected_skipinitdata.txt @@ -1,6 +1,6 @@ ********* Start testing of tst_SkipInitData ********* Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE@ -SKIP : tst_SkipInitData::initTestCase() Skip inside initTestCase. This should skip all tests in the class. - Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp(23)] +SKIP : tst_SkipInitData::initTestCase() Skip inside initTestCase_data. This should skip all tests in the class. + Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp(56)] Totals: 0 passed, 0 failed, 1 skipped ********* Finished testing of tst_SkipInitData ********* diff --git a/tests/auto/testlib/selftests/expected_skipinitdata.xml b/tests/auto/testlib/selftests/expected_skipinitdata.xml index 7ad34d9..8371700 100644 --- a/tests/auto/testlib/selftests/expected_skipinitdata.xml +++ b/tests/auto/testlib/selftests/expected_skipinitdata.xml @@ -6,7 +6,7 @@ - + diff --git a/tests/auto/testlib/selftests/expected_skipinitdata.xunitxml b/tests/auto/testlib/selftests/expected_skipinitdata.xunitxml index 93967eb..699d50f 100644 --- a/tests/auto/testlib/selftests/expected_skipinitdata.xunitxml +++ b/tests/auto/testlib/selftests/expected_skipinitdata.xunitxml @@ -5,9 +5,9 @@ - + - + diff --git a/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp b/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp index 18b91cc..898cccf 100644 --- a/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp +++ b/tests/auto/testlib/selftests/globaldata/tst_globaldata.cpp @@ -126,7 +126,7 @@ void tst_globaldata::skip_data() QTest::newRow("local 1") << false; QTest::newRow("local 2") << true; - QSKIP("skipping", SkipAll); + QSKIP("skipping"); } void tst_globaldata::skip() @@ -140,13 +140,13 @@ void tst_globaldata::skipSingle() QFETCH(bool, booll); if (booli && !booll) - QSKIP("skipping", SkipSingle); + QSKIP("skipping"); qDebug() << "global:" << booli << "local:" << booll; } void tst_globaldata::skipLocal() { - QSKIP("skipping", SkipAll); + QSKIP("skipping"); } QTEST_MAIN(tst_globaldata) diff --git a/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp b/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp index 5567b88..7701eb8 100644 --- a/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp +++ b/tests/auto/testlib/selftests/singleskip/tst_singleskip.cpp @@ -53,7 +53,7 @@ private slots: void tst_SingleSkip::myTest() const { - QSKIP("skipping test", SkipAll); + QSKIP("skipping test"); } QTEST_MAIN(tst_SingleSkip) diff --git a/tests/auto/testlib/selftests/skip/tst_skip.cpp b/tests/auto/testlib/selftests/skip/tst_skip.cpp index b5b5388..905fe2d 100644 --- a/tests/auto/testlib/selftests/skip/tst_skip.cpp +++ b/tests/auto/testlib/selftests/skip/tst_skip.cpp @@ -65,7 +65,7 @@ void tst_Skip::test_data() QTest::newRow("local 1") << false; QTest::newRow("local 2") << true; - QSKIP("skipping all", SkipAll); + QSKIP("skipping all"); } void tst_Skip::test() @@ -75,7 +75,7 @@ void tst_Skip::test() void tst_Skip::emptytest_data() { - QSKIP("skipping all", SkipAll); + QSKIP("skipping all"); } void tst_Skip::emptytest() @@ -94,7 +94,7 @@ void tst_Skip::singleSkip() { QFETCH(bool, booll); if (!booll) - QSKIP("skipping one", SkipSingle); + QSKIP("skipping one"); qDebug("this line should only be reached once (%s)", booll ? "true" : "false"); } diff --git a/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp b/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp index 4cd882f..1a9fc1e 100644 --- a/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp +++ b/tests/auto/testlib/selftests/skipinit/tst_skipinit.cpp @@ -52,7 +52,7 @@ private slots: void tst_SkipInit::initTestCase() const { - QSKIP("Skip inside initTestCase. This should skip all tests in the class.", SkipAll); + QSKIP("Skip inside initTestCase. This should skip all tests in the class."); } /*! \internal diff --git a/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp b/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp index fb97fc9..b09e99b 100644 --- a/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp +++ b/tests/auto/testlib/selftests/skipinitdata/tst_skipinitdata.cpp @@ -53,7 +53,7 @@ private slots: void tst_SkipInitData::initTestCase_data() const { - QSKIP("Skip inside initTestCase. This should skip all tests in the class.", SkipAll); + QSKIP("Skip inside initTestCase_data. This should skip all tests in the class."); } void tst_SkipInitData::initTestCase() const diff --git a/tests/auto/testlib/selftests/xunit/tst_xunit.cpp b/tests/auto/testlib/selftests/xunit/tst_xunit.cpp index 8f2f712..05acc8c 100644 --- a/tests/auto/testlib/selftests/xunit/tst_xunit.cpp +++ b/tests/auto/testlib/selftests/xunit/tst_xunit.cpp @@ -76,7 +76,7 @@ void tst_Xunit::testFunc2() void tst_Xunit::testFunc3() { - QSKIP("skipping this function!", SkipAll); + QSKIP("skipping this function!"); } void tst_Xunit::testFunc4() -- 2.7.4