Improve coverage of expectfail selftest.

Add a test for the case where QEXPECT_FAIL is called twice in a row
without an intervening verification step, which is supposed to result in
a failure.

Change-Id: Ie004652a76a2c35c2679945670ff5561e8f5b62e
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>

diff --git a/tests/auto/testlib/selftests/expected_expectfail.lightxml b/tests/auto/testlib/selftests/expected_expectfail.lightxml
index a2bf6fa..d2659fa 100644 (file)
--- a/tests/auto/testlib/selftests/expected_expectfail.lightxml
+++ b/tests/auto/testlib/selftests/expected_expectfail.lightxml
@@ -9,7 +9,7 @@
 <Message type="qdebug" file="" line="0">
     <Description><![CDATA[begin]]></Description>
 </Message>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="65">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="66">
     <Description><![CDATA[This should xfail]]></Description>
 </Incident>
 <Message type="qdebug" file="" line="0">
@@ -21,31 +21,36 @@
 <Message type="qdebug" file="" line="0">
     <Description><![CDATA[begin]]></Description>
 </Message>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="73">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="74">
     <Description><![CDATA[This should xfail]]></Description>
 </Incident>
 <Incident type="pass" file="" line="0" />
 </TestFunction>
+<TestFunction name="expectTwice">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="83">
+    <Description><![CDATA[Already expecting a fail]]></Description>
+</Incident>
+</TestFunction>
 <TestFunction name="xfailWithQString">
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="82">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="93">
     <Description><![CDATA[A string]]></Description>
 </Incident>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="87">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="98">
     <Description><![CDATA[Bug 5 (The message)]]></Description>
 </Incident>
 <Incident type="pass" file="" line="0" />
 </TestFunction>
 <TestFunction name="xpass">
-<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="93">
+<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="104">
     <Description><![CDATA['true' returned FALSE. ()]]></Description>
 </Incident>
 </TestFunction>
 <TestFunction name="dataDrivenTest">
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="126">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="137">
     <DataTag><![CDATA[Abort]]></DataTag>
     <Description><![CDATA[This test should xfail]]></Description>
 </Incident>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="126">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="137">
     <DataTag><![CDATA[Continue]]></DataTag>
     <Description><![CDATA[This test should xfail]]></Description>
 </Incident>

diff --git a/tests/auto/testlib/selftests/expected_expectfail.txt b/tests/auto/testlib/selftests/expected_expectfail.txt
index ad1d1f2..4fe3b51 100644 (file)
--- a/tests/auto/testlib/selftests/expected_expectfail.txt
+++ b/tests/auto/testlib/selftests/expected_expectfail.txt
@@ -3,25 +3,27 @@ Config: Using QTest library @INSERT_QT_VERSION_HERE@, Qt @INSERT_QT_VERSION_HERE
 PASS   : tst_ExpectFail::initTestCase()
 QDEBUG : tst_ExpectFail::expectAndContinue() begin
 XFAIL  : tst_ExpectFail::expectAndContinue() This should xfail
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(65)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(66)]
 QDEBUG : tst_ExpectFail::expectAndContinue() after
 PASS   : tst_ExpectFail::expectAndContinue()
 QDEBUG : tst_ExpectFail::expectAndAbort() begin
 XFAIL  : tst_ExpectFail::expectAndAbort() This should xfail
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(73)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(74)]
 PASS   : tst_ExpectFail::expectAndAbort()
+FAIL!  : tst_ExpectFail::expectTwice() Already expecting a fail
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(83)]
 XFAIL  : tst_ExpectFail::xfailWithQString() A string
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(82)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(93)]
 XFAIL  : tst_ExpectFail::xfailWithQString() Bug 5 (The message)
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(87)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(98)]
 PASS   : tst_ExpectFail::xfailWithQString()
 XPASS  : tst_ExpectFail::xpass() 'true' returned FALSE. ()
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(93)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(104)]
 XFAIL  : tst_ExpectFail::dataDrivenTest(Abort) This test should xfail
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(126)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(137)]
 XFAIL  : tst_ExpectFail::dataDrivenTest(Continue) This test should xfail
-   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(126)]
+   Loc: [/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp(137)]
 PASS   : tst_ExpectFail::dataDrivenTest()
 PASS   : tst_ExpectFail::cleanupTestCase()
-Totals: 6 passed, 1 failed, 0 skipped
+Totals: 6 passed, 2 failed, 0 skipped
 ********* Finished testing of tst_ExpectFail *********

diff --git a/tests/auto/testlib/selftests/expected_expectfail.xml b/tests/auto/testlib/selftests/expected_expectfail.xml
index 4380e06..a719928 100644 (file)
--- a/tests/auto/testlib/selftests/expected_expectfail.xml
+++ b/tests/auto/testlib/selftests/expected_expectfail.xml
@@ -11,7 +11,7 @@
 <Message type="qdebug" file="" line="0">
     <Description><![CDATA[begin]]></Description>
 </Message>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="65">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="66">
     <Description><![CDATA[This should xfail]]></Description>
 </Incident>
 <Message type="qdebug" file="" line="0">
@@ -23,31 +23,36 @@
 <Message type="qdebug" file="" line="0">
     <Description><![CDATA[begin]]></Description>
 </Message>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="73">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="74">
     <Description><![CDATA[This should xfail]]></Description>
 </Incident>
 <Incident type="pass" file="" line="0" />
 </TestFunction>
+<TestFunction name="expectTwice">
+<Incident type="fail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="83">
+    <Description><![CDATA[Already expecting a fail]]></Description>
+</Incident>
+</TestFunction>
 <TestFunction name="xfailWithQString">
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="82">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="93">
     <Description><![CDATA[A string]]></Description>
 </Incident>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="87">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="98">
     <Description><![CDATA[Bug 5 (The message)]]></Description>
 </Incident>
 <Incident type="pass" file="" line="0" />
 </TestFunction>
 <TestFunction name="xpass">
-<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="93">
+<Incident type="xpass" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="104">
     <Description><![CDATA['true' returned FALSE. ()]]></Description>
 </Incident>
 </TestFunction>
 <TestFunction name="dataDrivenTest">
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="126">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="137">
     <DataTag><![CDATA[Abort]]></DataTag>
     <Description><![CDATA[This test should xfail]]></Description>
 </Incident>
-<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="126">
+<Incident type="xfail" file="/home/user/dev/qt5/qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp" line="137">
     <DataTag><![CDATA[Continue]]></DataTag>
     <Description><![CDATA[This test should xfail]]></Description>
 </Incident>

diff --git a/tests/auto/testlib/selftests/expected_expectfail.xunitxml b/tests/auto/testlib/selftests/expected_expectfail.xunitxml
index 6d3642a..e93cb03 100644 (file)
--- a/tests/auto/testlib/selftests/expected_expectfail.xunitxml
+++ b/tests/auto/testlib/selftests/expected_expectfail.xunitxml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<testsuite errors="9" failures="1" tests="7" name="tst_ExpectFail">
+<testsuite errors="9" failures="2" tests="8" name="tst_ExpectFail">
   <properties>
     <property value="@INSERT_QT_VERSION_HERE@" name="QTestVersion"/>
     <property value="@INSERT_QT_VERSION_HERE@" name="QtVersion"/>
@@ -14,6 +14,9 @@
     <!-- message="begin" type="qdebug" -->
     <!-- message="This should xfail" type="info" -->
   </testcase>
+  <testcase result="fail" name="expectTwice">
+    <failure message="Already expecting a fail" result="fail"/>
+  </testcase>
   <testcase result="xfail" name="xfailWithQString">
     <!-- message="A string" type="info" -->
     <!-- message="Bug 5 (The message)" type="info" -->

diff --git a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
index 8f67afc..73d3e0c 100644 (file)
--- a/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
+++ b/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
@@ -52,6 +52,7 @@ class tst_ExpectFail: public QObject
 private slots:
     void expectAndContinue() const;
     void expectAndAbort() const;
+    void expectTwice() const;
     void xfailWithQString() const;
     void xpass() const;
     void dataDrivenTest_data() const;
@@ -76,6 +77,16 @@ void tst_ExpectFail::expectAndAbort() const
     QVERIFY2(false, "This should not be reached");
 }
 
+void tst_ExpectFail::expectTwice() const
+{
+    QEXPECT_FAIL("", "Calling QEXPECT_FAIL once is fine", Abort);
+    QEXPECT_FAIL("", "Calling QEXPECT_FAIL when already expecting a failure is "
+                     "an error and should abort this test function", Abort);
+
+    // If we get here the test did not correctly abort on the double call to QEXPECT_FAIL.
+    QVERIFY2(false, "This should not be reached");
+}
+
 void tst_ExpectFail::xfailWithQString() const
 {
     QEXPECT_FAIL("", QString("A string").toLatin1().constData(), Continue);