Integrate testcocoon support into Qt build system.

To instrument a Qt application or library with the TestCocoon coverage
tool, do `CONFIG+=testcocoon' in the application .pro file.

To instrument Qt itself with testcocoon, use the `-testcocoon' configure
option.

Change-Id: Ie77109a078d11ea51f7a073621e0df9c752c44ae
Reviewed-by: Oswald Buddenhagen <oswald.buddenhagen@nokia.com>
Reviewed-by: Rohan McGovern <rohan.mcgovern@nokia.com>

diff --git a/configure b/configure
index 974f6ac..62d1e6f 100755 (executable)
--- a/configure
+++ b/configure
@@ -1038,7 +1038,7 @@ while [ "$#" -gt 0 ]; do
         VAL=no
         ;;
     #Qt style yes options
-    -incremental|-qvfb|-profile|-shared|-static|-sm|-xinerama|-xshape|-xsync|-xinput|-xinput2|-egl|-reduce-exports|-pch|-separate-debug-info|-stl|-freetype|-xcursor|-xfixes|-xrandr|-xrender|-mitshm|-fontconfig|-xkb|-xcb|-wayland|-nis|-qdbus|-dbus|-dbus-linked|-glib|-gstreamer|-gtkstyle|-cups|-iconv|-largefile|-h|-help|-v|-verbose|-debug|-release|-fast|-accessibility|-confirm-license|-gnumake|-framework|-qt3support|-debug-and-release|-exceptions|-cocoa|-carbon|-universal|-harfbuzz|-prefix-install|-silent|-armfpa|-optimized-qmake|-dwarf2|-reduce-relocations|-sse|-openssl|-openssl-linked|-ptmalloc|-xmlpatterns|-phonon|-phonon-backend|-multimedia|-audio-backend|-svg|-v8|-declarative|-declarative-debug|-javascript-jit|-script|-scripttools|-rpath|-force-pkg-config|-icu|-force-asserts)
+    -incremental|-qvfb|-profile|-shared|-static|-sm|-xinerama|-xshape|-xsync|-xinput|-xinput2|-egl|-reduce-exports|-pch|-separate-debug-info|-stl|-freetype|-xcursor|-xfixes|-xrandr|-xrender|-mitshm|-fontconfig|-xkb|-xcb|-wayland|-nis|-qdbus|-dbus|-dbus-linked|-glib|-gstreamer|-gtkstyle|-cups|-iconv|-largefile|-h|-help|-v|-verbose|-debug|-release|-fast|-accessibility|-confirm-license|-gnumake|-framework|-qt3support|-debug-and-release|-exceptions|-cocoa|-carbon|-universal|-harfbuzz|-prefix-install|-silent|-armfpa|-optimized-qmake|-dwarf2|-reduce-relocations|-sse|-openssl|-openssl-linked|-ptmalloc|-xmlpatterns|-phonon|-phonon-backend|-multimedia|-audio-backend|-svg|-v8|-declarative|-declarative-debug|-javascript-jit|-script|-scripttools|-rpath|-force-pkg-config|-icu|-force-asserts|-testcocoon)
         VAR=`echo $1 | sed "s,^-\(.*\),\1,"`
         VAL=yes
         ;;
@@ -1520,6 +1520,11 @@ while [ "$#" -gt 0 ]; do
             UNKNOWN_OPT=yes
         fi
         ;;
+    testcocoon)
+        if [ "$VAL" = "yes" ]; then
+            QTCONFIG_CONFIG="$QTCONFIG_CONFIG testcocoon"
+        fi
+        ;;
     exceptions|g++-exceptions)
         if [ "$VAL" = "no" ]; then
             CFG_EXCEPTIONS=no
@@ -3925,6 +3930,8 @@ cat << EOF
     -qtnamespace <name>  Wraps all Qt library code in 'namespace <name> {...}'.
     -qtlibinfix <infix>  Renames all libQt*.so to libQt*<infix>.so.
 
+    -testcocoon          Instrument Qt with the TestCocoon code coverage tool.
+
     -D <string> ........ Add an explicit define to the preprocessor.
     -I <string> ........ Add an explicit include path.
     -L <string> ........ Add an explicit library path.

diff --git a/mkspecs/features/testcocoon.prf b/mkspecs/features/testcocoon.prf
new file mode 100644 (file)
index 0000000..e6ad733
--- /dev/null
+++ b/mkspecs/features/testcocoon.prf
@@ -0,0 +1,57 @@
+#
+#    Tested with TestCocoon 1.6.14
+#
+
+load(resolve_target)
+
+# Retrieve the target basename
+TARGET_BASENAME = $$basename(QMAKE_RESOLVED_TARGET)
+
+# Configure testcocoon for a full instrumentation - excluding the moc, ui and qrc files from the instrumentation
+# --cs-output defines the name to give to the execution report (.csexe).
+TESTCOCOON_COVERAGE_OPTIONS = \
+    --cs-qt4 \
+    --cs-exclude-file-regex=\'(^|[/\\\\])ui_.*\\.h\$\$\' \
+    --cs-exclude-file-regex=\'(^|[/\\\\])(qrc|moc)_.*\\.cpp\$\$\' \
+    --cs-exclude-file-regex=\'.*\\.moc\$\$\' \
+    --cs-exclude-file-regex=\'.*\\.g\$\$\' \
+    --cs-output=\'$$TARGET_BASENAME\' # name of the csexe file (execution report)
+
+# The .csmes file should be placed alongside the .so or binary.
+# Unfortunately, testcocoon has no option to specify the output directory,
+# so we must move it into place if a custom destdir was used.
+# We don't move applications' csmes because some qt applications (tools, examples)
+# are using DESTDIR in some cases but always alongside target.path, so the binary
+# is built directly in target.path and there is no need to move the csmes.
+!isEmpty(DESTDIR):contains(TEMPLATE, lib) {
+    !isEmpty(QMAKE_POST_LINK):QMAKE_POST_LINK = $$escape_expand(\\n\\t)$$QMAKE_POST_LINK
+    QMAKE_POST_LINK = -$(MOVE) $${TARGET_BASENAME}.csmes $${QMAKE_RESOLVED_TARGET}.csmes$$QMAKE_POST_LINK
+}
+
+QMAKE_CLEAN +=  *.csexe *.csmes
+
+# The compiler/linker is replaced by the coveragescanner which is named after the name of the
+# compiler/linker preceded by cs (ie gcc is replaced by csgcc).
+# Testcocoon options defined in TESTCOCOON_COVERAGE_OPTIONS are added as argument to the coveragescanner (ie csgcc).
+# In practice they are added as compiler/linker flags.
+
+*-g++* {
+    QMAKE_CXX ~= s/(\\S*g\\+\\+)/cs\\1/
+    QMAKE_CC ~= s/(\\S*gcc)/cs\\1/
+    QMAKE_LINK ~= s/(\\S*g\\+\\+|\\S*gcc)/cs\\1/
+    QMAKE_AR ~= s/(\\S*ar)/cs\\1/
+    QMAKE_AR += $$TESTCOCOON_COVERAGE_OPTIONS
+} else {
+    error("Non-gcc qmake specs not supported by TestCocoon integration yet")
+}
+
+QMAKE_CFLAGS += $$TESTCOCOON_COVERAGE_OPTIONS
+QMAKE_CXXFLAGS += $$TESTCOCOON_COVERAGE_OPTIONS
+QMAKE_LFLAGS += $$TESTCOCOON_COVERAGE_OPTIONS
+
+unix {
+    QMAKE_LFLAGS += --cs-libgen=-fPIC
+}
+
+unset(TARGET_BASENAME)
+unset(TESTCOCOON_COVERAGE_OPTIONS)

diff --git a/src/corelib/plugin/qlibrary.cpp b/src/corelib/plugin/qlibrary.cpp
index 9eeb783..6a40b5b 100644 (file)
--- a/src/corelib/plugin/qlibrary.cpp
+++ b/src/corelib/plugin/qlibrary.cpp
@@ -400,6 +400,60 @@ static bool qt_unix_query(const QString &library, uint *version, bool *debug, QL
 
 #endif // Q_OS_UNIX && !Q_OS_MAC && !defined(Q_OS_SYMBIAN) && !defined(QT_NO_PLUGIN_CHECK)
 
+static void installCoverageTool(QLibraryPrivate *libPrivate)
+{
+#ifdef __COVERAGESCANNER__
+    /*
+      __COVERAGESCANNER__ is defined when Qt has been instrumented for code
+      coverage by TestCocoon. CoverageScanner is the name of the tool that
+      generates the code instrumentation.
+      This code is required here when code coverage analysis with TestCocoon
+      is enabled in order to allow the loading application to register the plugin
+      and then store its execution report. The execution report gathers information
+      about each part of the plugin's code that has been used when
+      the plugin was loaded by the launching application.
+      The execution report for the plugin will go to the same execution report
+      as the one defined for the application loading it.
+    */
+
+    int ret = __coveragescanner_register_library(libPrivate->fileName.toLocal8Bit());
+
+    if (qt_debug_component()) {
+        if (ret >= 0) {
+            qDebug("%s: coverage data for %s registered",
+                     Q_FUNC_INFO,
+                     qPrintable(libPrivate->fileName));
+        } else {
+            qWarning("%s: could not register %s: error %d; coverage data may be incomplete",
+                     Q_FUNC_INFO,
+                     qPrintable(libPrivate->fileName),
+                     ret);
+        }
+    }
+#else
+    Q_UNUSED(libPrivate);
+#endif
+}
+
+static void releaseCoverageTool(QLibraryPrivate *libPrivate)
+{
+#ifdef __COVERAGESCANNER__
+    /*
+      __COVERAGESCANNER__ is defined when Qt has been instrumented for code
+      coverage by TestCocoon.
+      Here is the code to save the execution data.
+      See comments about initialization in QLibraryPrivate::load().
+    */
+    if (libPrivate->pHnd) {
+        __coveragescanner_save();
+        __coveragescanner_clear();
+        __coveragescanner_unregister_library(libPrivate->fileName.toLocal8Bit());
+    }
+#else
+    Q_UNUSED(libPrivate);
+#endif
+}
+
 typedef QMap<QString, QLibraryPrivate*> LibraryMap;
 
 struct LibraryData {
@@ -465,6 +519,8 @@ bool QLibraryPrivate::load()
             lib->loadedLibs += this;
             libraryRefCount.ref();
         }
+
+        installCoverageTool(this);
     }
 
     return ret;
@@ -494,6 +550,8 @@ bool QLibraryPrivate::unload()
 
 void QLibraryPrivate::release()
 {
+    releaseCoverageTool(this);
+
     QMutexLocker locker(qt_library_mutex());
     if (!libraryRefCount.deref())
         delete this;

diff --git a/src/gui/gui.pro b/src/gui/gui.pro
index 40888ad..db04593 100644 (file)
--- a/src/gui/gui.pro
+++ b/src/gui/gui.pro
@@ -13,6 +13,14 @@ unix|win32-g++*:QMAKE_PKGCONFIG_REQUIRES = QtCore
 
 load(qt_module_config)
 
+# Code coverage with TestCocoon
+# The following is required as extra compilers use $$QMAKE_CXX instead of $(CXX).
+# Without this, testcocoon.prf is read only after $$QMAKE_CXX is used by the
+# extra compilers.
+testcocoon {
+    load(testcocoon)
+}
+
 HEADERS += $$QT_SOURCE_TREE/src/gui/qtguiversion.h
 
 include(accessible/accessible.pri)

diff --git a/src/testlib/qtestcase.cpp b/src/testlib/qtestcase.cpp
index 4e0205c..f01c588 100644 (file)
--- a/src/testlib/qtestcase.cpp
+++ b/src/testlib/qtestcase.cpp
@@ -857,6 +857,35 @@ QT_BEGIN_NAMESPACE
     QTouchEventSequence is called (ie when the object returned runs out of scope).
 */
 
+static void installCoverageTool(const char * appname, const char * testname)
+{
+#ifdef __COVERAGESCANNER__
+    // Install Coverage Tool
+    __coveragescanner_install(appname);
+    __coveragescanner_testname(testname);
+    __coveragescanner_clear();
+#else
+    Q_UNUSED(appname);
+    Q_UNUSED(testname);
+#endif
+}
+
+static void saveCoverageTool(const char * appname, bool testfailed)
+{
+#ifdef __COVERAGESCANNER__
+    // install again to make sure the filename is correct.
+    // without this, a plugin or similar may have changed the filename.
+    __coveragescanner_install(appname);
+    __coveragescanner_teststate(testfailed ? "FAILED" : "PASSED");
+    __coveragescanner_save();
+    __coveragescanner_testname("");
+    __coveragescanner_clear();
+#else
+    Q_UNUSED(appname);
+    Q_UNUSED(testfailed);
+#endif
+}
+
 namespace QTest
 {
     static QObject *currentTestObject = 0;
@@ -1904,6 +1933,8 @@ int QTest::qExec(QObject *testObject, int argc, char **argv)
     const QMetaObject *metaObject = testObject->metaObject();
     QTEST_ASSERT(metaObject);
 
+    installCoverageTool(argv[0], metaObject->className());
+
     QTestResult::setCurrentTestObject(metaObject->className());
     qtest_qParseArgs(argc, argv, false);
 #ifdef QTESTLIB_USE_VALGRIND
@@ -1954,6 +1985,8 @@ int QTest::qExec(QObject *testObject, int argc, char **argv)
      }
 #endif
 
+     saveCoverageTool(argv[0], QTestResult::failCount());
+
 #ifdef QTESTLIB_USE_VALGRIND
     if (QBenchmarkGlobalData::current->mode() == QBenchmarkGlobalData::CallgrindParentProcess)
         return callgrindChildExitCode;

diff --git a/src/widgets/widgets.pro b/src/widgets/widgets.pro
index d0425ca..8f6a971 100644 (file)
--- a/src/widgets/widgets.pro
+++ b/src/widgets/widgets.pro
@@ -44,6 +44,14 @@ contains(DEFINES,QT_EVAL):include($$QT_SOURCE_TREE/src/corelib/eval.pri)
 
 QMAKE_DYNAMIC_LIST_FILE = $$PWD/QtGui.dynlist
 
+# Code coverage with TestCocoon
+# The following is required as extra compilers use $$QMAKE_CXX instead of $(CXX).
+# Without this, testcocoon.prf is read only after $$QMAKE_CXX is used by the
+# extra compilers.
+testcocoon {
+    load(testcocoon)
+}
+
 DEFINES += Q_INTERNAL_QAPP_SRC
 
 INCLUDEPATH += ../3rdparty/harfbuzz/src

diff --git a/tests/tests.pro b/tests/tests.pro
index d91d696..7e129e0 100644 (file)
--- a/tests/tests.pro
+++ b/tests/tests.pro
@@ -3,5 +3,6 @@ TEMPLATE = subdirs
 SUBDIRS = auto
 
 # benchmarks in debug mode is rarely sensible
-contains(QT_CONFIG,release):SUBDIRS += benchmarks
+# benchmarks are not sensible for code coverage (here with tool testcocoon)
+!testcocoon:contains(QT_CONFIG,release):SUBDIRS += benchmarks
 }