1 /****************************************************************************
3 ** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
4 ** All rights reserved.
5 ** Contact: Nokia Corporation (qt-info@nokia.com)
7 ** This file is part of the test suite of the Qt Toolkit.
9 ** $QT_BEGIN_LICENSE:LGPL$
10 ** GNU Lesser General Public License Usage
11 ** This file may be used under the terms of the GNU Lesser General Public
12 ** License version 2.1 as published by the Free Software Foundation and
13 ** appearing in the file LICENSE.LGPL included in the packaging of this
14 ** file. Please review the following information to ensure the GNU Lesser
15 ** General Public License version 2.1 requirements will be met:
16 ** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
18 ** In addition, as a special exception, Nokia gives you certain additional
19 ** rights. These rights are described in the Nokia Qt LGPL Exception
20 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
22 ** GNU General Public License Usage
23 ** Alternatively, this file may be used under the terms of the GNU General
24 ** Public License version 3.0 as published by the Free Software Foundation
25 ** and appearing in the file LICENSE.GPL included in the packaging of this
26 ** file. Please review the following information to ensure the GNU General
27 ** Public License version 3.0 requirements will be met:
28 ** http://www.gnu.org/copyleft/gpl.html.
31 ** Alternatively, this file may be used in accordance with the terms and
32 ** conditions contained in a signed written agreement between you and Nokia.
40 ****************************************************************************/
42 #include "quicktestresult_p.h"
43 #include <QtTest/qtestcase.h>
44 #include <QtTest/qtestsystem.h>
45 #include <QtTest/private/qtestresult_p.h>
46 #include <QtTest/private/qtesttable_p.h>
47 #include <QtTest/private/qtestlog_p.h>
48 #include "qtestoptions_p.h"
49 #include <QtTest/qbenchmark.h>
50 #include <QtTest/private/qbenchmark_p.h>
51 #include <QtCore/qset.h>
52 #include <QtCore/qmap.h>
53 #include <QtCore/qbytearray.h>
54 #include <QtCore/qcoreapplication.h>
55 #include <QtCore/qdebug.h>
59 static const char *globalProgramName = 0;
60 static bool loggingStarted = false;
61 static QBenchmarkGlobalData globalBenchmarkData;
63 class QuickTestResultPrivate
66 QuickTestResultPrivate()
73 ~QuickTestResultPrivate()
80 QByteArray intern(const QString &str);
81 void updateTestObjectName();
85 QSet<QByteArray> internedStrings;
87 QTest::QBenchmarkIterationController *benchmarkIter;
88 QBenchmarkTestMethodData *benchmarkData;
90 QList<QBenchmarkResult> results;
93 QByteArray QuickTestResultPrivate::intern(const QString &str)
95 QByteArray bstr = str.toUtf8();
96 return *(internedStrings.insert(bstr));
99 void QuickTestResultPrivate::updateTestObjectName()
101 // In plain logging mode we use the TestCase name as the
102 // class name so that multiple TestCase elements will report
103 // results with "testCase::function". In XML logging mode,
104 // we use the program name as the class name and report test
105 // functions as "testCase__function".
106 if (QTestLog::logMode() == QTestLog::Plain) {
107 if (testCaseName.isEmpty()) {
108 QTestResult::setCurrentTestObject(globalProgramName);
109 } else if (QTestLog::logMode() == QTestLog::Plain) {
110 QTestResult::setCurrentTestObject
111 (intern(testCaseName).constData());
114 QTestResult::setCurrentTestObject(globalProgramName);
118 QuickTestResult::QuickTestResult(QObject *parent)
119 : QObject(parent), d_ptr(new QuickTestResultPrivate)
121 if (!QBenchmarkGlobalData::current)
122 QBenchmarkGlobalData::current = &globalBenchmarkData;
125 QuickTestResult::~QuickTestResult()
130 \qmlproperty string TestResult::testCaseName
132 This property defines the name of current TestCase element
133 that is running test cases.
137 QString QuickTestResult::testCaseName() const
139 Q_D(const QuickTestResult);
140 return d->testCaseName;
143 void QuickTestResult::setTestCaseName(const QString &name)
145 Q_D(QuickTestResult);
146 d->testCaseName = name;
147 d->updateTestObjectName();
148 emit testCaseNameChanged();
152 \qmlproperty string TestResult::functionName
154 This property defines the name of current test function
155 within a TestCase element that is running. If this string is
156 empty, then no function is currently running.
160 QString QuickTestResult::functionName() const
162 Q_D(const QuickTestResult);
163 return d->functionName;
166 void QuickTestResult::setFunctionName(const QString &name)
168 Q_D(QuickTestResult);
169 if (!name.isEmpty()) {
170 // In plain logging mode, we use the function name directly.
171 // In XML logging mode, we use "testCase__functionName" as the
172 // program name is acting as the class name.
173 if (QTestLog::logMode() == QTestLog::Plain ||
174 d->testCaseName.isEmpty()) {
175 QTestResult::setCurrentTestFunction
176 (d->intern(name).constData());
178 QString fullName = d->testCaseName + QLatin1String("__") + name;
179 QTestResult::setCurrentTestFunction
180 (d->intern(fullName).constData());
183 QTestResult::setCurrentTestFunction(0);
185 d->functionName = name;
186 emit functionNameChanged();
189 QuickTestResult::FunctionType QuickTestResult::functionType() const
191 return FunctionType(QTestResult::currentTestLocation());
194 void QuickTestResult::setFunctionType(FunctionType type)
196 QTestResult::setCurrentTestLocation(QTestResult::TestLocation(type));
197 emit functionTypeChanged();
201 \qmlproperty string TestResult::dataTag
203 This property defines the tag for the current row in a
204 data-driven test, or an empty string if not a data-driven test.
206 QString QuickTestResult::dataTag() const
208 const char *tag = QTestResult::currentDataTag();
210 return QString::fromUtf8(tag);
215 void QuickTestResult::setDataTag(const QString &tag)
217 if (!tag.isEmpty()) {
218 QTestData *data = &(QTest::newRow(tag.toUtf8().constData()));
219 QTestResult::setCurrentTestData(data);
220 emit dataTagChanged();
222 QTestResult::setCurrentTestData(0);
227 \qmlproperty bool TestResult::failed
229 This property returns true if the current test function has
230 failed; false otherwise. The fail state is reset when
231 functionName is changed or finishTestFunction() is called.
233 \sa skipped, dataFailed
235 bool QuickTestResult::isFailed() const
237 return QTestResult::testFailed();
241 \qmlproperty bool TestResult::dataFailed
243 This property returns true if the current data function has
244 failed; false otherwise. The fail state is reset when
245 functionName is changed or finishTestFunction() is called.
249 bool QuickTestResult::isDataFailed() const
251 return QTestResult::currentTestFailed();
255 \qmlproperty bool TestResult::skipped
257 This property returns true if the current test function was
258 marked as skipped; false otherwise.
262 bool QuickTestResult::isSkipped() const
264 return QTestResult::skipCurrentTest();
267 void QuickTestResult::setSkipped(bool skip)
269 QTestResult::setSkipCurrentTest(skip);
270 emit skippedChanged();
274 \qmlproperty int TestResult::passCount
276 This property returns the number of tests that have passed.
278 \sa failCount, skipCount
280 int QuickTestResult::passCount() const
282 return QTestResult::passCount();
286 \qmlproperty int TestResult::failCount
288 This property returns the number of tests that have failed.
290 \sa passCount, skipCount
292 int QuickTestResult::failCount() const
294 return QTestResult::failCount();
298 \qmlproperty int TestResult::skipCount
300 This property returns the number of tests that have been skipped.
302 \sa passCount, failCount
304 int QuickTestResult::skipCount() const
306 return QTestResult::skipCount();
310 \qmlproperty list<string> TestResult::functionsToRun
312 This property returns the list of function names to be run.
314 QStringList QuickTestResult::functionsToRun() const
316 return QTest::testFunctions;
320 \qmlmethod TestResult::reset()
322 Resets all pass/fail/skip counters and prepare for testing.
324 void QuickTestResult::reset()
326 if (!globalProgramName) // Only if run via qmlviewer.
327 QTestResult::reset();
331 \qmlmethod TestResult::startLogging()
333 Starts logging to the test output stream and writes the
338 void QuickTestResult::startLogging()
340 // The program name is used for logging headers and footers if it
341 // is set. Otherwise the test case name is used.
342 Q_D(QuickTestResult);
345 const char *saved = QTestResult::currentTestObjectName();
346 if (globalProgramName) {
347 QTestResult::setCurrentTestObject(globalProgramName);
349 QTestResult::setCurrentTestObject
350 (d->intern(d->testCaseName).constData());
352 QTestLog::startLogging();
353 QTestResult::setCurrentTestObject(saved);
354 loggingStarted = true;
358 \qmlmethod TestResult::stopLogging()
360 Writes the test footer to the test output stream and then stops logging.
364 void QuickTestResult::stopLogging()
366 Q_D(QuickTestResult);
367 if (globalProgramName)
368 return; // Logging will be stopped by setProgramName(0).
369 const char *saved = QTestResult::currentTestObjectName();
370 QTestResult::setCurrentTestObject(d->intern(d->testCaseName).constData());
371 QTestLog::stopLogging();
372 QTestResult::setCurrentTestObject(saved);
375 void QuickTestResult::initTestTable()
377 Q_D(QuickTestResult);
379 d->table = new QTestTable;
382 void QuickTestResult::clearTestTable()
384 Q_D(QuickTestResult);
389 void QuickTestResult::finishTestFunction()
391 QTestResult::finishedCurrentTestFunction();
394 static QString qtest_fixFile(const QString &file)
396 if (file.startsWith(QLatin1String("file://")))
402 void QuickTestResult::fail
403 (const QString &message, const QString &file, int line)
405 QTestResult::addFailure(message.toLatin1().constData(),
406 qtest_fixFile(file).toLatin1().constData(), line);
409 bool QuickTestResult::verify
410 (bool success, const QString &message, const QString &file, int line)
412 if (!success && message.isEmpty()) {
413 return QTestResult::verify
414 (success, "verify()", "",
415 qtest_fixFile(file).toLatin1().constData(), line);
417 return QTestResult::verify
418 (success, message.toLatin1().constData(), "",
419 qtest_fixFile(file).toLatin1().constData(), line);
423 bool QuickTestResult::compare
424 (bool success, const QString &message,
425 const QString &val1, const QString &val2,
426 const QString &file, int line)
429 return QTestResult::compare
430 (success, message.toLocal8Bit().constData(),
431 qtest_fixFile(file).toLatin1().constData(), line);
433 return QTestResult::compare
434 (success, message.toLocal8Bit().constData(),
435 QTest::toString(val1.toLatin1().constData()),
436 QTest::toString(val2.toLatin1().constData()),
438 qtest_fixFile(file).toLatin1().constData(), line);
442 void QuickTestResult::skipSingle
443 (const QString &message, const QString &file, int line)
445 QTestResult::addSkip(message.toLatin1().constData(), QTest::SkipSingle,
446 qtest_fixFile(file).toLatin1().constData(), line);
449 void QuickTestResult::skipAll
450 (const QString &message, const QString &file, int line)
452 QTestResult::addSkip(message.toLatin1().constData(), QTest::SkipAll,
453 qtest_fixFile(file).toLatin1().constData(), line);
454 QTestResult::setSkipCurrentTest(true);
457 bool QuickTestResult::expectFail
458 (const QString &tag, const QString &comment, const QString &file, int line)
460 return QTestResult::expectFail
461 (tag.toLatin1().constData(),
462 QTest::toString(comment.toLatin1().constData()),
463 QTest::Abort, qtest_fixFile(file).toLatin1().constData(), line);
466 bool QuickTestResult::expectFailContinue
467 (const QString &tag, const QString &comment, const QString &file, int line)
469 return QTestResult::expectFail
470 (tag.toLatin1().constData(),
471 QTest::toString(comment.toLatin1().constData()),
472 QTest::Continue, qtest_fixFile(file).toLatin1().constData(), line);
475 void QuickTestResult::warn(const QString &message)
477 QTestLog::warn(message.toLatin1().constData());
480 void QuickTestResult::ignoreWarning(const QString &message)
482 QTestResult::ignoreMessage(QtWarningMsg, message.toLatin1().constData());
485 void QuickTestResult::wait(int ms)
490 void QuickTestResult::sleep(int ms)
495 void QuickTestResult::startMeasurement()
497 Q_D(QuickTestResult);
498 delete d->benchmarkData;
499 d->benchmarkData = new QBenchmarkTestMethodData();
500 QBenchmarkTestMethodData::current = d->benchmarkData;
501 d->iterCount = (QBenchmarkGlobalData::current->measurer->needsWarmupIteration()) ? -1 : 0;
505 void QuickTestResult::beginDataRun()
507 QBenchmarkTestMethodData::current->beginDataRun();
510 void QuickTestResult::endDataRun()
512 Q_D(QuickTestResult);
513 QBenchmarkTestMethodData::current->endDataRun();
514 if (d->iterCount > -1) // iteration -1 is the warmup iteration.
515 d->results.append(QBenchmarkTestMethodData::current->result);
517 if (QBenchmarkGlobalData::current->verboseOutput) {
518 if (d->iterCount == -1) {
519 qDebug() << "warmup stage result :" << QBenchmarkTestMethodData::current->result.value;
521 qDebug() << "accumulation stage result:" << QBenchmarkTestMethodData::current->result.value;
526 bool QuickTestResult::measurementAccepted()
528 return QBenchmarkTestMethodData::current->resultsAccepted();
531 static QBenchmarkResult qMedian(const QList<QBenchmarkResult> &container)
533 const int count = container.count();
535 return QBenchmarkResult();
538 return container.at(0);
540 QList<QBenchmarkResult> containerCopy = container;
541 qSort(containerCopy);
543 const int middle = count / 2;
545 // ### handle even-sized containers here by doing an aritmetic mean of the two middle items.
546 return containerCopy.at(middle);
549 bool QuickTestResult::needsMoreMeasurements()
551 Q_D(QuickTestResult);
553 if (d->iterCount < QBenchmarkGlobalData::current->adjustMedianIterationCount())
555 if (QBenchmarkTestMethodData::current->resultsAccepted())
556 QTestLog::addBenchmarkResult(qMedian(d->results));
560 void QuickTestResult::startBenchmark(RunMode runMode, const QString &tag)
562 QBenchmarkTestMethodData::current->result = QBenchmarkResult();
563 QBenchmarkTestMethodData::current->resultAccepted = false;
564 QBenchmarkGlobalData::current->context.tag = tag;
565 QBenchmarkGlobalData::current->context.slotName = functionName();
567 Q_D(QuickTestResult);
568 delete d->benchmarkIter;
569 d->benchmarkIter = new QTest::QBenchmarkIterationController
570 (QTest::QBenchmarkIterationController::RunMode(runMode));
573 bool QuickTestResult::isBenchmarkDone() const
575 Q_D(const QuickTestResult);
576 if (d->benchmarkIter)
577 return d->benchmarkIter->isDone();
582 void QuickTestResult::nextBenchmark()
584 Q_D(QuickTestResult);
585 if (d->benchmarkIter)
586 d->benchmarkIter->next();
589 void QuickTestResult::stopBenchmark()
591 Q_D(QuickTestResult);
592 delete d->benchmarkIter;
593 d->benchmarkIter = 0;
597 void qtest_qParseArgs(int argc, char *argv[], bool qml);
600 void QuickTestResult::parseArgs(int argc, char *argv[])
602 if (!QBenchmarkGlobalData::current)
603 QBenchmarkGlobalData::current = &globalBenchmarkData;
604 QTest::qtest_qParseArgs(argc, argv, true);
607 void QuickTestResult::setProgramName(const char *name)
610 QTestResult::reset();
611 } else if (!name && loggingStarted) {
612 QTestResult::setCurrentTestObject(globalProgramName);
613 QTestLog::stopLogging();
614 QTestResult::setCurrentTestObject(0);
616 globalProgramName = name;
619 int QuickTestResult::exitCode()
621 #if defined(QTEST_NOEXITCODE)
624 // make sure our exit code is never going above 127
625 // since that could wrap and indicate 0 test fails
626 return qMin(QTestResult::failCount(), 127);