1 /****************************************************************************
3 ** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
4 ** Contact: http://www.qt-project.org/
6 ** This file is part of the test suite of the Qt Toolkit.
8 ** $QT_BEGIN_LICENSE:LGPL$
9 ** GNU Lesser General Public License Usage
10 ** This file may be used under the terms of the GNU Lesser General Public
11 ** License version 2.1 as published by the Free Software Foundation and
12 ** appearing in the file LICENSE.LGPL included in the packaging of this
13 ** file. Please review the following information to ensure the GNU Lesser
14 ** General Public License version 2.1 requirements will be met:
15 ** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
17 ** In addition, as a special exception, Nokia gives you certain additional
18 ** rights. These rights are described in the Nokia Qt LGPL Exception
19 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
21 ** GNU General Public License Usage
22 ** Alternatively, this file may be used under the terms of the GNU General
23 ** Public License version 3.0 as published by the Free Software Foundation
24 ** and appearing in the file LICENSE.GPL included in the packaging of this
25 ** file. Please review the following information to ensure the GNU General
26 ** Public License version 3.0 requirements will be met:
27 ** http://www.gnu.org/copyleft/gpl.html.
30 ** Alternatively, this file may be used in accordance with the terms and
31 ** conditions contained in a signed written agreement between you and Nokia.
40 ****************************************************************************/
42 #include "quicktestresult_p.h"
43 #include <QtTest/qtestcase.h>
44 #include <QtTest/qtestsystem.h>
45 #include <QtTest/private/qtestresult_p.h>
46 #include <QtTest/private/qtesttable_p.h>
47 #include <QtTest/private/qtestlog_p.h>
48 #include "qtestoptions_p.h"
49 #include <QtTest/qbenchmark.h>
50 #include <QtTest/private/qbenchmark_p.h>
51 #include <QtCore/qset.h>
52 #include <QtCore/qmap.h>
53 #include <QtCore/qbytearray.h>
54 #include <QtCore/qcoreapplication.h>
55 #include <QtCore/qdebug.h>
56 #include <QtCore/QUrl>
57 #include <QtCore/QDir>
61 static const char *globalProgramName = 0;
62 static bool loggingStarted = false;
63 static QBenchmarkGlobalData globalBenchmarkData;
65 class QuickTestResultPrivate
68 QuickTestResultPrivate()
75 ~QuickTestResultPrivate()
82 QByteArray intern(const QString &str);
86 QSet<QByteArray> internedStrings;
88 QTest::QBenchmarkIterationController *benchmarkIter;
89 QBenchmarkTestMethodData *benchmarkData;
91 QList<QBenchmarkResult> results;
94 QByteArray QuickTestResultPrivate::intern(const QString &str)
96 QByteArray bstr = str.toUtf8();
97 return *(internedStrings.insert(bstr));
100 QuickTestResult::QuickTestResult(QObject *parent)
101 : QObject(parent), d_ptr(new QuickTestResultPrivate)
103 if (!QBenchmarkGlobalData::current)
104 QBenchmarkGlobalData::current = &globalBenchmarkData;
107 QuickTestResult::~QuickTestResult()
112 \qmlproperty string TestResult::testCaseName
114 This property defines the name of current TestCase element
115 that is running test cases.
119 QString QuickTestResult::testCaseName() const
121 Q_D(const QuickTestResult);
122 return d->testCaseName;
125 void QuickTestResult::setTestCaseName(const QString &name)
127 Q_D(QuickTestResult);
128 d->testCaseName = name;
129 emit testCaseNameChanged();
133 \qmlproperty string TestResult::functionName
135 This property defines the name of current test function
136 within a TestCase element that is running. If this string is
137 empty, then no function is currently running.
141 QString QuickTestResult::functionName() const
143 Q_D(const QuickTestResult);
144 return d->functionName;
147 void QuickTestResult::setFunctionName(const QString &name)
149 Q_D(QuickTestResult);
150 if (!name.isEmpty()) {
151 if (d->testCaseName.isEmpty()) {
152 QTestResult::setCurrentTestFunction
153 (d->intern(name).constData());
155 QString fullName = d->testCaseName + QLatin1String("::") + name;
156 QTestResult::setCurrentTestFunction
157 (d->intern(fullName).constData());
160 QTestResult::setCurrentTestFunction(0);
162 d->functionName = name;
163 emit functionNameChanged();
166 QuickTestResult::FunctionType QuickTestResult::functionType() const
168 return FunctionType(QTestResult::currentTestLocation());
171 void QuickTestResult::setFunctionType(FunctionType type)
173 QTestResult::setCurrentTestLocation(QTestResult::TestLocation(type));
174 emit functionTypeChanged();
178 \qmlproperty string TestResult::dataTag
180 This property defines the tag for the current row in a
181 data-driven test, or an empty string if not a data-driven test.
183 QString QuickTestResult::dataTag() const
185 const char *tag = QTestResult::currentDataTag();
187 return QString::fromUtf8(tag);
192 void QuickTestResult::setDataTag(const QString &tag)
194 if (!tag.isEmpty()) {
195 QTestData *data = &(QTest::newRow(tag.toUtf8().constData()));
196 QTestResult::setCurrentTestData(data);
197 emit dataTagChanged();
199 QTestResult::setCurrentTestData(0);
204 \qmlproperty bool TestResult::failed
206 This property returns true if the current test function has
207 failed; false otherwise. The fail state is reset when
208 functionName is changed or finishTestFunction() is called.
210 \sa skipped, dataFailed
212 bool QuickTestResult::isFailed() const
214 return QTestResult::testFailed();
218 \qmlproperty bool TestResult::dataFailed
220 This property returns true if the current data function has
221 failed; false otherwise. The fail state is reset when
222 functionName is changed or finishTestFunction() is called.
226 bool QuickTestResult::isDataFailed() const
228 return QTestResult::currentTestFailed();
232 \qmlproperty bool TestResult::skipped
234 This property returns true if the current test function was
235 marked as skipped; false otherwise.
239 bool QuickTestResult::isSkipped() const
241 return QTestResult::skipCurrentTest();
244 void QuickTestResult::setSkipped(bool skip)
246 QTestResult::setSkipCurrentTest(skip);
247 emit skippedChanged();
251 \qmlproperty int TestResult::passCount
253 This property returns the number of tests that have passed.
255 \sa failCount, skipCount
257 int QuickTestResult::passCount() const
259 return QTestLog::passCount();
263 \qmlproperty int TestResult::failCount
265 This property returns the number of tests that have failed.
267 \sa passCount, skipCount
269 int QuickTestResult::failCount() const
271 return QTestLog::failCount();
275 \qmlproperty int TestResult::skipCount
277 This property returns the number of tests that have been skipped.
279 \sa passCount, failCount
281 int QuickTestResult::skipCount() const
283 return QTestLog::skipCount();
287 \qmlproperty list<string> TestResult::functionsToRun
289 This property returns the list of function names to be run.
291 QStringList QuickTestResult::functionsToRun() const
293 return QTest::testFunctions;
297 \qmlmethod TestResult::reset()
299 Resets all pass/fail/skip counters and prepare for testing.
301 void QuickTestResult::reset()
303 if (!globalProgramName) // Only if run via qmlviewer.
304 QTestResult::reset();
308 \qmlmethod TestResult::startLogging()
310 Starts logging to the test output stream and writes the
315 void QuickTestResult::startLogging()
317 // The program name is used for logging headers and footers if it
318 // is set. Otherwise the test case name is used.
321 QTestLog::startLogging();
322 loggingStarted = true;
326 \qmlmethod TestResult::stopLogging()
328 Writes the test footer to the test output stream and then stops logging.
332 void QuickTestResult::stopLogging()
334 Q_D(QuickTestResult);
335 if (globalProgramName)
336 return; // Logging will be stopped by setProgramName(0).
337 QTestResult::setCurrentTestObject(d->intern(d->testCaseName).constData());
338 QTestLog::stopLogging();
341 void QuickTestResult::initTestTable()
343 Q_D(QuickTestResult);
345 d->table = new QTestTable;
346 //qmltest does not really need the column for data driven test
347 //add this to avoid warnings.
348 d->table->addColumn(qMetaTypeId<QString>(), "qmltest_dummy_data_column");
351 void QuickTestResult::clearTestTable()
353 Q_D(QuickTestResult);
358 void QuickTestResult::finishTestData()
360 QTestResult::finishedCurrentTestData();
363 void QuickTestResult::finishTestFunction()
365 QTestResult::finishedCurrentTestFunction();
368 static QString qtestFixUrl(const QUrl &location)
370 if (location.isLocalFile()) // Use QUrl's logic for Windows drive letters.
371 return QDir::toNativeSeparators(location.toLocalFile());
372 return location.toString();
375 void QuickTestResult::fail
376 (const QString &message, const QUrl &location, int line)
378 QTestResult::addFailure(message.toLatin1().constData(),
379 qtestFixUrl(location).toLatin1().constData(), line);
382 bool QuickTestResult::verify
383 (bool success, const QString &message, const QUrl &location, int line)
385 if (!success && message.isEmpty()) {
386 return QTestResult::verify
387 (success, "verify()", "",
388 qtestFixUrl(location).toLatin1().constData(), line);
390 return QTestResult::verify
391 (success, message.toLatin1().constData(), "",
392 qtestFixUrl(location).toLatin1().constData(), line);
396 bool QuickTestResult::compare
397 (bool success, const QString &message,
398 const QString &val1, const QString &val2,
399 const QUrl &location, int line)
402 return QTestResult::compare
403 (success, message.toLocal8Bit().constData(),
404 qtestFixUrl(location).toLatin1().constData(), line);
406 return QTestResult::compare
407 (success, message.toLocal8Bit().constData(),
408 QTest::toString(val1.toLatin1().constData()),
409 QTest::toString(val2.toLatin1().constData()),
411 qtestFixUrl(location).toLatin1().constData(), line);
415 void QuickTestResult::skip
416 (const QString &message, const QUrl &location, int line)
418 QTestResult::addSkip(message.toLatin1().constData(),
419 qtestFixUrl(location).toLatin1().constData(), line);
420 QTestResult::setSkipCurrentTest(true);
423 bool QuickTestResult::expectFail
424 (const QString &tag, const QString &comment, const QUrl &location, int line)
426 return QTestResult::expectFail
427 (tag.toLatin1().constData(),
428 QTest::toString(comment.toLatin1().constData()),
429 QTest::Abort, qtestFixUrl(location).toLatin1().constData(), line);
432 bool QuickTestResult::expectFailContinue
433 (const QString &tag, const QString &comment, const QUrl &location, int line)
435 return QTestResult::expectFail
436 (tag.toLatin1().constData(),
437 QTest::toString(comment.toLatin1().constData()),
438 QTest::Continue, qtestFixUrl(location).toLatin1().constData(), line);
441 void QuickTestResult::warn(const QString &message, const QUrl &location, int line)
443 QTestLog::warn(message.toLatin1().constData(), qtestFixUrl(location).toLatin1().constData(), line);
446 void QuickTestResult::ignoreWarning(const QString &message)
448 QTestLog::ignoreMessage(QtWarningMsg, message.toLatin1().constData());
451 void QuickTestResult::wait(int ms)
456 void QuickTestResult::sleep(int ms)
461 void QuickTestResult::startMeasurement()
463 Q_D(QuickTestResult);
464 delete d->benchmarkData;
465 d->benchmarkData = new QBenchmarkTestMethodData();
466 QBenchmarkTestMethodData::current = d->benchmarkData;
467 d->iterCount = (QBenchmarkGlobalData::current->measurer->needsWarmupIteration()) ? -1 : 0;
471 void QuickTestResult::beginDataRun()
473 QBenchmarkTestMethodData::current->beginDataRun();
476 void QuickTestResult::endDataRun()
478 Q_D(QuickTestResult);
479 QBenchmarkTestMethodData::current->endDataRun();
480 if (d->iterCount > -1) // iteration -1 is the warmup iteration.
481 d->results.append(QBenchmarkTestMethodData::current->result);
483 if (QBenchmarkGlobalData::current->verboseOutput) {
484 if (d->iterCount == -1) {
485 qDebug() << "warmup stage result :" << QBenchmarkTestMethodData::current->result.value;
487 qDebug() << "accumulation stage result:" << QBenchmarkTestMethodData::current->result.value;
492 bool QuickTestResult::measurementAccepted()
494 return QBenchmarkTestMethodData::current->resultsAccepted();
497 static QBenchmarkResult qMedian(const QList<QBenchmarkResult> &container)
499 const int count = container.count();
501 return QBenchmarkResult();
504 return container.at(0);
506 QList<QBenchmarkResult> containerCopy = container;
507 qSort(containerCopy);
509 const int middle = count / 2;
511 // ### handle even-sized containers here by doing an aritmetic mean of the two middle items.
512 return containerCopy.at(middle);
515 bool QuickTestResult::needsMoreMeasurements()
517 Q_D(QuickTestResult);
519 if (d->iterCount < QBenchmarkGlobalData::current->adjustMedianIterationCount())
521 if (QBenchmarkTestMethodData::current->resultsAccepted())
522 QTestLog::addBenchmarkResult(qMedian(d->results));
526 void QuickTestResult::startBenchmark(RunMode runMode, const QString &tag)
528 QBenchmarkTestMethodData::current->result = QBenchmarkResult();
529 QBenchmarkTestMethodData::current->resultAccepted = false;
530 QBenchmarkGlobalData::current->context.tag = tag;
531 QBenchmarkGlobalData::current->context.slotName = functionName();
533 Q_D(QuickTestResult);
534 delete d->benchmarkIter;
535 d->benchmarkIter = new QTest::QBenchmarkIterationController
536 (QTest::QBenchmarkIterationController::RunMode(runMode));
539 bool QuickTestResult::isBenchmarkDone() const
541 Q_D(const QuickTestResult);
542 if (d->benchmarkIter)
543 return d->benchmarkIter->isDone();
548 void QuickTestResult::nextBenchmark()
550 Q_D(QuickTestResult);
551 if (d->benchmarkIter)
552 d->benchmarkIter->next();
555 void QuickTestResult::stopBenchmark()
557 Q_D(QuickTestResult);
558 delete d->benchmarkIter;
559 d->benchmarkIter = 0;
563 void qtest_qParseArgs(int argc, char *argv[], bool qml);
566 void QuickTestResult::parseArgs(int argc, char *argv[])
568 if (!QBenchmarkGlobalData::current)
569 QBenchmarkGlobalData::current = &globalBenchmarkData;
570 QTest::qtest_qParseArgs(argc, argv, true);
573 void QuickTestResult::setProgramName(const char *name)
576 QTestResult::reset();
577 } else if (!name && loggingStarted) {
578 QTestResult::setCurrentTestObject(globalProgramName);
579 QTestLog::stopLogging();
580 QTestResult::setCurrentTestObject(0);
582 globalProgramName = name;
583 QTestResult::setCurrentTestObject(globalProgramName);
586 int QuickTestResult::exitCode()
588 #if defined(QTEST_NOEXITCODE)
591 // make sure our exit code is never going above 127
592 // since that could wrap and indicate 0 test fails
593 return qMin(QTestLog::failCount(), 127);