1 /****************************************************************************
3 ** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
4 ** Contact: http://www.qt-project.org/
6 ** This file is part of the test suite of the Qt Toolkit.
8 ** $QT_BEGIN_LICENSE:LGPL$
9 ** GNU Lesser General Public License Usage
10 ** This file may be used under the terms of the GNU Lesser General Public
11 ** License version 2.1 as published by the Free Software Foundation and
12 ** appearing in the file LICENSE.LGPL included in the packaging of this
13 ** file. Please review the following information to ensure the GNU Lesser
14 ** General Public License version 2.1 requirements will be met:
15 ** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
17 ** In addition, as a special exception, Nokia gives you certain additional
18 ** rights. These rights are described in the Nokia Qt LGPL Exception
19 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
21 ** GNU General Public License Usage
22 ** Alternatively, this file may be used under the terms of the GNU General
23 ** Public License version 3.0 as published by the Free Software Foundation
24 ** and appearing in the file LICENSE.GPL included in the packaging of this
25 ** file. Please review the following information to ensure the GNU General
26 ** Public License version 3.0 requirements will be met:
27 ** http://www.gnu.org/copyleft/gpl.html.
30 ** Alternatively, this file may be used in accordance with the terms and
31 ** conditions contained in a signed written agreement between you and Nokia.
40 ****************************************************************************/
42 #include "quicktestresult_p.h"
43 #include <QtTest/qtestcase.h>
44 #include <QtTest/qtestsystem.h>
45 #include <QtTest/private/qtestresult_p.h>
46 #include <QtTest/private/qtesttable_p.h>
47 #include <QtTest/private/qtestlog_p.h>
48 #include "qtestoptions_p.h"
49 #include <QtTest/qbenchmark.h>
50 #include <QtTest/private/qbenchmark_p.h>
51 #include <QtCore/qset.h>
52 #include <QtCore/qmap.h>
53 #include <QtCore/qbytearray.h>
54 #include <QtCore/qcoreapplication.h>
55 #include <QtCore/qdebug.h>
56 #include <QtCore/QUrl>
57 #include <QtCore/QDir>
58 #include <QtQuick/qquickwindow.h>
59 #include <QtGui/qvector3d.h>
60 #include <QtQml/private/qqmlglobal_p.h>
64 static const char *globalProgramName = 0;
65 static bool loggingStarted = false;
66 static QBenchmarkGlobalData globalBenchmarkData;
68 extern bool qWaitForSignal(QObject *obj, const char* signal, int timeout = 5000);
70 class Q_QUICK_TEST_EXPORT QuickTestImageObject : public QObject
74 QuickTestImageObject(const QImage& img, QObject *parent = 0)
80 ~QuickTestImageObject() {}
83 int red(int x, int y) const
85 return pixel(x, y).value<QColor>().red();
88 int green(int x, int y) const
90 return pixel(x, y).value<QColor>().green();
93 int blue(int x, int y) const
95 return pixel(x, y).value<QColor>().blue();
98 int alpha(int x, int y) const
100 return pixel(x, y).value<QColor>().alpha();
103 QVariant pixel(int x, int y) const
106 || x >= m_image.width()
107 || y >= m_image.height()
110 || x * y >= m_image.width() * m_image.height())
113 const QRgb* pixel = reinterpret_cast<const QRgb*>(m_image.constScanLine(y));
115 return QColor::fromRgba(*pixel);
121 class QuickTestResultPrivate
124 QuickTestResultPrivate()
131 ~QuickTestResultPrivate()
134 delete benchmarkIter;
135 delete benchmarkData;
138 QByteArray intern(const QString &str);
140 QString testCaseName;
141 QString functionName;
142 QSet<QByteArray> internedStrings;
144 QTest::QBenchmarkIterationController *benchmarkIter;
145 QBenchmarkTestMethodData *benchmarkData;
147 QList<QBenchmarkResult> results;
150 QByteArray QuickTestResultPrivate::intern(const QString &str)
152 QByteArray bstr = str.toUtf8();
153 return *(internedStrings.insert(bstr));
156 QuickTestResult::QuickTestResult(QObject *parent)
157 : QObject(parent), d_ptr(new QuickTestResultPrivate)
159 if (!QBenchmarkGlobalData::current)
160 QBenchmarkGlobalData::current = &globalBenchmarkData;
163 QuickTestResult::~QuickTestResult()
168 \qmlproperty string TestResult::testCaseName
170 This property defines the name of current TestCase element
171 that is running test cases.
175 QString QuickTestResult::testCaseName() const
177 Q_D(const QuickTestResult);
178 return d->testCaseName;
181 void QuickTestResult::setTestCaseName(const QString &name)
183 Q_D(QuickTestResult);
184 d->testCaseName = name;
185 emit testCaseNameChanged();
189 \qmlproperty string TestResult::functionName
191 This property defines the name of current test function
192 within a TestCase element that is running. If this string is
193 empty, then no function is currently running.
197 QString QuickTestResult::functionName() const
199 Q_D(const QuickTestResult);
200 return d->functionName;
203 void QuickTestResult::setFunctionName(const QString &name)
205 Q_D(QuickTestResult);
206 if (!name.isEmpty()) {
207 if (d->testCaseName.isEmpty()) {
208 QTestResult::setCurrentTestFunction
209 (d->intern(name).constData());
211 QString fullName = d->testCaseName + QLatin1String("::") + name;
212 QTestResult::setCurrentTestFunction
213 (d->intern(fullName).constData());
216 QTestResult::setCurrentTestFunction(0);
218 d->functionName = name;
219 emit functionNameChanged();
223 \qmlproperty string TestResult::dataTag
225 This property defines the tag for the current row in a
226 data-driven test, or an empty string if not a data-driven test.
228 QString QuickTestResult::dataTag() const
230 const char *tag = QTestResult::currentDataTag();
232 return QString::fromUtf8(tag);
237 void QuickTestResult::setDataTag(const QString &tag)
239 if (!tag.isEmpty()) {
240 QTestData *data = &(QTest::newRow(tag.toUtf8().constData()));
241 QTestResult::setCurrentTestData(data);
242 emit dataTagChanged();
244 QTestResult::setCurrentTestData(0);
249 \qmlproperty bool TestResult::failed
251 This property returns true if the current test function (or
252 current test data row for a data-driven test) has failed;
253 false otherwise. The fail state is reset when functionName
254 is changed or finishTestDataCleanup() is called.
258 bool QuickTestResult::isFailed() const
260 return QTestResult::currentTestFailed();
264 \qmlproperty bool TestResult::skipped
266 This property returns true if the current test function was
267 marked as skipped; false otherwise.
271 bool QuickTestResult::isSkipped() const
273 return QTestResult::skipCurrentTest();
276 void QuickTestResult::setSkipped(bool skip)
278 QTestResult::setSkipCurrentTest(skip);
279 emit skippedChanged();
283 \qmlproperty int TestResult::passCount
285 This property returns the number of tests that have passed.
287 \sa failCount, skipCount
289 int QuickTestResult::passCount() const
291 return QTestLog::passCount();
295 \qmlproperty int TestResult::failCount
297 This property returns the number of tests that have failed.
299 \sa passCount, skipCount
301 int QuickTestResult::failCount() const
303 return QTestLog::failCount();
307 \qmlproperty int TestResult::skipCount
309 This property returns the number of tests that have been skipped.
311 \sa passCount, failCount
313 int QuickTestResult::skipCount() const
315 return QTestLog::skipCount();
319 \qmlproperty list<string> TestResult::functionsToRun
321 This property returns the list of function names to be run.
323 QStringList QuickTestResult::functionsToRun() const
325 return QTest::testFunctions;
329 \qmlmethod TestResult::reset()
331 Resets all pass/fail/skip counters and prepare for testing.
333 void QuickTestResult::reset()
335 if (!globalProgramName) // Only if run via qmlviewer.
336 QTestResult::reset();
340 \qmlmethod TestResult::startLogging()
342 Starts logging to the test output stream and writes the
347 void QuickTestResult::startLogging()
349 // The program name is used for logging headers and footers if it
350 // is set. Otherwise the test case name is used.
353 QTestLog::startLogging();
354 loggingStarted = true;
358 \qmlmethod TestResult::stopLogging()
360 Writes the test footer to the test output stream and then stops logging.
364 void QuickTestResult::stopLogging()
366 Q_D(QuickTestResult);
367 if (globalProgramName)
368 return; // Logging will be stopped by setProgramName(0).
369 QTestResult::setCurrentTestObject(d->intern(d->testCaseName).constData());
370 QTestLog::stopLogging();
373 void QuickTestResult::initTestTable()
375 Q_D(QuickTestResult);
377 d->table = new QTestTable;
378 //qmltest does not really need the column for data driven test
379 //add this to avoid warnings.
380 d->table->addColumn(qMetaTypeId<QString>(), "qmltest_dummy_data_column");
383 void QuickTestResult::clearTestTable()
385 Q_D(QuickTestResult);
390 void QuickTestResult::finishTestData()
392 QTestResult::finishedCurrentTestData();
395 void QuickTestResult::finishTestDataCleanup()
397 QTestResult::finishedCurrentTestDataCleanup();
400 void QuickTestResult::finishTestFunction()
402 QTestResult::finishedCurrentTestFunction();
405 static QString qtestFixUrl(const QUrl &location)
407 if (location.isLocalFile()) // Use QUrl's logic for Windows drive letters.
408 return QDir::toNativeSeparators(location.toLocalFile());
409 return location.toString();
412 void QuickTestResult::fail
413 (const QString &message, const QUrl &location, int line)
415 QTestResult::addFailure(message.toLatin1().constData(),
416 qtestFixUrl(location).toLatin1().constData(), line);
419 bool QuickTestResult::verify
420 (bool success, const QString &message, const QUrl &location, int line)
422 if (!success && message.isEmpty()) {
423 return QTestResult::verify
424 (success, "verify()", "",
425 qtestFixUrl(location).toLatin1().constData(), line);
427 return QTestResult::verify
428 (success, message.toLatin1().constData(), "",
429 qtestFixUrl(location).toLatin1().constData(), line);
433 bool QuickTestResult::fuzzyCompare(const QVariant &actual, const QVariant &expected, qreal delta)
435 if (actual.type() == QVariant::Color || expected.type() == QVariant::Color) {
436 if (!actual.canConvert(QVariant::Color) || !expected.canConvert(QVariant::Color))
439 //fuzzy color comparison
444 QVariant var = QQml_colorProvider()->colorFromString(actual.toString(), &ok);
447 act = var.value<QColor>();
449 QQml_colorProvider()->colorFromString(expected.toString(), &ok);
452 exp = var.value<QColor>();
454 return ( qAbs(act.red() - exp.red()) <= delta
455 && qAbs(act.green() - exp.green()) <= delta
456 && qAbs(act.blue() - exp.blue()) <= delta
457 && qAbs(act.alpha() - exp.alpha()) <= delta);
461 qreal act = actual.toFloat(&ok);
465 qreal exp = expected.toFloat(&ok);
469 return (qAbs(act - exp) <= delta);
475 void QuickTestResult::stringify(QQmlV8Function *args)
477 if (args->Length() < 1)
478 args->returnValue(v8::Null());
480 v8::Local<v8::Value> value = (*args)[0];
483 QV8Engine *engine = args->engine();
485 //Check for Object Type
486 if (value->IsObject()
487 && !value->IsFunction()
490 && !value->IsRegExp()) {
491 QVariant v = engine->toVariant(value, QMetaType::UnknownType);
494 case QVariant::Vector3D:
496 QVector3D v3d = v.value<QVector3D>();
497 result = QString::fromLatin1("Qt.vector3d(%1, %2, %3)").arg(v3d.x()).arg(v3d.y()).arg(v3d.z());
501 result = v.toString();
505 result = QLatin1String("Object");
508 v8::Local<v8::String> jsstr = value->ToString();
509 QString tmp = engine->toString(jsstr);
510 if (value->IsArray())
511 result.append(QString::fromLatin1("[%1]").arg(tmp));
516 args->returnValue(args->engine()->toString(result));
519 bool QuickTestResult::compare
520 (bool success, const QString &message,
521 const QVariant &val1, const QVariant &val2,
522 const QUrl &location, int line)
524 return QTestResult::compare
525 (success, message.toLocal8Bit().constData(),
526 QTest::toString(val1.toString().toLatin1().constData()),
527 QTest::toString(val2.toString().toLatin1().constData()),
529 qtestFixUrl(location).toLatin1().constData(), line);
532 void QuickTestResult::skip
533 (const QString &message, const QUrl &location, int line)
535 QTestResult::addSkip(message.toLatin1().constData(),
536 qtestFixUrl(location).toLatin1().constData(), line);
537 QTestResult::setSkipCurrentTest(true);
540 bool QuickTestResult::expectFail
541 (const QString &tag, const QString &comment, const QUrl &location, int line)
543 return QTestResult::expectFail
544 (tag.toLatin1().constData(),
545 QTest::toString(comment.toLatin1().constData()),
546 QTest::Abort, qtestFixUrl(location).toLatin1().constData(), line);
549 bool QuickTestResult::expectFailContinue
550 (const QString &tag, const QString &comment, const QUrl &location, int line)
552 return QTestResult::expectFail
553 (tag.toLatin1().constData(),
554 QTest::toString(comment.toLatin1().constData()),
555 QTest::Continue, qtestFixUrl(location).toLatin1().constData(), line);
558 void QuickTestResult::warn(const QString &message, const QUrl &location, int line)
560 QTestLog::warn(message.toLatin1().constData(), qtestFixUrl(location).toLatin1().constData(), line);
563 void QuickTestResult::ignoreWarning(const QString &message)
565 QTestLog::ignoreMessage(QtWarningMsg, message.toLatin1().constData());
568 void QuickTestResult::wait(int ms)
573 void QuickTestResult::sleep(int ms)
578 bool QuickTestResult::waitForRendering(QQuickItem *item, int timeout)
582 return qWaitForSignal(item->canvas(), SIGNAL(frameSwapped()), timeout);
585 void QuickTestResult::startMeasurement()
587 Q_D(QuickTestResult);
588 delete d->benchmarkData;
589 d->benchmarkData = new QBenchmarkTestMethodData();
590 QBenchmarkTestMethodData::current = d->benchmarkData;
591 d->iterCount = (QBenchmarkGlobalData::current->measurer->needsWarmupIteration()) ? -1 : 0;
595 void QuickTestResult::beginDataRun()
597 QBenchmarkTestMethodData::current->beginDataRun();
600 void QuickTestResult::endDataRun()
602 Q_D(QuickTestResult);
603 QBenchmarkTestMethodData::current->endDataRun();
604 if (d->iterCount > -1) // iteration -1 is the warmup iteration.
605 d->results.append(QBenchmarkTestMethodData::current->result);
607 if (QBenchmarkGlobalData::current->verboseOutput) {
608 if (d->iterCount == -1) {
609 qDebug() << "warmup stage result :" << QBenchmarkTestMethodData::current->result.value;
611 qDebug() << "accumulation stage result:" << QBenchmarkTestMethodData::current->result.value;
616 bool QuickTestResult::measurementAccepted()
618 return QBenchmarkTestMethodData::current->resultsAccepted();
621 static QBenchmarkResult qMedian(const QList<QBenchmarkResult> &container)
623 const int count = container.count();
625 return QBenchmarkResult();
628 return container.at(0);
630 QList<QBenchmarkResult> containerCopy = container;
631 qSort(containerCopy);
633 const int middle = count / 2;
635 // ### handle even-sized containers here by doing an aritmetic mean of the two middle items.
636 return containerCopy.at(middle);
639 bool QuickTestResult::needsMoreMeasurements()
641 Q_D(QuickTestResult);
643 if (d->iterCount < QBenchmarkGlobalData::current->adjustMedianIterationCount())
645 if (QBenchmarkTestMethodData::current->resultsAccepted())
646 QTestLog::addBenchmarkResult(qMedian(d->results));
650 void QuickTestResult::startBenchmark(RunMode runMode, const QString &tag)
652 QBenchmarkTestMethodData::current->result = QBenchmarkResult();
653 QBenchmarkTestMethodData::current->resultAccepted = false;
654 QBenchmarkGlobalData::current->context.tag = tag;
655 QBenchmarkGlobalData::current->context.slotName = functionName();
657 Q_D(QuickTestResult);
658 delete d->benchmarkIter;
659 d->benchmarkIter = new QTest::QBenchmarkIterationController
660 (QTest::QBenchmarkIterationController::RunMode(runMode));
663 bool QuickTestResult::isBenchmarkDone() const
665 Q_D(const QuickTestResult);
666 if (d->benchmarkIter)
667 return d->benchmarkIter->isDone();
672 void QuickTestResult::nextBenchmark()
674 Q_D(QuickTestResult);
675 if (d->benchmarkIter)
676 d->benchmarkIter->next();
679 void QuickTestResult::stopBenchmark()
681 Q_D(QuickTestResult);
682 delete d->benchmarkIter;
683 d->benchmarkIter = 0;
686 QObject *QuickTestResult::grabImage(QQuickItem *item)
689 QQuickWindow *window = item->window();
690 QImage grabbed = window->grabWindow();
691 QRectF rf(item->x(), item->y(), item->width(), item->height());
692 rf = rf.intersected(QRectF(0, 0, grabbed.width(), grabbed.height()));
693 return new QuickTestImageObject(grabbed.copy(rf.toAlignedRect()));
698 void qtest_qParseArgs(int argc, char *argv[], bool qml);
701 void QuickTestResult::parseArgs(int argc, char *argv[])
703 if (!QBenchmarkGlobalData::current)
704 QBenchmarkGlobalData::current = &globalBenchmarkData;
705 QTest::qtest_qParseArgs(argc, argv, true);
708 void QuickTestResult::setProgramName(const char *name)
711 QTestResult::reset();
712 } else if (!name && loggingStarted) {
713 QTestResult::setCurrentTestObject(globalProgramName);
714 QTestLog::stopLogging();
715 QTestResult::setCurrentTestObject(0);
717 globalProgramName = name;
718 QTestResult::setCurrentTestObject(globalProgramName);
721 void QuickTestResult::setCurrentAppname(const char *appname)
723 QTestResult::setCurrentAppname(appname);
726 int QuickTestResult::exitCode()
728 #if defined(QTEST_NOEXITCODE)
731 // make sure our exit code is never going above 127
732 // since that could wrap and indicate 0 test fails
733 return qMin(QTestLog::failCount(), 127);
737 #include "quicktestresult.moc"