1 /****************************************************************************
3 ** Copyright (C) 2012 Digia Plc and/or its subsidiary(-ies).
4 ** Contact: http://www.qt-project.org/legal
6 ** This file is part of the test suite of the Qt Toolkit.
8 ** $QT_BEGIN_LICENSE:LGPL$
9 ** Commercial License Usage
10 ** Licensees holding valid commercial Qt licenses may use this file in
11 ** accordance with the commercial license agreement provided with the
12 ** Software or, alternatively, in accordance with the terms contained in
13 ** a written agreement between you and Digia. For licensing terms and
14 ** conditions see http://qt.digia.com/licensing. For further information
15 ** use the contact form at http://qt.digia.com/contact-us.
17 ** GNU Lesser General Public License Usage
18 ** Alternatively, this file may be used under the terms of the GNU Lesser
19 ** General Public License version 2.1 as published by the Free Software
20 ** Foundation and appearing in the file LICENSE.LGPL included in the
21 ** packaging of this file. Please review the following information to
22 ** ensure the GNU Lesser General Public License version 2.1 requirements
23 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
25 ** In addition, as a special exception, Digia gives you certain additional
26 ** rights. These rights are described in the Digia Qt LGPL Exception
27 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
29 ** GNU General Public License Usage
30 ** Alternatively, this file may be used under the terms of the GNU
31 ** General Public License version 3.0 as published by the Free Software
32 ** Foundation and appearing in the file LICENSE.GPL included in the
33 ** packaging of this file. Please review the following information to
34 ** ensure the GNU General Public License version 3.0 requirements will be
35 ** met: http://www.gnu.org/copyleft/gpl.html.
40 ****************************************************************************/
51 using namespace QPatternistSDK;
53 const char *const Worker::m_indent = " ";
55 Worker::Worker(QEventLoop &ev,
56 const QFileInfo &baseline,
57 const QFileInfo &result) : m_finishedCount(0)
58 , m_baselineFile(baseline)
59 , m_resultFile(result)
64 void Worker::list(QTextStream &out, const QString &msg, QStringList &list)
66 Q_ASSERT(!msg.isEmpty());
71 list.sort(); /* Make it pretty, and easy to read. */
75 const QStringList::const_iterator end(list.constEnd());
76 QStringList::const_iterator it(list.constBegin());
78 for(; it != end; ++it)
79 out << m_indent << qPrintable(*it) << '\n';
82 static inline int count(const ResultThreader::Hash &list, const TestResult::Status stat)
84 const ResultThreader::Hash::const_iterator end(list.constEnd());
85 ResultThreader::Hash::const_iterator it(list.constBegin());
88 for(; it != end; ++it)
90 if(it.value() == stat)
97 void Worker::threadFinished()
100 Q_ASSERT(m_finishedCount == 1 || m_finishedCount == 2);
102 const ResultThreader *const handler = static_cast<ResultThreader *>(sender());
105 switch(handler->type())
107 case ResultThreader::Baseline:
109 m_baseline = handler->result();
112 case ResultThreader::Result:
113 m_result = handler->result();
116 if(m_finishedCount == 1) /* One thread's missing. */
119 /* Ok, both threads have now finished, and we got their results in m_result and m_baseline. */
121 /* No matter how this function exits, we want to delete this Worker. */
124 ResultThreader::Hash::const_iterator itA(m_result.constBegin());
125 ResultThreader::Hash::const_iterator itB(m_baseline.constBegin());
126 const ResultThreader::Hash::const_iterator endA(m_result.constEnd());
127 const ResultThreader::Hash::const_iterator endB(m_baseline.constEnd());
128 const int baselineCount = m_baseline.count();
129 const int resultCount = m_result.count();
131 /* If you want useful output, change the QTextStream to use stderr. */
132 //QTextStream err(stderr);
134 QTextStream err(&out);
136 if(resultCount < baselineCount)
138 err << qPrintable(QString(QLatin1String("WARNING: Test result contains %1 reports, "
139 "but the baseline contains %2, a DECREASE "
143 .arg(resultCount - baselineCount));
145 else if(resultCount > baselineCount)
147 err << qPrintable(QString(QLatin1String("NOTE: The number of tests run is more than what "
148 "the baseline specifies. Run was %1 test cases, the "
149 "baseline specifies %2; an increase of %3 tests.\n"))
152 .arg(resultCount - baselineCount));
155 for(; itA != endA; ++itA)
157 const TestResult::Status result = itA.value();
158 const TestResult::Status baseline = m_baseline.value(itA.key());
160 if(result == baseline) /* We have no change. */
162 if(result == TestResult::NotTested)
163 m_notTested.append(itA.key());
167 else if(baseline == TestResult::Pass && result == TestResult::Fail)
168 m_unexpectedFailures.append(itA.key());
169 else if(baseline == TestResult::Fail && result == TestResult::Pass)
170 m_unexpectedPasses.append(itA.key());
173 list(err, QLatin1String("Not tested"), m_notTested);
174 list(err, QLatin1String("Unexpected failures"), m_unexpectedFailures);
175 list(err, QLatin1String("Unexpected passes"), m_unexpectedPasses);
178 typedef QPair<QString, int> Info;
179 typedef QList<Info> InfoList;
182 const int totFail = count(m_result, TestResult::Fail);
183 const int totPass = count(m_result, TestResult::Pass);
184 const int total = resultCount;
185 const int notTested = m_notTested.count();
186 const int percentage = int((static_cast<double>(totPass) / total) * 100);
188 Q_ASSERT_X(percentage >= 0 && percentage <= 100, Q_FUNC_INFO,
189 qPrintable(QString(QLatin1String("Percentage was: %1")).arg(percentage)));
191 info.append(Info(QLatin1String("Total"), total));
192 info.append(Info(QLatin1String("Failures"), totFail));
193 info.append(Info(QLatin1String("Passes"), totPass));
194 info.append(Info(QLatin1String("Not tested"), notTested));
195 info.append(Info(QLatin1String("Pass percentage(%)"), percentage));
196 info.append(Info(QLatin1String("Unexpected failures"), m_unexpectedFailures.count()));
197 info.append(Info(QLatin1String("Unexpected passes"), m_unexpectedPasses.count()));
199 const InfoList::const_iterator end(info.constEnd());
200 InfoList::const_iterator it(info.constBegin());
202 /* List the statistics nicely in a row with padded columns. */
203 for(; it != end; ++it)
205 const QString result((((*it).first) + QLatin1Char(':')).leftJustified(22, QLatin1Char(' ')));
206 err << m_indent << qPrintable(result) << (*it).second << '\n';
209 if(!m_unexpectedFailures.isEmpty())
211 err << "FAILURE: Regressions discovered, baseline was not updated.\n";
213 QTextStream(stderr) << out;
214 m_eventLoop.exit(ExitCode::Regression);
217 else if(m_unexpectedPasses.isEmpty() && baselineCount == resultCount)
219 err << "Result was identical to the baseline, baseline was not updated.\n";
220 m_eventLoop.exit(ExitCode::Success);
224 /* Ok, we got unexpected successes and no regressions: let's update the baseline. */
226 QFile resultFile(m_resultFile.absoluteFilePath());
228 /* Remove the old file, otherwise QFile::copy() will fail. */
229 QDir baselineDir(m_baselineFile.absolutePath());
230 baselineDir.remove(m_baselineFile.fileName());
232 if(resultFile.copy(m_baselineFile.absoluteFilePath()))
234 /* Give a detailed message of what's going on. */
235 if(resultCount > baselineCount)
236 err << "More tests was run than specified in the baseline, updating the baseline.\n";
238 err << "Improvement, the baseline was updated.\n";
240 /* We actually flag this as an error, because the new baseline must be submitted. */
242 QTextStream(stderr) << out;
243 m_eventLoop.exit(ExitCode::Regression);
248 err << qPrintable(QString(QLatin1String("Encountered error when updating "
249 "the baseline: %1\n"))
250 .arg(resultFile.errorString()));
252 QTextStream(stderr) << out;
253 m_eventLoop.exit(ExitCode::WriteError);
258 // vim: et:ts=4:sw=4:sts=4