1 /*-------------------------------------------------------------------------
2 * drawElements Quality Program Test Executor
3 * ------------------------------------------
5 * Copyright 2014 The Android Open Source Project
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
21 * \brief Test log compare utility.
22 *//*--------------------------------------------------------------------*/
24 #include "xeTestLogParser.hpp"
25 #include "xeTestResultParser.hpp"
26 #include "deFilePath.hpp"
28 #include "deThread.hpp"
29 #include "deCommandLine.hpp"
55 OUTPUTFORMAT_TEXT = 0,
63 OUTPUTVALUE_STATUS_CODE = 0,
64 OUTPUTVALUE_STATUS_DETAILS,
72 DE_DECLARE_COMMAND_LINE_OPT(OutMode, OutputMode);
73 DE_DECLARE_COMMAND_LINE_OPT(OutFormat, OutputFormat);
74 DE_DECLARE_COMMAND_LINE_OPT(OutValue, OutputValue);
76 static void registerOptions (de::cmdline::Parser& parser)
78 using de::cmdline::Option;
79 using de::cmdline::NamedValue;
81 static const NamedValue<OutputMode> s_outputModes[] =
83 { "all", OUTPUTMODE_ALL },
84 { "diff", OUTPUTMODE_DIFF }
86 static const NamedValue<OutputFormat> s_outputFormats[] =
88 { "text", OUTPUTFORMAT_TEXT },
89 { "csv", OUTPUTFORMAT_CSV }
91 static const NamedValue<OutputValue> s_outputValues[] =
93 { "code", OUTPUTVALUE_STATUS_CODE },
94 { "details", OUTPUTVALUE_STATUS_DETAILS }
97 parser << Option<OutFormat> ("f", "format", "Output format", s_outputFormats, "csv")
98 << Option<OutMode> ("m", "mode", "Output mode", s_outputModes, "all")
99 << Option<OutValue> ("v", "value", "Value to extract", s_outputValues, "code");
107 : outMode (OUTPUTMODE_ALL)
108 , outFormat (OUTPUTFORMAT_CSV)
109 , outValue (OUTPUTVALUE_STATUS_CODE)
114 OutputFormat outFormat;
115 OutputValue outValue;
116 vector<string> filenames;
119 struct ShortBatchResult
121 vector<xe::TestCaseResultHeader> resultHeaders;
122 map<string, int> resultMap;
125 class ShortResultHandler : public xe::TestLogHandler
128 ShortResultHandler (ShortBatchResult& result)
133 void setSessionInfo (const xe::SessionInfo&)
138 xe::TestCaseResultPtr startTestCaseResult (const char* casePath)
140 return xe::TestCaseResultPtr(new xe::TestCaseResultData(casePath));
143 void testCaseResultUpdated (const xe::TestCaseResultPtr&)
148 void testCaseResultComplete (const xe::TestCaseResultPtr& caseData)
150 xe::TestCaseResultHeader header;
151 int caseNdx = (int)m_result.resultHeaders.size();
153 header.casePath = caseData->getTestCasePath();
154 header.caseType = xe::TESTCASETYPE_SELF_VALIDATE;
155 header.statusCode = caseData->getStatusCode();
156 header.statusDetails = caseData->getStatusDetails();
158 if (header.statusCode == xe::TESTSTATUSCODE_LAST)
160 xe::TestCaseResult fullResult;
162 xe::parseTestCaseResultFromData(&m_testResultParser, &fullResult, *caseData.get());
164 header = xe::TestCaseResultHeader(fullResult);
167 // Insert into result list & map.
168 m_result.resultHeaders.push_back(header);
169 m_result.resultMap[header.casePath] = caseNdx;
173 ShortBatchResult& m_result;
174 xe::TestResultParser m_testResultParser;
177 static void readLogFile (ShortBatchResult& batchResult, const char* filename)
179 std::ifstream in (filename, std::ifstream::binary|std::ifstream::in);
180 ShortResultHandler resultHandler (batchResult);
181 xe::TestLogParser parser (&resultHandler);
187 in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf));
188 numRead = (int)in.gcount();
193 parser.parse(&buf[0], numRead);
199 class LogFileReader : public de::Thread
202 LogFileReader (ShortBatchResult& batchResult, const char* filename)
203 : m_batchResult (batchResult)
204 , m_filename (filename)
210 readLogFile(m_batchResult, m_filename.c_str());
214 ShortBatchResult& m_batchResult;
215 std::string m_filename;
218 static void computeCaseList (vector<string>& cases, const vector<ShortBatchResult>& batchResults)
220 // \todo [2012-07-10 pyry] Do proper case ordering (eg. handle missing cases nicely).
221 set<string> addedCases;
223 for (vector<ShortBatchResult>::const_iterator batchIter = batchResults.begin(); batchIter != batchResults.end(); batchIter++)
225 for (vector<xe::TestCaseResultHeader>::const_iterator caseIter = batchIter->resultHeaders.begin(); caseIter != batchIter->resultHeaders.end(); caseIter++)
227 if (addedCases.find(caseIter->casePath) == addedCases.end())
229 cases.push_back(caseIter->casePath);
230 addedCases.insert(caseIter->casePath);
236 static void getTestResultHeaders (vector<xe::TestCaseResultHeader>& headers, const vector<ShortBatchResult>& batchResults, const char* casePath)
238 headers.resize(batchResults.size());
240 for (int ndx = 0; ndx < (int)batchResults.size(); ndx++)
242 const ShortBatchResult& batchResult = batchResults[ndx];
243 map<string, int>::const_iterator resultPos = batchResult.resultMap.find(casePath);
245 if (resultPos != batchResult.resultMap.end())
246 headers[ndx] = batchResult.resultHeaders[resultPos->second];
249 headers[ndx].casePath = casePath;
250 headers[ndx].caseType = xe::TESTCASETYPE_SELF_VALIDATE;
251 headers[ndx].statusCode = xe::TESTSTATUSCODE_LAST;
256 static const char* getStatusCodeName (xe::TestStatusCode code)
258 if (code == xe::TESTSTATUSCODE_LAST)
261 return xe::getTestStatusCodeName(code);
264 static bool runCompare (const CommandLine& cmdLine, std::ostream& dst)
266 vector<ShortBatchResult> results;
267 vector<string> batchNames;
268 bool compareOk = true;
270 XE_CHECK(!cmdLine.filenames.empty());
274 // Read in batch results
275 results.resize(cmdLine.filenames.size());
277 std::vector<de::SharedPtr<LogFileReader> > readers;
279 for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++)
281 readers.push_back(de::SharedPtr<LogFileReader>(new LogFileReader(results[ndx], cmdLine.filenames[ndx].c_str())));
282 readers.back()->start();
285 for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++)
287 readers[ndx]->join();
289 // Use file name as batch name.
290 batchNames.push_back(de::FilePath(cmdLine.filenames[ndx].c_str()).getBaseName());
294 // Compute unified case list.
295 vector<string> caseList;
296 computeCaseList(caseList, results);
299 int numCases = (int)caseList.size();
302 if (cmdLine.outFormat == OUTPUTFORMAT_CSV)
304 dst << "TestCasePath";
305 for (vector<string>::const_iterator nameIter = batchNames.begin(); nameIter != batchNames.end(); nameIter++)
306 dst << "," << *nameIter;
311 for (vector<string>::const_iterator caseIter = caseList.begin(); caseIter != caseList.end(); caseIter++)
313 const string& caseName = *caseIter;
314 vector<xe::TestCaseResultHeader> headers;
315 bool allEqual = true;
317 getTestResultHeaders(headers, results, caseName.c_str());
319 for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin()+1; iter != headers.end(); iter++)
321 if (iter->statusCode != headers[0].statusCode)
331 if (cmdLine.outMode == OUTPUTMODE_ALL || !allEqual)
333 if (cmdLine.outFormat == OUTPUTFORMAT_TEXT)
335 dst << caseName << "\n";
336 for (int ndx = 0; ndx < (int)headers.size(); ndx++)
337 dst << " " << batchNames[ndx] << ": " << getStatusCodeName(headers[ndx].statusCode) << " (" << headers[ndx].statusDetails << ")\n";
340 else if (cmdLine.outFormat == OUTPUTFORMAT_CSV)
343 for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin(); iter != headers.end(); iter++)
344 dst << "," << (cmdLine.outValue == OUTPUTVALUE_STATUS_CODE ? getStatusCodeName(iter->statusCode) : iter->statusDetails.c_str());
350 compareOk = numEqual == numCases;
352 if (cmdLine.outFormat == OUTPUTFORMAT_TEXT)
354 dst << " " << numEqual << " / " << numCases << " test case results match.\n";
355 dst << " Comparison " << (compareOk ? "passed" : "FAILED") << "!\n";
358 catch (const std::exception& e)
360 printf("%s\n", e.what());
367 static bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv)
369 de::cmdline::Parser parser;
370 de::cmdline::CommandLine opts;
374 opt::registerOptions(parser);
376 if (!parser.parse(argc-1, &argv[1], &opts, std::cerr) ||
377 opts.getArgs().empty())
379 std::cout << argv[0] << ": [options] [filenames]\n";
380 parser.help(std::cout);
384 cmdLine.outFormat = opts.getOption<opt::OutFormat>();
385 cmdLine.outMode = opts.getOption<opt::OutMode>();
386 cmdLine.outValue = opts.getOption<opt::OutValue>();
387 cmdLine.filenames = opts.getArgs();
392 int main (int argc, const char* const* argv)
396 if (!parseCommandLine(cmdLine, argc, argv))
401 bool compareOk = runCompare(cmdLine, std::cout);
402 return compareOk ? 0 : -1;
404 catch (const std::exception& e)
406 printf("FATAL ERROR: %s\n", e.what());