1 /*-------------------------------------------------------------------------
2 * drawElements Quality Program Test Executor
3 * ------------------------------------------
5 * Copyright 2014 The Android Open Source Project
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
21 * \brief Command line test executor.
22 *//*--------------------------------------------------------------------*/
24 #include "xeBatchExecutor.hpp"
25 #include "xeLocalTcpIpLink.hpp"
26 #include "xeTcpIpLink.hpp"
27 #include "xeTestCaseListParser.hpp"
28 #include "xeTestLogWriter.hpp"
29 #include "xeTestResultParser.hpp"
31 #include "deCommandLine.hpp"
32 #include "deDirectoryIterator.hpp"
33 #include "deStringUtil.hpp"
34 #include "deUniquePtr.hpp"
48 #if (DE_OS == DE_OS_UNIX) || (DE_OS == DE_OS_ANDROID) || (DE_OS == DE_OS_WIN32)
58 // Command line arguments.
62 DE_DECLARE_COMMAND_LINE_OPT(StartServer, string);
63 DE_DECLARE_COMMAND_LINE_OPT(Host, string);
64 DE_DECLARE_COMMAND_LINE_OPT(Port, int);
65 DE_DECLARE_COMMAND_LINE_OPT(CaseListDir, string);
66 DE_DECLARE_COMMAND_LINE_OPT(TestSet, vector<string>);
67 DE_DECLARE_COMMAND_LINE_OPT(ExcludeSet, vector<string>);
68 DE_DECLARE_COMMAND_LINE_OPT(ContinueFile, string);
69 DE_DECLARE_COMMAND_LINE_OPT(TestLogFile, string);
70 DE_DECLARE_COMMAND_LINE_OPT(InfoLogFile, string);
71 DE_DECLARE_COMMAND_LINE_OPT(Summary, bool);
73 // TargetConfiguration
74 DE_DECLARE_COMMAND_LINE_OPT(BinaryName, string);
75 DE_DECLARE_COMMAND_LINE_OPT(WorkingDir, string);
76 DE_DECLARE_COMMAND_LINE_OPT(CmdLineArgs, string);
78 void parseCommaSeparatedList (const char* src, vector<string>* dst)
80 std::istringstream inStr (src);
83 while (std::getline(inStr, comp, ','))
87 void registerOptions (de::cmdline::Parser& parser)
89 using de::cmdline::Option;
90 using de::cmdline::NamedValue;
92 static const NamedValue<bool> s_yesNo[] =
98 parser << Option<StartServer> ("s", "start-server", "Start local execserver. Path to the execserver binary.")
99 << Option<Host> ("c", "connect", "Connect to host. Address of the execserver.")
100 << Option<Port> ("p", "port", "TCP port of the execserver.", "50016")
101 << Option<CaseListDir> ("cd", "caselistdir", "Path to the directory containing test case XML files.", ".")
102 << Option<TestSet> ("t", "testset", "Comma-separated list of include filters.", parseCommaSeparatedList)
103 << Option<ExcludeSet> ("e", "exclude", "Comma-separated list of exclude filters.", parseCommaSeparatedList, "")
104 << Option<ContinueFile> (DE_NULL, "continue", "Continue execution by initializing results from existing test log.")
105 << Option<TestLogFile> ("o", "out", "Output test log filename.", "TestLog.qpa")
106 << Option<InfoLogFile> ("i", "info", "Output info log filename.", "InfoLog.txt")
107 << Option<Summary> (DE_NULL, "summary", "Print summary after running tests.", s_yesNo, "yes")
108 << Option<BinaryName> ("b", "binaryname", "Test binary path. Relative to working directory.", "<Unused>")
109 << Option<WorkingDir> ("wd", "workdir", "Working directory for the test execution.", ".")
110 << Option<CmdLineArgs> (DE_NULL, "cmdline", "Additional command line arguments for the test binary.", "");
129 xe::TargetConfiguration targetCfg;
131 string serverBinOrAddress;
134 vector<string> testset;
135 vector<string> exclude;
142 bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv)
144 de::cmdline::Parser parser;
145 de::cmdline::CommandLine opts;
149 opt::registerOptions(parser);
151 if (!parser.parse(argc-1, argv+1, &opts, std::cerr))
153 std::cout << argv[0] << " [options]\n";
154 parser.help(std::cout);
158 if (opts.hasOption<opt::StartServer>() && opts.hasOption<opt::Host>())
160 std::cout << "Invalid command line arguments. Both --start-server and --connect defined." << std::endl;
163 else if (!opts.hasOption<opt::StartServer>() && !opts.hasOption<opt::Host>())
165 std::cout << "Invalid command line arguments. Must define either --start-server or --connect." << std::endl;
169 if (!opts.hasOption<opt::TestSet>())
171 std::cout << "Invalid command line arguments. --testset not defined." << std::endl;
175 if (opts.hasOption<opt::StartServer>())
177 cmdLine.runMode = RUNMODE_START_SERVER;
178 cmdLine.serverBinOrAddress = opts.getOption<opt::StartServer>();
182 cmdLine.runMode = RUNMODE_CONNECT;
183 cmdLine.serverBinOrAddress = opts.getOption<opt::Host>();
186 if (opts.hasOption<opt::ContinueFile>())
188 cmdLine.inFile = opts.getOption<opt::ContinueFile>();
190 if (cmdLine.inFile.empty())
192 std::cout << "Invalid command line arguments. --continue argument is empty." << std::endl;
197 cmdLine.port = opts.getOption<opt::Port>();
198 cmdLine.caseListDir = opts.getOption<opt::CaseListDir>();
199 cmdLine.testset = opts.getOption<opt::TestSet>();
200 cmdLine.exclude = opts.getOption<opt::ExcludeSet>();
201 cmdLine.outFile = opts.getOption<opt::TestLogFile>();
202 cmdLine.infoFile = opts.getOption<opt::InfoLogFile>();
203 cmdLine.summary = opts.getOption<opt::Summary>();
204 cmdLine.targetCfg.binaryName = opts.getOption<opt::BinaryName>();
205 cmdLine.targetCfg.workingDir = opts.getOption<opt::WorkingDir>();
206 cmdLine.targetCfg.cmdLineArgs = opts.getOption<opt::CmdLineArgs>();
211 bool checkCasePathPatternMatch (const char* pattern, const char* casePath, bool isTestGroup)
218 char c = casePath[casePos];
219 char p = pattern[ptrnPos];
223 /* Recurse to rest of positions. */
227 if (checkCasePathPatternMatch(pattern+ptrnPos+1, casePath+next, isTestGroup))
230 if (casePath[next] == 0)
231 return DE_FALSE; /* No match found. */
237 else if (c == 0 && p == 0)
241 /* Incomplete match is ok for test groups. */
255 void readCaseList (xe::TestGroup* root, const char* filename)
257 xe::TestCaseListParser caseListParser;
258 std::ifstream in (filename, std::ios_base::binary);
263 caseListParser.init(root);
267 in.read((char*)&buf[0], sizeof(buf));
268 int numRead = (int)in.gcount();
271 caseListParser.parse(&buf[0], numRead);
273 if (numRead < (int)sizeof(buf))
278 void readCaseLists (xe::TestRoot& root, const char* caseListDir)
280 int testCaseListCount = 0;
281 de::DirectoryIterator iter (caseListDir);
283 for (; iter.hasItem(); iter.next())
285 de::FilePath item = iter.getItem();
287 if (item.getType() == de::FilePath::TYPE_FILE)
289 string baseName = item.getBaseName();
290 if (baseName.find("-cases.xml") == baseName.length()-10)
292 string packageName = baseName.substr(0, baseName.length()-10);
293 xe::TestGroup* package = root.createGroup(packageName.c_str(), "");
295 readCaseList(package, item.getPath());
301 if (testCaseListCount == 0)
302 throw xe::Error("Couldn't find test case lists from test case list directory: '" + string(caseListDir) + "'");
305 void addMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
307 for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
309 const xe::TestNode* child = group.getChild(childNdx);
310 const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP;
311 const string fullPath = child->getFullPath();
313 if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
317 // Recurse into group.
318 addMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
322 DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
329 void removeMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
331 for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
333 const xe::TestNode* child = group.getChild(childNdx);
334 const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP;
335 const string fullPath = child->getFullPath();
337 if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
341 // Recurse into group.
342 removeMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
346 DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
347 testSet.remove(child);
353 class BatchResultHandler : public xe::TestLogHandler
356 BatchResultHandler (xe::BatchResult* batchResult)
357 : m_batchResult(batchResult)
361 void setSessionInfo (const xe::SessionInfo& sessionInfo)
363 m_batchResult->getSessionInfo() = sessionInfo;
366 xe::TestCaseResultPtr startTestCaseResult (const char* casePath)
368 // \todo [2012-11-01 pyry] What to do with duplicate results?
369 if (m_batchResult->hasTestCaseResult(casePath))
370 return m_batchResult->getTestCaseResult(casePath);
372 return m_batchResult->createTestCaseResult(casePath);
375 void testCaseResultUpdated (const xe::TestCaseResultPtr&)
379 void testCaseResultComplete (const xe::TestCaseResultPtr&)
384 xe::BatchResult* m_batchResult;
387 void readLogFile (xe::BatchResult* batchResult, const char* filename)
389 std::ifstream in (filename, std::ifstream::binary|std::ifstream::in);
390 BatchResultHandler handler (batchResult);
391 xe::TestLogParser parser (&handler);
397 in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf));
398 numRead = (int)in.gcount();
403 parser.parse(&buf[0], numRead);
409 void printBatchResultSummary (const xe::TestNode* root, const xe::TestSet& testSet, const xe::BatchResult& batchResult)
411 int countByStatusCode[xe::TESTSTATUSCODE_LAST];
412 std::fill(&countByStatusCode[0], &countByStatusCode[0]+DE_LENGTH_OF_ARRAY(countByStatusCode), 0);
414 for (xe::ConstTestNodeIterator iter = xe::ConstTestNodeIterator::begin(root); iter != xe::ConstTestNodeIterator::end(root); ++iter)
416 const xe::TestNode* node = *iter;
417 if (node->getNodeType() == xe::TESTNODETYPE_TEST_CASE && testSet.hasNode(node))
419 const xe::TestCase* testCase = static_cast<const xe::TestCase*>(node);
421 xe::TestStatusCode statusCode = xe::TESTSTATUSCODE_PENDING;
422 testCase->getFullPath(fullPath);
424 // Parse result data if such exists.
425 if (batchResult.hasTestCaseResult(fullPath.c_str()))
427 xe::ConstTestCaseResultPtr resultData = batchResult.getTestCaseResult(fullPath.c_str());
428 xe::TestCaseResult result;
429 xe::TestResultParser parser;
431 xe::parseTestCaseResultFromData(&parser, &result, *resultData.get());
432 statusCode = result.statusCode;
435 countByStatusCode[statusCode] += 1;
439 printf("\nTest run summary:\n");
441 for (int code = 0; code < xe::TESTSTATUSCODE_LAST; code++)
443 if (countByStatusCode[code] > 0)
444 printf(" %20s: %5d\n", xe::getTestStatusCodeName((xe::TestStatusCode)code), countByStatusCode[code]);
446 totalCases += countByStatusCode[code];
448 printf(" %20s: %5d\n", "Total", totalCases);
451 void writeInfoLog (const xe::InfoLog& log, const char* filename)
453 std::ofstream out(filename, std::ios_base::binary);
454 XE_CHECK(out.good());
455 out.write((const char*)log.getBytes(), log.getSize());
459 xe::CommLink* createCommLink (const CommandLine& cmdLine)
461 if (cmdLine.runMode == RUNMODE_START_SERVER)
463 xe::LocalTcpIpLink* link = new xe::LocalTcpIpLink();
466 link->start(cmdLine.serverBinOrAddress.c_str(), DE_NULL, cmdLine.port);
475 else if (cmdLine.runMode == RUNMODE_CONNECT)
477 de::SocketAddress address;
479 address.setFamily(DE_SOCKETFAMILY_INET4);
480 address.setProtocol(DE_SOCKETPROTOCOL_TCP);
481 address.setHost(cmdLine.serverBinOrAddress.c_str());
482 address.setPort(cmdLine.port);
484 xe::TcpIpLink* link = new xe::TcpIpLink();
489 link->connect(address);
492 catch (const std::exception& error)
495 throw xe::Error("Failed to connect to ExecServer at: " + cmdLine.serverBinOrAddress + ":" + de::toString(cmdLine.port) + ", " + error.what());
510 #if (DE_OS == DE_OS_UNIX) || (DE_OS == DE_OS_ANDROID)
512 static xe::BatchExecutor* s_executor = DE_NULL;
514 void signalHandler (int, siginfo_t*, void*)
517 s_executor->cancel();
520 void setupSignalHandler (xe::BatchExecutor* executor)
522 s_executor = executor;
525 sa.sa_sigaction = signalHandler;
526 sa.sa_flags = SA_SIGINFO | SA_RESTART;
527 sigfillset(&sa.sa_mask);
529 sigaction(SIGINT, &sa, DE_NULL);
532 void resetSignalHandler (void)
536 sa.sa_handler = SIG_DFL;
537 sa.sa_flags = SA_RESTART;
538 sigfillset(&sa.sa_mask);
540 sigaction(SIGINT, &sa, DE_NULL);
541 s_executor = DE_NULL;
544 #elif (DE_OS == DE_OS_WIN32)
546 static xe::BatchExecutor* s_executor = DE_NULL;
548 void signalHandler (int)
551 s_executor->cancel();
554 void setupSignalHandler (xe::BatchExecutor* executor)
556 s_executor = executor;
557 signal(SIGINT, signalHandler);
560 void resetSignalHandler (void)
562 signal(SIGINT, SIG_DFL);
563 s_executor = DE_NULL;
568 void setupSignalHandler (xe::BatchExecutor*)
572 void resetSignalHandler (void)
578 void runExecutor (const CommandLine& cmdLine)
582 // Read case list definitions.
583 readCaseLists(root, cmdLine.caseListDir.c_str());
589 for (vector<string>::const_iterator filterIter = cmdLine.testset.begin(); filterIter != cmdLine.testset.end(); ++filterIter)
590 addMatchingCases(root, testSet, filterIter->c_str());
593 throw xe::Error("None of the test case lists contains tests matching any of the test sets.");
595 // Remove excluded cases.
596 for (vector<string>::const_iterator filterIter = cmdLine.exclude.begin(); filterIter != cmdLine.exclude.end(); ++filterIter)
597 removeMatchingCases(root, testSet, filterIter->c_str());
599 // Initialize batch result.
600 xe::BatchResult batchResult;
603 // Read existing results from input file (if supplied).
604 if (!cmdLine.inFile.empty())
605 readLogFile(&batchResult, cmdLine.inFile.c_str());
607 // Initialize commLink.
608 de::UniquePtr<xe::CommLink> commLink(createCommLink(cmdLine));
610 xe::BatchExecutor executor(cmdLine.targetCfg, commLink.get(), &root, testSet, &batchResult, &infoLog);
614 setupSignalHandler(&executor);
616 resetSignalHandler();
620 resetSignalHandler();
622 if (!cmdLine.outFile.empty())
624 xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str());
625 printf("Test log written to %s\n", cmdLine.outFile.c_str());
628 if (!cmdLine.infoFile.empty())
630 writeInfoLog(infoLog, cmdLine.infoFile.c_str());
631 printf("Info log written to %s\n", cmdLine.infoFile.c_str());
635 printBatchResultSummary(&root, testSet, batchResult);
640 if (!cmdLine.outFile.empty())
642 xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str());
643 printf("Test log written to %s\n", cmdLine.outFile.c_str());
646 if (!cmdLine.infoFile.empty())
648 writeInfoLog(infoLog, cmdLine.infoFile.c_str());
649 printf("Info log written to %s\n", cmdLine.infoFile.c_str());
653 printBatchResultSummary(&root, testSet, batchResult);
658 if (commLink->getState(err) == xe::COMMLINKSTATE_ERROR)
659 throw xe::Error(err);
665 int main (int argc, const char* const* argv)
669 if (!parseCommandLine(cmdLine, argc, argv))
674 runExecutor(cmdLine);
676 catch (const std::exception& e)
678 printf("%s\n", e.what());