Merge "Fix extension support checks in negative api tests" into nougat-cts-dev am...
[platform/upstream/VK-GL-CTS.git] / executor / xeBatchExecutor.cpp
1 /*-------------------------------------------------------------------------
2  * drawElements Quality Program Test Executor
3  * ------------------------------------------
4  *
5  * Copyright 2014 The Android Open Source Project
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  *      http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  *
19  *//*!
20  * \file
21  * \brief Test batch executor.
22  *//*--------------------------------------------------------------------*/
23
24 #include "xeBatchExecutor.hpp"
25 #include "xeTestResultParser.hpp"
26
27 #include <sstream>
28 #include <cstdio>
29
30 namespace xe
31 {
32
33 using std::string;
34 using std::vector;
35
36 enum
37 {
38         TEST_LOG_TMP_BUFFER_SIZE        = 1024,
39         INFO_LOG_TMP_BUFFER_SIZE        = 256
40 };
41
42 // \todo [2012-11-01 pyry] Update execute set in handler.
43
44 static inline bool isExecutedInBatch (const BatchResult* batchResult, const TestCase* testCase)
45 {
46         std::string fullPath;
47         testCase->getFullPath(fullPath);
48
49         if (batchResult->hasTestCaseResult(fullPath.c_str()))
50         {
51                 ConstTestCaseResultPtr data = batchResult->getTestCaseResult(fullPath.c_str());
52                 return data->getStatusCode() != TESTSTATUSCODE_PENDING && data->getStatusCode() != TESTSTATUSCODE_RUNNING;
53         }
54         else
55                 return false;
56 }
57
58 // \todo [2012-06-19 pyry] These can be optimized using TestSetIterator (once implemented)
59
60 static void computeExecuteSet (TestSet& executeSet, const TestNode* root, const TestSet& testSet, const BatchResult* batchResult)
61 {
62         ConstTestNodeIterator   iter    = ConstTestNodeIterator::begin(root);
63         ConstTestNodeIterator   end             = ConstTestNodeIterator::end(root);
64
65         for (; iter != end; ++iter)
66         {
67                 const TestNode* node = *iter;
68
69                 if (node->getNodeType() == TESTNODETYPE_TEST_CASE && testSet.hasNode(node))
70                 {
71                         const TestCase* testCase = static_cast<const TestCase*>(node);
72
73                         if (!isExecutedInBatch(batchResult, testCase))
74                                 executeSet.addCase(testCase);
75                 }
76         }
77 }
78
79 static void computeBatchRequest (TestSet& requestSet, const TestSet& executeSet, const TestNode* root, int maxCasesInSet)
80 {
81         ConstTestNodeIterator   iter            = ConstTestNodeIterator::begin(root);
82         ConstTestNodeIterator   end                     = ConstTestNodeIterator::end(root);
83         int                                             numCases        = 0;
84
85         for (; (iter != end) && (numCases < maxCasesInSet); ++iter)
86         {
87                 const TestNode* node = *iter;
88
89                 if (node->getNodeType() == TESTNODETYPE_TEST_CASE && executeSet.hasNode(node))
90                 {
91                         const TestCase* testCase = static_cast<const TestCase*>(node);
92                         requestSet.addCase(testCase);
93                         numCases += 1;
94                 }
95         }
96 }
97
98 static int removeExecuted (TestSet& set, const TestNode* root, const BatchResult* batchResult)
99 {
100         TestSet                                 oldSet          (set);
101         ConstTestNodeIterator   iter            = ConstTestNodeIterator::begin(root);
102         ConstTestNodeIterator   end                     = ConstTestNodeIterator::end(root);
103         int                                             numRemoved      = 0;
104
105         for (; iter != end; ++iter)
106         {
107                 const TestNode* node = *iter;
108
109                 if (node->getNodeType() == TESTNODETYPE_TEST_CASE && oldSet.hasNode(node))
110                 {
111                         const TestCase* testCase = static_cast<const TestCase*>(node);
112
113                         if (isExecutedInBatch(batchResult, testCase))
114                         {
115                                 set.removeCase(testCase);
116                                 numRemoved += 1;
117                         }
118                 }
119         }
120
121         return numRemoved;
122 }
123
124 BatchExecutorLogHandler::BatchExecutorLogHandler (BatchResult* batchResult)
125         : m_batchResult(batchResult)
126 {
127 }
128
129 BatchExecutorLogHandler::~BatchExecutorLogHandler (void)
130 {
131 }
132
133 void BatchExecutorLogHandler::setSessionInfo (const SessionInfo& sessionInfo)
134 {
135         m_batchResult->getSessionInfo() = sessionInfo;
136 }
137
138 TestCaseResultPtr BatchExecutorLogHandler::startTestCaseResult (const char* casePath)
139 {
140         // \todo [2012-11-01 pyry] What to do with duplicate results?
141         if (m_batchResult->hasTestCaseResult(casePath))
142                 return m_batchResult->getTestCaseResult(casePath);
143         else
144                 return m_batchResult->createTestCaseResult(casePath);
145 }
146
147 void BatchExecutorLogHandler::testCaseResultUpdated (const TestCaseResultPtr&)
148 {
149 }
150
151 void BatchExecutorLogHandler::testCaseResultComplete (const TestCaseResultPtr& result)
152 {
153         // \todo [2012-11-01 pyry] Remove from execute set here instead of updating it between sessions.
154         printf("%s\n", result->getTestCasePath());
155 }
156
157 BatchExecutor::BatchExecutor (const TargetConfiguration& config, CommLink* commLink, const TestNode* root, const TestSet& testSet, BatchResult* batchResult, InfoLog* infoLog)
158         : m_config                      (config)
159         , m_commLink            (commLink)
160         , m_root                        (root)
161         , m_testSet                     (testSet)
162         , m_logHandler          (batchResult)
163         , m_batchResult         (batchResult)
164         , m_infoLog                     (infoLog)
165         , m_state                       (STATE_NOT_STARTED)
166         , m_testLogParser       (&m_logHandler)
167 {
168 }
169
170 BatchExecutor::~BatchExecutor (void)
171 {
172 }
173
174 void BatchExecutor::run (void)
175 {
176         XE_CHECK(m_state == STATE_NOT_STARTED);
177
178         // Check commlink state.
179         {
180                 CommLinkState   commState       = COMMLINKSTATE_LAST;
181                 std::string             stateStr        = "";
182
183                 commState = m_commLink->getState(stateStr);
184
185                 if (commState == COMMLINKSTATE_ERROR)
186                 {
187                         // Report error.
188                         XE_FAIL((string("CommLink error: '") + stateStr + "'").c_str());
189                 }
190                 else if (commState != COMMLINKSTATE_READY)
191                         XE_FAIL("CommLink is not ready");
192         }
193
194         // Compute initial execute set.
195         computeExecuteSet(m_casesToExecute, m_root, m_testSet, m_batchResult);
196
197         // Register callbacks.
198         m_commLink->setCallbacks(enqueueStateChanged, enqueueTestLogData, enqueueInfoLogData, this);
199
200         try
201         {
202                 if (!m_casesToExecute.empty())
203                 {
204                         TestSet batchRequest;
205                         computeBatchRequest(batchRequest, m_casesToExecute, m_root, m_config.maxCasesPerSession);
206                         launchTestSet(batchRequest);
207
208                         m_state = STATE_STARTED;
209                 }
210                 else
211                         m_state = STATE_FINISHED;
212
213                 // Run handler loop until we are finished.
214                 while (m_state != STATE_FINISHED)
215                         m_dispatcher.callNext();
216         }
217         catch (...)
218         {
219                 m_commLink->setCallbacks(DE_NULL, DE_NULL, DE_NULL, DE_NULL);
220                 throw;
221         }
222
223         // De-register callbacks.
224         m_commLink->setCallbacks(DE_NULL, DE_NULL, DE_NULL, DE_NULL);
225 }
226
227 void BatchExecutor::cancel (void)
228 {
229         m_state = STATE_FINISHED;
230         m_dispatcher.cancel();
231 }
232
233 void BatchExecutor::onStateChanged (CommLinkState state, const char* message)
234 {
235         switch (state)
236         {
237                 case COMMLINKSTATE_READY:
238                 case COMMLINKSTATE_TEST_PROCESS_LAUNCHING:
239                 case COMMLINKSTATE_TEST_PROCESS_RUNNING:
240                         break; // Ignore.
241
242                 case COMMLINKSTATE_TEST_PROCESS_FINISHED:
243                 {
244                         // Feed end of string to parser. This terminates open test case if such exists.
245                         {
246                                 deUint8 eos = 0;
247                                 onTestLogData(&eos, 1);
248                         }
249
250                         int numExecuted = removeExecuted(m_casesToExecute, m_root, m_batchResult);
251
252                         // \note No new batch is launched if no cases were executed in last one. Otherwise excutor
253                         //       could end up in infinite loop.
254                         if (!m_casesToExecute.empty() && numExecuted > 0)
255                         {
256                                 // Reset state and start batch.
257                                 m_testLogParser.reset();
258
259                                 m_commLink->reset();
260                                 XE_CHECK(m_commLink->getState() == COMMLINKSTATE_READY);
261
262                                 TestSet batchRequest;
263                                 computeBatchRequest(batchRequest, m_casesToExecute, m_root, m_config.maxCasesPerSession);
264                                 launchTestSet(batchRequest);
265                         }
266                         else
267                                 m_state = STATE_FINISHED;
268
269                         break;
270                 }
271
272                 case COMMLINKSTATE_TEST_PROCESS_LAUNCH_FAILED:
273                         printf("Failed to start test process: '%s'\n", message);
274                         m_state = STATE_FINISHED;
275                         break;
276
277                 case COMMLINKSTATE_ERROR:
278                         printf("CommLink error: '%s'\n", message);
279                         m_state = STATE_FINISHED;
280                         break;
281
282                 default:
283                         XE_FAIL("Unknown state");
284         }
285 }
286
287 void BatchExecutor::onTestLogData (const deUint8* bytes, size_t numBytes)
288 {
289         try
290         {
291                 m_testLogParser.parse(bytes, numBytes);
292         }
293         catch (const ParseError& e)
294         {
295                 // \todo [2012-07-06 pyry] Log error.
296                 DE_UNREF(e);
297         }
298 }
299
300 void BatchExecutor::onInfoLogData (const deUint8* bytes, size_t numBytes)
301 {
302         if (numBytes > 0 && m_infoLog)
303                 m_infoLog->append(bytes, numBytes);
304 }
305
306 static void writeCaseListNode (std::ostream& str, const TestNode* node, const TestSet& testSet)
307 {
308         DE_ASSERT(testSet.hasNode(node));
309
310         TestNodeType nodeType = node->getNodeType();
311
312         if (nodeType != TESTNODETYPE_ROOT)
313                 str << node->getName();
314
315         if (nodeType == TESTNODETYPE_ROOT || nodeType == TESTNODETYPE_GROUP)
316         {
317                 const TestGroup*        group   = static_cast<const TestGroup*>(node);
318                 bool                            isFirst = true;
319
320                 str << "{";
321
322                 for (int ndx = 0; ndx < group->getNumChildren(); ndx++)
323                 {
324                         const TestNode* child = group->getChild(ndx);
325
326                         if (testSet.hasNode(child))
327                         {
328                                 if (!isFirst)
329                                         str << ",";
330
331                                 writeCaseListNode(str, child, testSet);
332                                 isFirst = false;
333                         }
334                 }
335
336                 str << "}";
337         }
338 }
339
340 void BatchExecutor::launchTestSet (const TestSet& testSet)
341 {
342         std::ostringstream caseList;
343         XE_CHECK(testSet.hasNode(m_root));
344         XE_CHECK(m_root->getNodeType() == TESTNODETYPE_ROOT);
345         writeCaseListNode(caseList, m_root, testSet);
346
347         m_commLink->startTestProcess(m_config.binaryName.c_str(), m_config.cmdLineArgs.c_str(), m_config.workingDir.c_str(), caseList.str().c_str());
348 }
349
350 void BatchExecutor::enqueueStateChanged (void* userPtr, CommLinkState state, const char* message)
351 {
352         BatchExecutor*  executor        = static_cast<BatchExecutor*>(userPtr);
353         CallWriter              writer          (&executor->m_dispatcher, BatchExecutor::dispatchStateChanged);
354
355         writer << executor
356                    << state
357                    << message;
358
359         writer.enqueue();
360 }
361
362 void BatchExecutor::enqueueTestLogData (void* userPtr, const deUint8* bytes, size_t numBytes)
363 {
364         BatchExecutor*  executor        = static_cast<BatchExecutor*>(userPtr);
365         CallWriter              writer          (&executor->m_dispatcher, BatchExecutor::dispatchTestLogData);
366
367         writer << executor
368                    << numBytes;
369
370         writer.write(bytes, numBytes);
371         writer.enqueue();
372 }
373
374 void BatchExecutor::enqueueInfoLogData (void* userPtr, const deUint8* bytes, size_t numBytes)
375 {
376         BatchExecutor*  executor        = static_cast<BatchExecutor*>(userPtr);
377         CallWriter              writer          (&executor->m_dispatcher, BatchExecutor::dispatchInfoLogData);
378
379         writer << executor
380                    << numBytes;
381
382         writer.write(bytes, numBytes);
383         writer.enqueue();
384 }
385
386 void BatchExecutor::dispatchStateChanged (CallReader& data)
387 {
388         BatchExecutor*  executor        = DE_NULL;
389         CommLinkState   state           = COMMLINKSTATE_LAST;
390         std::string             message;
391
392         data >> executor
393                  >> state
394                  >> message;
395
396         executor->onStateChanged(state, message.c_str());
397 }
398
399 void BatchExecutor::dispatchTestLogData (CallReader& data)
400 {
401         BatchExecutor*  executor        = DE_NULL;
402         size_t                  numBytes;
403
404         data >> executor
405                  >> numBytes;
406
407         executor->onTestLogData(data.getDataBlock(numBytes), numBytes);
408 }
409
410 void BatchExecutor::dispatchInfoLogData (CallReader& data)
411 {
412         BatchExecutor*  executor        = DE_NULL;
413         size_t                  numBytes;
414
415         data >> executor
416                  >> numBytes;
417
418         executor->onInfoLogData(data.getDataBlock(numBytes), numBytes);
419 }
420
421 } // xe