2 * Copyright (C) 2014 The Android Open Source Project
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
16 package com.drawelements.deqp.runner;
18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
19 import com.android.compatibility.common.util.AbiUtils;
20 import com.android.ddmlib.IDevice;
21 import com.android.ddmlib.IShellOutputReceiver;
22 import com.android.ddmlib.ShellCommandUnresponsiveException;
23 import com.android.ddmlib.testrunner.TestIdentifier;
24 import com.android.tradefed.build.IFolderBuildInfo;
25 import com.android.tradefed.config.ConfigurationException;
26 import com.android.tradefed.config.OptionSetter;
27 import com.android.tradefed.device.DeviceNotAvailableException;
28 import com.android.tradefed.device.ITestDevice;
29 import com.android.tradefed.result.ITestInvocationListener;
30 import com.android.tradefed.testtype.Abi;
31 import com.android.tradefed.testtype.IAbi;
32 import com.android.tradefed.testtype.IRemoteTest;
33 import com.android.tradefed.testtype.IRuntimeHintProvider;
34 import com.android.tradefed.util.IRunUtil;
35 import com.android.tradefed.util.RunInterruptedException;
37 import junit.framework.TestCase;
39 import org.easymock.EasyMock;
40 import org.easymock.IAnswer;
41 import org.easymock.IMocksControl;
44 import java.io.FileNotFoundException;
45 import java.io.StringReader;
46 import java.io.StringWriter;
47 import java.util.ArrayList;
48 import java.util.Collection;
49 import java.util.HashMap;
50 import java.util.HashSet;
51 import java.util.List;
54 import java.util.concurrent.TimeUnit;
57 * Unit tests for {@link DeqpTestRunner}.
59 public class DeqpTestRunnerTest extends TestCase {
60 private static final String NAME = "dEQP-GLES3";
61 private static final IAbi ABI = new Abi("armeabi-v7a", "32");
62 private static final String CASE_LIST_FILE_NAME = "/sdcard/dEQP-TestCaseList.txt";
63 private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa";
64 private static final String INSTRUMENTATION_NAME =
65 "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
66 private static final String QUERY_INSTRUMENTATION_NAME =
67 "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation";
68 private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk";
69 private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp";
70 private static final String ONLY_LANDSCAPE_FEATURES =
71 "feature:"+DeqpTestRunner.FEATURE_LANDSCAPE;
72 private static final String ALL_FEATURES =
73 ONLY_LANDSCAPE_FEATURES + "\nfeature:"+DeqpTestRunner.FEATURE_PORTRAIT;
74 private static List<Map<String,String>> DEFAULT_INSTANCE_ARGS;
77 DEFAULT_INSTANCE_ARGS = new ArrayList<>(1);
78 DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>());
79 DEFAULT_INSTANCE_ARGS.iterator().next().put("glconfig", "rgba8888d24s8");
80 DEFAULT_INSTANCE_ARGS.iterator().next().put("rotation", "unspecified");
81 DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window");
84 private static class StubRecovery implements DeqpTestRunner.IRecovery {
89 public void setSleepProvider(DeqpTestRunner.ISleepProvider sleepProvider) {
96 public void setDevice(ITestDevice device) {
103 public void onExecutionProgressed() {
110 public void recoverConnectionRefused() throws DeviceNotAvailableException {
117 public void recoverComLinkKilled() throws DeviceNotAvailableException {
121 public static class BuildHelperMock extends CompatibilityBuildHelper {
122 public BuildHelperMock(IFolderBuildInfo buildInfo) {
126 public File getTestsDir() throws FileNotFoundException {
127 return new File("logs");
136 protected void setUp() throws Exception {
140 private static DeqpTestRunner buildGlesTestRunner(int majorVersion,
142 Collection<TestIdentifier> tests) throws ConfigurationException, FileNotFoundException {
143 StringWriter testlist = new StringWriter();
144 for (TestIdentifier test : tests) {
145 testlist.write(test.getClassName() + "." + test.getTestName() + "\n");
147 return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString());
150 private static CompatibilityBuildHelper getMockBuildHelper() {
151 IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class);
152 EasyMock.replay(mockIFolderBuildInfo);
153 return new BuildHelperMock(mockIFolderBuildInfo);
156 private static DeqpTestRunner buildGlesTestRunner(int majorVersion,
158 String testlist) throws ConfigurationException, FileNotFoundException {
159 DeqpTestRunner runner = new DeqpTestRunner();
160 OptionSetter setter = new OptionSetter(runner);
162 String deqpPackage = "dEQP-GLES" + Integer.toString(majorVersion)
163 + (minorVersion > 0 ? Integer.toString(minorVersion) : "");
165 setter.setOptionValue("deqp-package", deqpPackage);
166 setter.setOptionValue("deqp-gl-config-name", "rgba8888d24s8");
167 setter.setOptionValue("deqp-caselist-file", "dummyfile.txt");
168 setter.setOptionValue("deqp-screen-rotation", "unspecified");
169 setter.setOptionValue("deqp-surface-type", "window");
171 runner.setCaselistReader(new StringReader(testlist));
173 runner.setBuildHelper(getMockBuildHelper());
178 private static String getTestId(DeqpTestRunner runner) {
179 return AbiUtils.createId(ABI.getName(), runner.getPackageName());
183 * Test version of OpenGL ES.
185 private void testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion) throws Exception {
186 final TestIdentifier testId = new TestIdentifier("dEQP-GLES"
187 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
188 + ".info", "version");
190 final String testPath = "dEQP-GLES"
191 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
194 final String testTrie = "{dEQP-GLES"
195 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
196 + "{info{version}}}";
198 final String resultCode = "Pass";
200 /* MultiLineReceiver expects "\r\n" line ending. */
201 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
202 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
203 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
204 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
205 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
206 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
207 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
208 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
209 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
210 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
211 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
212 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
213 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
214 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
215 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
216 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
217 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
218 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n"
219 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n"
220 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
221 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
222 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
223 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
224 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
225 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
226 + "INSTRUMENTATION_CODE: 0\r\n";
228 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
229 ITestInvocationListener mockListener
230 = EasyMock.createStrictMock(ITestInvocationListener.class);
231 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
232 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
235 DeqpTestRunner deqpTest = buildGlesTestRunner(requiredMajorVersion, requiredMinorVersion, tests);
237 int version = (majorVersion << 16) | minorVersion;
238 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
239 .andReturn(Integer.toString(version)).atLeastOnce();
241 if (majorVersion > requiredMajorVersion
242 || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion)) {
244 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
245 .andReturn("").once();
246 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
248 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
249 .andReturn(null).once();
251 expectRenderConfigQuery(mockDevice, requiredMajorVersion,
252 requiredMinorVersion);
254 String commandLine = String.format(
255 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
256 + "--deqp-screen-rotation=unspecified "
257 + "--deqp-surface-type=window "
258 + "--deqp-log-images=disable "
259 + "--deqp-watchdog=enable",
260 CASE_LIST_FILE_NAME);
262 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
265 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
266 .andReturn("").once();
269 mockListener.testRunStarted(getTestId(deqpTest), 1);
270 EasyMock.expectLastCall().once();
272 mockListener.testStarted(EasyMock.eq(testId));
273 EasyMock.expectLastCall().once();
275 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
276 EasyMock.expectLastCall().once();
278 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
279 EasyMock.expectLastCall().once();
281 EasyMock.replay(mockDevice, mockIDevice);
282 EasyMock.replay(mockListener);
284 deqpTest.setDevice(mockDevice);
285 deqpTest.run(mockListener);
287 EasyMock.verify(mockListener);
288 EasyMock.verify(mockDevice, mockIDevice);
291 private void expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion,
292 int minorVersion) throws Exception {
293 expectRenderConfigQuery(mockDevice,
294 String.format("--deqp-gl-config-name=rgba8888d24s8 "
295 + "--deqp-screen-rotation=unspecified "
296 + "--deqp-surface-type=window "
297 + "--deqp-gl-major-version=%d "
298 + "--deqp-gl-minor-version=%d", majorVersion, minorVersion));
301 private void expectRenderConfigQuery(ITestDevice mockDevice, String commandLine)
303 expectRenderConfigQueryAndReturn(mockDevice, commandLine, "Yes");
306 private void expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine,
307 String output) throws Exception {
308 final String queryOutput = "INSTRUMENTATION_RESULT: Supported=" + output + "\r\n"
309 + "INSTRUMENTATION_CODE: 0\r\n";
310 final String command = String.format(
311 "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine "
313 AbiUtils.createAbiFlag(ABI.getName()), commandLine,
314 QUERY_INSTRUMENTATION_NAME);
316 mockDevice.executeShellCommand(EasyMock.eq(command),
317 EasyMock.<IShellOutputReceiver>notNull());
319 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
321 public Object answer() {
322 IShellOutputReceiver receiver
323 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
325 receiver.addOutput(queryOutput.getBytes(), 0, queryOutput.length());
334 * Test that result code produces correctly pass or fail.
336 private void testResultCode(final String resultCode, boolean pass) throws Exception {
337 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.info", "version");
338 final String testPath = "dEQP-GLES3.info.version";
339 final String testTrie = "{dEQP-GLES3{info{version}}}";
341 /* MultiLineReceiver expects "\r\n" line ending. */
342 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
343 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
344 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
345 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
346 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
347 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
348 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
349 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
350 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
351 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
352 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
353 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
354 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
355 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
356 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
357 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
358 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
359 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n"
360 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n"
361 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
362 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
363 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
364 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
365 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
366 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
367 + "INSTRUMENTATION_CODE: 0\r\n";
369 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
370 ITestInvocationListener mockListener
371 = EasyMock.createStrictMock(ITestInvocationListener.class);
372 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
374 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
377 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
379 int version = 3 << 16;
380 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
381 .andReturn(Integer.toString(version)).atLeastOnce();
383 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
386 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
387 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
388 .andReturn(null).once();
390 expectRenderConfigQuery(mockDevice, 3, 0);
392 String commandLine = String.format(
393 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
394 + "--deqp-screen-rotation=unspecified "
395 + "--deqp-surface-type=window "
396 + "--deqp-log-images=disable "
397 + "--deqp-watchdog=enable",
398 CASE_LIST_FILE_NAME);
400 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
402 mockListener.testRunStarted(getTestId(deqpTest), 1);
403 EasyMock.expectLastCall().once();
405 mockListener.testStarted(EasyMock.eq(testId));
406 EasyMock.expectLastCall().once();
409 mockListener.testFailed(testId,
410 "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n"
411 + resultCode + ": Detail" + resultCode);
413 EasyMock.expectLastCall().once();
416 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
417 EasyMock.expectLastCall().once();
419 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
420 EasyMock.expectLastCall().once();
422 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
425 EasyMock.replay(mockDevice, mockIDevice);
426 EasyMock.replay(mockListener);
428 deqpTest.setDevice(mockDevice);
429 deqpTest.run(mockListener);
431 EasyMock.verify(mockListener);
432 EasyMock.verify(mockDevice, mockIDevice);
436 * Test running multiple test cases.
438 public void testRun_multipleTests() throws Exception {
439 /* MultiLineReceiver expects "\r\n" line ending. */
440 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
441 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
442 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
443 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
444 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
445 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
446 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
447 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
448 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
449 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
450 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
451 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
452 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
453 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
454 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
455 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n"
456 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
457 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
458 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
459 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
460 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
461 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
462 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
463 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
464 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n"
465 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
466 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
467 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
468 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
469 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
470 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
471 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
472 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
473 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n"
474 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
475 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
476 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
477 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
478 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
479 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
480 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
481 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
482 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n"
483 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
484 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
485 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
486 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
487 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
488 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
489 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
490 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
491 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n"
492 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
493 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
494 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
495 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
496 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
497 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
498 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
499 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
500 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n"
501 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
502 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
503 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
504 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
505 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
506 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
507 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
508 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
509 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
510 + "INSTRUMENTATION_CODE: 0\r\n";
512 final TestIdentifier[] testIds = {
513 new TestIdentifier("dEQP-GLES3.info", "vendor"),
514 new TestIdentifier("dEQP-GLES3.info", "renderer"),
515 new TestIdentifier("dEQP-GLES3.info", "version"),
516 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
517 new TestIdentifier("dEQP-GLES3.info", "extensions"),
518 new TestIdentifier("dEQP-GLES3.info", "render_target")
521 final String[] testPaths = {
522 "dEQP-GLES3.info.vendor",
523 "dEQP-GLES3.info.renderer",
524 "dEQP-GLES3.info.version",
525 "dEQP-GLES3.info.shading_language_version",
526 "dEQP-GLES3.info.extensions",
527 "dEQP-GLES3.info.render_target"
530 final String testTrie
531 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}";
533 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
534 ITestInvocationListener mockListener
535 = EasyMock.createStrictMock(ITestInvocationListener.class);
536 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
538 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
540 for (TestIdentifier id : testIds) {
544 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
546 int version = 3 << 16;
547 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
548 .andReturn(Integer.toString(version)).atLeastOnce();
550 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
552 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
553 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
554 .andReturn(null).once();
556 expectRenderConfigQuery(mockDevice, 3, 0);
558 String commandLine = String.format(
559 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
560 + "--deqp-screen-rotation=unspecified "
561 + "--deqp-surface-type=window "
562 + "--deqp-log-images=disable "
563 + "--deqp-watchdog=enable",
564 CASE_LIST_FILE_NAME);
566 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
568 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
569 EasyMock.expectLastCall().once();
571 for (int i = 0; i < testPaths.length; i++) {
572 mockListener.testStarted(EasyMock.eq(testIds[i]));
573 EasyMock.expectLastCall().once();
575 mockListener.testEnded(EasyMock.eq(testIds[i]),
576 EasyMock.<Map<String, String>>notNull());
578 EasyMock.expectLastCall().once();
581 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
582 EasyMock.expectLastCall().once();
584 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
587 EasyMock.replay(mockDevice, mockIDevice);
588 EasyMock.replay(mockListener);
590 deqpTest.setDevice(mockDevice);
591 deqpTest.run(mockListener);
593 EasyMock.verify(mockListener);
594 EasyMock.verify(mockDevice, mockIDevice);
597 static private String buildTestProcessOutput(List<TestIdentifier> tests) {
598 /* MultiLineReceiver expects "\r\n" line ending. */
599 final String outputHeader = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
600 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
601 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
602 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
603 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
604 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
605 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
606 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
607 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
608 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
609 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
610 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
611 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
612 + "INSTRUMENTATION_STATUS_CODE: 0\r\n";
614 final String outputEnd = "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
615 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
616 + "INSTRUMENTATION_CODE: 0\r\n";
618 StringWriter output = new StringWriter();
619 output.write(outputHeader);
620 for (TestIdentifier test : tests) {
621 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n");
622 output.write("INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=");
623 output.write(test.getClassName());
625 output.write(test.getTestName());
626 output.write("\r\n");
627 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
628 output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n");
629 output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n");
630 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n");
631 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
632 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n");
633 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
635 output.write(outputEnd);
636 return output.toString();
639 private void testFiltering(Set<String> includes,
640 Set<String> excludes,
641 List<TestIdentifier> fullTestList,
643 List<TestIdentifier> expectedTests) throws Exception {
645 boolean thereAreTests = !expectedTests.isEmpty();
646 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
647 ITestInvocationListener mockListener
648 = EasyMock.createStrictMock(ITestInvocationListener.class);
649 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
651 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, fullTestList);
652 if (includes != null) {
653 deqpTest.addAllIncludeFilters(includes);
655 if (excludes != null) {
656 deqpTest.addAllExcludeFilters(excludes);
659 int version = 3 << 16;
660 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
661 .andReturn(Integer.toString(version)).atLeastOnce();
665 // only expect to install/uninstall packages if there are any tests
666 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
668 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
669 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
670 .andReturn(null).once();
674 mockListener.testRunStarted(getTestId(deqpTest), expectedTests.size());
675 EasyMock.expectLastCall().once();
679 expectRenderConfigQuery(mockDevice, 3, 0);
681 String testOut = buildTestProcessOutput(expectedTests);
682 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut);
684 for (int i = 0; i < expectedTests.size(); i++) {
685 mockListener.testStarted(EasyMock.eq(expectedTests.get(i)));
686 EasyMock.expectLastCall().once();
688 mockListener.testEnded(EasyMock.eq(expectedTests.get(i)),
689 EasyMock.<Map<String, String>>notNull());
691 EasyMock.expectLastCall().once();
695 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
696 EasyMock.expectLastCall().once();
700 // package will only be installed if there are tests to run
701 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
705 EasyMock.replay(mockDevice, mockIDevice);
706 EasyMock.replay(mockListener);
708 deqpTest.setDevice(mockDevice);
709 deqpTest.run(mockListener);
711 EasyMock.verify(mockListener);
712 EasyMock.verify(mockDevice, mockIDevice);
715 public void testRun_trivialIncludeFilter() throws Exception {
716 final TestIdentifier[] testIds = {
717 new TestIdentifier("dEQP-GLES3.missing", "no"),
718 new TestIdentifier("dEQP-GLES3.missing", "nope"),
719 new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
720 new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
721 new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
722 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
725 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
726 for (TestIdentifier id : testIds) {
730 List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
731 activeTests.add(testIds[3]);
732 activeTests.add(testIds[4]);
733 activeTests.add(testIds[5]);
735 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
737 Set<String> includes = new HashSet();
738 includes.add("dEQP-GLES3.pick_me#*");
739 testFiltering(includes, null, allTests, expectedTrie, activeTests);
742 public void testRun_trivialExcludeFilter() throws Exception {
743 final TestIdentifier[] testIds = {
744 new TestIdentifier("dEQP-GLES3.missing", "no"),
745 new TestIdentifier("dEQP-GLES3.missing", "nope"),
746 new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
747 new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
748 new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
749 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
752 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
753 for (TestIdentifier id : testIds) {
757 List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
758 activeTests.add(testIds[3]);
759 activeTests.add(testIds[4]);
760 activeTests.add(testIds[5]);
762 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
764 Set<String> excludes = new HashSet();
765 excludes.add("dEQP-GLES3.missing#*");
766 testFiltering(null, excludes, allTests, expectedTrie, activeTests);
769 public void testRun_includeAndExcludeFilter() throws Exception {
770 final TestIdentifier[] testIds = {
771 new TestIdentifier("dEQP-GLES3.group1", "foo"),
772 new TestIdentifier("dEQP-GLES3.group1", "nope"),
773 new TestIdentifier("dEQP-GLES3.group1", "donotwant"),
774 new TestIdentifier("dEQP-GLES3.group2", "foo"),
775 new TestIdentifier("dEQP-GLES3.group2", "yes"),
776 new TestIdentifier("dEQP-GLES3.group2", "thoushallnotpass"),
779 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
780 for (TestIdentifier id : testIds) {
784 List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
785 activeTests.add(testIds[4]);
787 String expectedTrie = "{dEQP-GLES3{group2{yes}}}";
789 Set<String> includes = new HashSet();
790 includes.add("dEQP-GLES3.group2#*");
792 Set<String> excludes = new HashSet();
793 excludes.add("*foo");
794 excludes.add("*thoushallnotpass");
795 testFiltering(includes, excludes, allTests, expectedTrie, activeTests);
798 public void testRun_includeAll() throws Exception {
799 final TestIdentifier[] testIds = {
800 new TestIdentifier("dEQP-GLES3.group1", "mememe"),
801 new TestIdentifier("dEQP-GLES3.group1", "yeah"),
802 new TestIdentifier("dEQP-GLES3.group1", "takeitall"),
803 new TestIdentifier("dEQP-GLES3.group2", "jeba"),
804 new TestIdentifier("dEQP-GLES3.group2", "yes"),
805 new TestIdentifier("dEQP-GLES3.group2", "granted"),
808 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
809 for (TestIdentifier id : testIds) {
813 String expectedTrie = "{dEQP-GLES3{group1{mememe,yeah,takeitall},group2{jeba,yes,granted}}}";
815 Set<String> includes = new HashSet();
818 testFiltering(includes, null, allTests, expectedTrie, allTests);
821 public void testRun_excludeAll() throws Exception {
822 final TestIdentifier[] testIds = {
823 new TestIdentifier("dEQP-GLES3.group1", "no"),
824 new TestIdentifier("dEQP-GLES3.group1", "nope"),
825 new TestIdentifier("dEQP-GLES3.group1", "nottoday"),
826 new TestIdentifier("dEQP-GLES3.group2", "banned"),
827 new TestIdentifier("dEQP-GLES3.group2", "notrecognized"),
828 new TestIdentifier("dEQP-GLES3.group2", "-2"),
831 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
832 for (TestIdentifier id : testIds) {
836 String expectedTrie = "";
838 Set<String> excludes = new HashSet();
841 testFiltering(null, excludes, allTests, expectedTrie, new ArrayList<TestIdentifier>());
845 * Test running a unexecutable test.
847 public void testRun_unexecutableTests() throws Exception {
848 final String instrumentationAnswerNoExecs =
849 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
850 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
851 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
852 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
853 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
854 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
855 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
856 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
857 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
858 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
859 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
860 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
861 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
862 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
863 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
864 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
865 + "INSTRUMENTATION_CODE: 0\r\n";
867 final TestIdentifier[] testIds = {
868 new TestIdentifier("dEQP-GLES3.missing", "no"),
869 new TestIdentifier("dEQP-GLES3.missing", "nope"),
870 new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
873 final String[] testPaths = {
874 "dEQP-GLES3.missing.no",
875 "dEQP-GLES3.missing.nope",
876 "dEQP-GLES3.missing.donotwant",
879 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
880 ITestInvocationListener mockListener
881 = EasyMock.createStrictMock(ITestInvocationListener.class);
882 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
884 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
886 for (TestIdentifier id : testIds) {
890 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
892 int version = 3 << 16;
893 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
894 .andReturn(Integer.toString(version)).atLeastOnce();
896 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
898 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
899 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
900 .andReturn(null).once();
902 expectRenderConfigQuery(mockDevice, 3, 0);
904 String commandLine = String.format(
905 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
906 + "--deqp-screen-rotation=unspecified "
907 + "--deqp-surface-type=window "
908 + "--deqp-log-images=disable "
909 + "--deqp-watchdog=enable",
910 CASE_LIST_FILE_NAME);
913 runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
914 "{dEQP-GLES3{missing{no,nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs);
917 runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
918 "{dEQP-GLES3{missing{no}}}", commandLine, instrumentationAnswerNoExecs);
919 runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
920 "{dEQP-GLES3{missing{nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs);
921 runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
922 "{dEQP-GLES3{missing{nope}}}", commandLine, instrumentationAnswerNoExecs);
923 runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
924 "{dEQP-GLES3{missing{donotwant}}}", commandLine, instrumentationAnswerNoExecs);
926 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
927 EasyMock.expectLastCall().once();
929 for (int i = 0; i < testPaths.length; i++) {
930 mockListener.testStarted(EasyMock.eq(testIds[i]));
931 EasyMock.expectLastCall().once();
933 mockListener.testFailed(EasyMock.eq(testIds[i]),
934 EasyMock.eq("=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n"
935 + "Abort: Test cannot be executed"));
936 EasyMock.expectLastCall().once();
938 mockListener.testEnded(EasyMock.eq(testIds[i]),
939 EasyMock.<Map<String, String>>notNull());
940 EasyMock.expectLastCall().once();
943 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
944 EasyMock.expectLastCall().once();
946 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
949 EasyMock.replay(mockDevice, mockIDevice);
950 EasyMock.replay(mockListener);
952 deqpTest.setDevice(mockDevice);
953 deqpTest.run(mockListener);
955 EasyMock.verify(mockListener);
956 EasyMock.verify(mockDevice, mockIDevice);
960 * Test that test are left unexecuted if pm list query fails
962 public void testRun_queryPmListFailure()
964 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
966 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
967 ITestInvocationListener mockListener
968 = EasyMock.createStrictMock(ITestInvocationListener.class);
969 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
972 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
973 OptionSetter setter = new OptionSetter(deqpTest);
974 // Note: If the rotation is the default unspecified, features are not queried at all
975 setter.setOptionValue("deqp-screen-rotation", "90");
977 deqpTest.setDevice(mockDevice);
979 int version = 3 << 16;
980 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
981 .andReturn(Integer.toString(version)).atLeastOnce();
983 EasyMock.expect(mockDevice.executeShellCommand("pm list features"))
984 .andReturn("not a valid format");
986 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
987 andReturn("").once();
989 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
991 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
994 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
995 .andReturn("").once();
997 mockListener.testRunStarted(getTestId(deqpTest), 1);
998 EasyMock.expectLastCall().once();
1000 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1001 EasyMock.expectLastCall().once();
1003 EasyMock.replay(mockDevice);
1004 EasyMock.replay(mockListener);
1005 deqpTest.run(mockListener);
1006 EasyMock.verify(mockListener);
1007 EasyMock.verify(mockDevice);
1011 * Test that test are left unexecuted if renderablity query fails
1013 public void testRun_queryRenderabilityFailure()
1015 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
1017 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1018 ITestInvocationListener mockListener
1019 = EasyMock.createStrictMock(ITestInvocationListener.class);
1021 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1024 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
1026 deqpTest.setDevice(mockDevice);
1028 int version = 3 << 16;
1029 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1030 .andReturn(Integer.toString(version)).atLeastOnce();
1032 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1033 andReturn("").once();
1035 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1037 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1040 expectRenderConfigQueryAndReturn(mockDevice,
1041 "--deqp-gl-config-name=rgba8888d24s8 "
1042 + "--deqp-screen-rotation=unspecified "
1043 + "--deqp-surface-type=window "
1044 + "--deqp-gl-major-version=3 "
1045 + "--deqp-gl-minor-version=0", "Maybe?");
1047 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
1048 .andReturn("").once();
1050 mockListener.testRunStarted(getTestId(deqpTest), 1);
1051 EasyMock.expectLastCall().once();
1053 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1054 EasyMock.expectLastCall().once();
1056 EasyMock.replay(mockDevice);
1057 EasyMock.replay(mockListener);
1058 deqpTest.run(mockListener);
1059 EasyMock.verify(mockListener);
1060 EasyMock.verify(mockDevice);
1064 * Test that orientation is supplied to runner correctly
1066 private void testOrientation(final String rotation, final String featureString)
1068 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
1069 final String testPath = "dEQP-GLES3.orientation.test";
1070 final String testTrie = "{dEQP-GLES3{orientation{test}}}";
1071 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
1072 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1073 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
1074 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1075 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
1076 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1077 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
1078 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1079 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
1080 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1081 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
1082 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1083 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
1084 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1085 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
1086 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
1087 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1088 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
1089 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
1090 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
1091 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1092 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
1093 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1094 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
1095 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1096 + "INSTRUMENTATION_CODE: 0\r\n";
1098 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1099 ITestInvocationListener mockListener
1100 = EasyMock.createStrictMock(ITestInvocationListener.class);
1101 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
1103 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1106 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
1107 OptionSetter setter = new OptionSetter(deqpTest);
1108 setter.setOptionValue("deqp-screen-rotation", rotation);
1110 deqpTest.setDevice(mockDevice);
1112 int version = 3 << 16;
1113 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1114 .andReturn(Integer.toString(version)).atLeastOnce();
1116 if (!rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED)) {
1117 EasyMock.expect(mockDevice.executeShellCommand("pm list features"))
1118 .andReturn(featureString);
1121 final boolean isPortraitOrientation =
1122 rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_PORTRAIT) ||
1123 rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT);
1124 final boolean isLandscapeOrientation =
1125 rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_LANDSCAPE) ||
1126 rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE);
1127 final boolean executable =
1128 rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED) ||
1129 (isPortraitOrientation &&
1130 featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) ||
1131 (isLandscapeOrientation &&
1132 featureString.contains(DeqpTestRunner.FEATURE_LANDSCAPE));
1134 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1135 andReturn("").once();
1137 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1139 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1143 expectRenderConfigQuery(mockDevice, String.format(
1144 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=%s "
1145 + "--deqp-surface-type=window --deqp-gl-major-version=3 "
1146 + "--deqp-gl-minor-version=0", rotation));
1148 String commandLine = String.format(
1149 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
1150 + "--deqp-screen-rotation=%s "
1151 + "--deqp-surface-type=window "
1152 + "--deqp-log-images=disable "
1153 + "--deqp-watchdog=enable",
1154 CASE_LIST_FILE_NAME, rotation);
1156 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
1160 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
1161 .andReturn("").once();
1163 mockListener.testRunStarted(getTestId(deqpTest), 1);
1164 EasyMock.expectLastCall().once();
1166 mockListener.testStarted(EasyMock.eq(testId));
1167 EasyMock.expectLastCall().once();
1169 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
1170 EasyMock.expectLastCall().once();
1172 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1173 EasyMock.expectLastCall().once();
1175 EasyMock.replay(mockDevice, mockIDevice);
1176 EasyMock.replay(mockListener);
1177 deqpTest.run(mockListener);
1178 EasyMock.verify(mockListener);
1179 EasyMock.verify(mockDevice, mockIDevice);
1183 * Test OpeGL ES3 tests on device with OpenGL ES2.
1185 public void testRun_require30DeviceVersion20() throws Exception {
1186 testGlesVersion(3, 0, 2, 0);
1190 * Test OpeGL ES3.1 tests on device with OpenGL ES2.
1192 public void testRun_require31DeviceVersion20() throws Exception {
1193 testGlesVersion(3, 1, 2, 0);
1197 * Test OpeGL ES3 tests on device with OpenGL ES3.
1199 public void testRun_require30DeviceVersion30() throws Exception {
1200 testGlesVersion(3, 0, 3, 0);
1204 * Test OpeGL ES3.1 tests on device with OpenGL ES3.
1206 public void testRun_require31DeviceVersion30() throws Exception {
1207 testGlesVersion(3, 1, 3, 0);
1211 * Test OpeGL ES3 tests on device with OpenGL ES3.1.
1213 public void testRun_require30DeviceVersion31() throws Exception {
1214 testGlesVersion(3, 0, 3, 1);
1218 * Test OpeGL ES3.1 tests on device with OpenGL ES3.1.
1220 public void testRun_require31DeviceVersion31() throws Exception {
1221 testGlesVersion(3, 1, 3, 1);
1225 * Test dEQP Pass result code.
1227 public void testRun_resultPass() throws Exception {
1228 testResultCode("Pass", true);
1232 * Test dEQP Fail result code.
1234 public void testRun_resultFail() throws Exception {
1235 testResultCode("Fail", false);
1239 * Test dEQP NotSupported result code.
1241 public void testRun_resultNotSupported() throws Exception {
1242 testResultCode("NotSupported", true);
1246 * Test dEQP QualityWarning result code.
1248 public void testRun_resultQualityWarning() throws Exception {
1249 testResultCode("QualityWarning", true);
1253 * Test dEQP CompatibilityWarning result code.
1255 public void testRun_resultCompatibilityWarning() throws Exception {
1256 testResultCode("CompatibilityWarning", true);
1260 * Test dEQP ResourceError result code.
1262 public void testRun_resultResourceError() throws Exception {
1263 testResultCode("ResourceError", false);
1267 * Test dEQP InternalError result code.
1269 public void testRun_resultInternalError() throws Exception {
1270 testResultCode("InternalError", false);
1274 * Test dEQP Crash result code.
1276 public void testRun_resultCrash() throws Exception {
1277 testResultCode("Crash", false);
1281 * Test dEQP Timeout result code.
1283 public void testRun_resultTimeout() throws Exception {
1284 testResultCode("Timeout", false);
1287 * Test dEQP Orientation
1289 public void testRun_orientationLandscape() throws Exception {
1290 testOrientation("90", ALL_FEATURES);
1294 * Test dEQP Orientation
1296 public void testRun_orientationPortrait() throws Exception {
1297 testOrientation("0", ALL_FEATURES);
1301 * Test dEQP Orientation
1303 public void testRun_orientationReverseLandscape() throws Exception {
1304 testOrientation("270", ALL_FEATURES);
1308 * Test dEQP Orientation
1310 public void testRun_orientationReversePortrait() throws Exception {
1311 testOrientation("180", ALL_FEATURES);
1315 * Test dEQP Orientation
1317 public void testRun_orientationUnspecified() throws Exception {
1318 testOrientation("unspecified", ALL_FEATURES);
1322 * Test dEQP Orientation with limited features
1324 public void testRun_orientationUnspecifiedLimitedFeatures() throws Exception {
1325 testOrientation("unspecified", ONLY_LANDSCAPE_FEATURES);
1329 * Test dEQP Orientation with limited features
1331 public void testRun_orientationLandscapeLimitedFeatures() throws Exception {
1332 testOrientation("90", ONLY_LANDSCAPE_FEATURES);
1336 * Test dEQP Orientation with limited features
1338 public void testRun_orientationPortraitLimitedFeatures() throws Exception {
1339 testOrientation("0", ONLY_LANDSCAPE_FEATURES);
1343 * Test dEQP unsupported pixel format
1345 public void testRun_unsupportedPixelFormat() throws Exception {
1346 final String pixelFormat = "rgba5658d16m4";
1347 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.pixelformat", "test");
1348 final String testPath = "dEQP-GLES3.pixelformat.test";
1349 final String testTrie = "{dEQP-GLES3{pixelformat{test}}}";
1350 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
1351 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1352 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
1353 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1354 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
1355 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1356 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
1357 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1358 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
1359 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1360 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
1361 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1362 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
1363 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1364 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
1365 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
1366 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1367 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
1368 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
1369 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
1370 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1371 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
1372 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1373 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
1374 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1375 + "INSTRUMENTATION_CODE: 0\r\n";
1377 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1378 ITestInvocationListener mockListener
1379 = EasyMock.createStrictMock(ITestInvocationListener.class);
1381 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1384 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
1385 OptionSetter setter = new OptionSetter(deqpTest);
1386 setter.setOptionValue("deqp-gl-config-name", pixelFormat);
1388 deqpTest.setDevice(mockDevice);
1390 int version = 3 << 16;
1391 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1392 .andReturn(Integer.toString(version)).atLeastOnce();
1394 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1395 andReturn("").once();
1397 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1399 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1402 expectRenderConfigQueryAndReturn(mockDevice, String.format(
1403 "--deqp-gl-config-name=%s --deqp-screen-rotation=unspecified "
1404 + "--deqp-surface-type=window "
1405 + "--deqp-gl-major-version=3 "
1406 + "--deqp-gl-minor-version=0", pixelFormat), "No");
1408 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
1409 .andReturn("").once();
1411 mockListener.testRunStarted(getTestId(deqpTest), 1);
1412 EasyMock.expectLastCall().once();
1414 mockListener.testStarted(EasyMock.eq(testId));
1415 EasyMock.expectLastCall().once();
1417 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
1418 EasyMock.expectLastCall().once();
1420 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1421 EasyMock.expectLastCall().once();
1423 EasyMock.replay(mockDevice);
1424 EasyMock.replay(mockListener);
1425 deqpTest.run(mockListener);
1426 EasyMock.verify(mockListener);
1427 EasyMock.verify(mockDevice);
1430 public static interface RecoverableTestDevice extends ITestDevice {
1431 public void recoverDevice() throws DeviceNotAvailableException;
1434 private static enum RecoveryEvent {
1436 FAIL_CONNECTION_REFUSED,
1440 private void runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events)
1441 throws DeviceNotAvailableException {
1442 for (RecoveryEvent event : events) {
1445 recovery.onExecutionProgressed();
1447 case FAIL_CONNECTION_REFUSED:
1448 recovery.recoverConnectionRefused();
1450 case FAIL_LINK_KILLED:
1451 recovery.recoverComLinkKilled();
1457 private void setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider) {
1458 mockSleepProvider.sleep(EasyMock.gt(0));
1459 EasyMock.expectLastCall().once();
1462 private void setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice,
1463 DeqpTestRunner.ISleepProvider mockSleepProvider) throws DeviceNotAvailableException {
1464 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1465 andReturn("root 1234 com.drawelement.deqp").once();
1467 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
1468 andReturn("").once();
1470 // Recovery checks if kill failed
1471 mockSleepProvider.sleep(EasyMock.gt(0));
1472 EasyMock.expectLastCall().once();
1473 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1474 andReturn("").once();
1477 private void setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice)
1478 throws DeviceNotAvailableException {
1479 mockDevice.recoverDevice();
1480 EasyMock.expectLastCall().once();
1483 private void setRecoveryExpectationReboot(RecoverableTestDevice mockDevice)
1484 throws DeviceNotAvailableException {
1485 mockDevice.reboot();
1486 EasyMock.expectLastCall().once();
1489 private int setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice,
1490 DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)
1491 throws DeviceNotAvailableException {
1492 switch (numConsecutiveErrors) {
1495 setRecoveryExpectationRecovery(mockDevice);
1498 setRecoveryExpectationReboot(mockDevice);
1505 private int setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice,
1506 DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)
1507 throws DeviceNotAvailableException {
1508 switch (numConsecutiveErrors) {
1510 setRecoveryExpectationWait(mockSleepProvider);
1511 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider);
1514 setRecoveryExpectationRecovery(mockDevice);
1515 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider);
1518 setRecoveryExpectationReboot(mockDevice);
1525 private void setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice,
1526 DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events)
1527 throws DeviceNotAvailableException {
1528 int numConsecutiveErrors = 0;
1529 for (RecoveryEvent event : events) {
1532 numConsecutiveErrors = 0;
1534 case FAIL_CONNECTION_REFUSED:
1535 numConsecutiveErrors = setRecoveryExpectationOfAConnFailure(mockDevice,
1536 mockSleepProvider, numConsecutiveErrors);
1538 case FAIL_LINK_KILLED:
1539 numConsecutiveErrors = setRecoveryExpectationOfAComKilled(mockDevice,
1540 mockSleepProvider, numConsecutiveErrors);
1547 * Test dEQP runner recovery state machine.
1549 private void testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern)
1551 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery();
1552 IMocksControl orderedControl = EasyMock.createStrictControl();
1553 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class);
1554 DeqpTestRunner.ISleepProvider mockSleepProvider =
1555 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class);
1557 setRecoveryExpectationsOfAPattern(mockDevice, mockSleepProvider, pattern);
1559 orderedControl.replay();
1561 recovery.setDevice(mockDevice);
1562 recovery.setSleepProvider(mockSleepProvider);
1564 runRecoveryWithPattern(recovery, pattern);
1565 if (!expectSuccess) {
1566 fail("Expected DeviceNotAvailableException");
1568 } catch (DeviceNotAvailableException ex) {
1569 if (expectSuccess) {
1570 fail("Did not expect DeviceNotAvailableException");
1574 orderedControl.verify();
1579 public void testRecovery_NoEvents() throws Exception {
1580 testRecoveryWithPattern(true);
1583 public void testRecovery_AllOk() throws Exception {
1584 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, RecoveryEvent.PROGRESS);
1587 // conn fail patterns
1589 public void testRecovery_OneConnectionFailureBegin() throws Exception {
1590 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1591 RecoveryEvent.PROGRESS);
1594 public void testRecovery_TwoConnectionFailuresBegin() throws Exception {
1595 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1596 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS);
1599 public void testRecovery_ThreeConnectionFailuresBegin() throws Exception {
1600 testRecoveryWithPattern(false, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1601 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED);
1604 public void testRecovery_OneConnectionFailureMid() throws Exception {
1605 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1606 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS);
1609 public void testRecovery_TwoConnectionFailuresMid() throws Exception {
1610 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1611 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1612 RecoveryEvent.PROGRESS);
1615 public void testRecovery_ThreeConnectionFailuresMid() throws Exception {
1616 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS,
1617 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1618 RecoveryEvent.FAIL_CONNECTION_REFUSED);
1621 // link fail patterns
1623 public void testRecovery_OneLinkFailureBegin() throws Exception {
1624 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
1625 RecoveryEvent.PROGRESS);
1628 public void testRecovery_TwoLinkFailuresBegin() throws Exception {
1629 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
1630 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
1633 public void testRecovery_ThreeLinkFailuresBegin() throws Exception {
1634 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
1635 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1636 RecoveryEvent.PROGRESS);
1639 public void testRecovery_FourLinkFailuresBegin() throws Exception {
1640 testRecoveryWithPattern(false, RecoveryEvent.FAIL_LINK_KILLED,
1641 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1642 RecoveryEvent.FAIL_LINK_KILLED);
1645 public void testRecovery_OneLinkFailureMid() throws Exception {
1646 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1647 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
1650 public void testRecovery_TwoLinkFailuresMid() throws Exception {
1651 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1652 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1653 RecoveryEvent.PROGRESS);
1656 public void testRecovery_ThreeLinkFailuresMid() throws Exception {
1657 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1658 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1659 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
1662 public void testRecovery_FourLinkFailuresMid() throws Exception {
1663 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
1664 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1665 RecoveryEvent.FAIL_LINK_KILLED);
1670 public void testRecovery_MixedFailuresProgressBetween() throws Exception {
1671 testRecoveryWithPattern(true,
1672 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
1673 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1674 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
1675 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1676 RecoveryEvent.PROGRESS);
1679 public void testRecovery_MixedFailuresNoProgressBetween() throws Exception {
1680 testRecoveryWithPattern(true,
1681 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
1682 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_LINK_KILLED,
1683 RecoveryEvent.PROGRESS);
1687 * Test recovery if process cannot be killed
1689 public void testRecovery_unkillableProcess () throws Exception {
1690 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery();
1691 IMocksControl orderedControl = EasyMock.createStrictControl();
1692 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class);
1693 DeqpTestRunner.ISleepProvider mockSleepProvider =
1694 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class);
1696 // recovery attempts to kill the process after a timeout
1697 mockSleepProvider.sleep(EasyMock.gt(0));
1698 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1699 andReturn("root 1234 com.drawelement.deqp").once();
1700 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
1701 andReturn("").once();
1703 // Recovery checks if kill failed
1704 mockSleepProvider.sleep(EasyMock.gt(0));
1705 EasyMock.expectLastCall().once();
1706 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1707 andReturn("root 1234 com.drawelement.deqp").once();
1709 // Recovery resets the connection
1710 mockDevice.recoverDevice();
1711 EasyMock.expectLastCall().once();
1713 // and attempts to kill the process again
1714 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1715 andReturn("root 1234 com.drawelement.deqp").once();
1716 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
1717 andReturn("").once();
1719 // Recovery checks if kill failed
1720 mockSleepProvider.sleep(EasyMock.gt(0));
1721 EasyMock.expectLastCall().once();
1722 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1723 andReturn("root 1234 com.drawelement.deqp").once();
1725 // recovery reboots the device
1726 mockDevice.reboot();
1727 EasyMock.expectLastCall().once();
1729 orderedControl.replay();
1730 recovery.setDevice(mockDevice);
1731 recovery.setSleepProvider(mockSleepProvider);
1732 recovery.recoverComLinkKilled();
1733 orderedControl.verify();
1737 * Test external interruption before batch run.
1739 public void testInterrupt_killBeforeBatch() throws Exception {
1740 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.interrupt", "test");
1742 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1745 ITestInvocationListener mockListener
1746 = EasyMock.createStrictMock(ITestInvocationListener.class);
1747 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1748 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
1749 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class);
1751 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
1753 deqpTest.setDevice(mockDevice);
1754 deqpTest.setRunUtil(mockRunUtil);
1756 int version = 3 << 16;
1757 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1758 .andReturn(Integer.toString(version)).atLeastOnce();
1760 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1761 andReturn("").once();
1763 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1765 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1768 expectRenderConfigQuery(mockDevice,
1769 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified "
1770 + "--deqp-surface-type=window --deqp-gl-major-version=3 "
1771 + "--deqp-gl-minor-version=0");
1773 mockRunUtil.sleep(0);
1774 EasyMock.expectLastCall().andThrow(new RunInterruptedException());
1776 mockListener.testRunStarted(getTestId(deqpTest), 1);
1777 EasyMock.expectLastCall().once();
1779 EasyMock.replay(mockDevice, mockIDevice);
1780 EasyMock.replay(mockListener);
1781 EasyMock.replay(mockRunUtil);
1783 deqpTest.run(mockListener);
1784 fail("expected RunInterruptedException");
1785 } catch (RunInterruptedException ex) {
1788 EasyMock.verify(mockRunUtil);
1789 EasyMock.verify(mockListener);
1790 EasyMock.verify(mockDevice, mockIDevice);
1793 private void runShardedTest(TestIdentifier[] testIds,
1794 ArrayList<ArrayList<TestIdentifier>> testsForShard) throws Exception {
1795 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1796 for (TestIdentifier id : testIds) tests.add(id);
1798 DeqpTestRunner runner = buildGlesTestRunner(3, 0, tests);
1799 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split();
1801 for (int shardIndex = 0; shardIndex < shards.size(); shardIndex++) {
1802 DeqpTestRunner shard = (DeqpTestRunner)shards.get(shardIndex);
1803 shard.setBuildHelper(getMockBuildHelper());
1805 ArrayList<TestIdentifier> shardTests = testsForShard.get(shardIndex);
1807 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1808 ITestInvocationListener mockListener
1809 = EasyMock.createStrictMock(ITestInvocationListener.class);
1810 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
1811 int version = 3 << 16;
1812 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1813 .andReturn(Integer.toString(version)).atLeastOnce();
1815 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
1817 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1818 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
1819 .andReturn(null).once();
1821 mockListener.testRunStarted(getTestId(shard), shardTests.size());
1822 EasyMock.expectLastCall().once();
1824 expectRenderConfigQuery(mockDevice, 3, 0);
1826 String testOut = buildTestProcessOutput(shardTests);
1827 // NOTE: This assumes that there won't be multiple batches per shard!
1828 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut);
1830 for (int i = 0; i < shardTests.size(); i++) {
1831 mockListener.testStarted(EasyMock.eq(shardTests.get(i)));
1832 EasyMock.expectLastCall().once();
1834 mockListener.testEnded(EasyMock.eq(shardTests.get(i)),
1835 EasyMock.<Map<String, String>>notNull());
1837 EasyMock.expectLastCall().once();
1840 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1841 EasyMock.expectLastCall().once();
1843 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
1846 EasyMock.replay(mockDevice, mockIDevice);
1847 EasyMock.replay(mockListener);
1849 shard.setDevice(mockDevice);
1850 shard.run(mockListener);
1852 EasyMock.verify(mockListener);
1853 EasyMock.verify(mockDevice, mockIDevice);
1857 public void testSharding_smallTrivial() throws Exception {
1858 final TestIdentifier[] testIds = {
1859 new TestIdentifier("dEQP-GLES3.info", "vendor"),
1860 new TestIdentifier("dEQP-GLES3.info", "renderer"),
1861 new TestIdentifier("dEQP-GLES3.info", "version"),
1862 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
1863 new TestIdentifier("dEQP-GLES3.info", "extensions"),
1864 new TestIdentifier("dEQP-GLES3.info", "render_target")
1866 ArrayList<ArrayList<TestIdentifier>> shardedTests = new ArrayList<>();
1867 ArrayList<TestIdentifier> shardOne = new ArrayList<>();
1868 for (int i = 0; i < testIds.length; i++) {
1869 shardOne.add(testIds[i]);
1871 shardedTests.add(shardOne);
1872 runShardedTest(testIds, shardedTests);
1875 public void testSharding_twoShards() throws Exception {
1876 final int TEST_COUNT = 1237;
1877 final int SHARD_SIZE = 1000;
1879 ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT);
1880 for (int i = 0; i < TEST_COUNT; i++) {
1881 testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i)));
1884 ArrayList<ArrayList<TestIdentifier>> shardedTests = new ArrayList<>();
1885 ArrayList<TestIdentifier> shard = new ArrayList<>();
1886 for (int i = 0; i < testIds.size(); i++) {
1887 if (i == SHARD_SIZE) {
1888 shardedTests.add(shard);
1889 shard = new ArrayList<>();
1891 shard.add(testIds.get(i));
1893 shardedTests.add(shard);
1894 runShardedTest(testIds.toArray(new TestIdentifier[testIds.size()]), shardedTests);
1897 public void testSharding_empty() throws Exception {
1898 DeqpTestRunner runner = buildGlesTestRunner(3, 0, new ArrayList<TestIdentifier>());
1899 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split();
1900 // \todo [2015-11-23 kalle] What should the result be? The runner or nothing?
1904 * Test external interruption in testFailed().
1906 public void testInterrupt_killReportTestFailed() throws Exception {
1907 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.interrupt", "test");
1908 final String testPath = "dEQP-GLES3.interrupt.test";
1909 final String testTrie = "{dEQP-GLES3{interrupt{test}}}";
1910 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
1911 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1912 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
1913 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1914 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
1915 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1916 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
1917 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1918 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
1919 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1920 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
1921 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1922 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
1923 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1924 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
1925 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
1926 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1927 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Fail\r\n"
1928 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Fail\r\n"
1929 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
1930 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1931 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
1932 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1933 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
1934 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1935 + "INSTRUMENTATION_CODE: 0\r\n";
1937 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1940 ITestInvocationListener mockListener
1941 = EasyMock.createStrictMock(ITestInvocationListener.class);
1942 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1943 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
1944 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class);
1946 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
1948 deqpTest.setDevice(mockDevice);
1949 deqpTest.setRunUtil(mockRunUtil);
1951 int version = 3 << 16;
1952 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1953 .andReturn(Integer.toString(version)).atLeastOnce();
1955 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1956 andReturn("").once();
1958 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1960 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1963 expectRenderConfigQuery(mockDevice,
1964 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified "
1965 + "--deqp-surface-type=window --deqp-gl-major-version=3 "
1966 + "--deqp-gl-minor-version=0");
1968 mockRunUtil.sleep(0);
1969 EasyMock.expectLastCall().once();
1971 String commandLine = String.format(
1972 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
1973 + "--deqp-screen-rotation=unspecified "
1974 + "--deqp-surface-type=window "
1975 + "--deqp-log-images=disable "
1976 + "--deqp-watchdog=enable",
1977 CASE_LIST_FILE_NAME);
1979 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
1982 mockListener.testRunStarted(getTestId(deqpTest), 1);
1983 EasyMock.expectLastCall().once();
1985 mockListener.testStarted(EasyMock.eq(testId));
1986 EasyMock.expectLastCall().once();
1988 mockListener.testFailed(EasyMock.eq(testId), EasyMock.<String>notNull());
1989 EasyMock.expectLastCall().andThrow(new RunInterruptedException());
1991 EasyMock.replay(mockDevice, mockIDevice);
1992 EasyMock.replay(mockListener);
1993 EasyMock.replay(mockRunUtil);
1995 deqpTest.run(mockListener);
1996 fail("expected RunInterruptedException");
1997 } catch (RunInterruptedException ex) {
2000 EasyMock.verify(mockRunUtil);
2001 EasyMock.verify(mockListener);
2002 EasyMock.verify(mockDevice, mockIDevice);
2005 public void testRuntimeHint_optionSet() throws Exception {
2006 /* MultiLineReceiver expects "\r\n" line ending. */
2007 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
2008 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
2009 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
2010 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2011 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
2012 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
2013 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
2014 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2015 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
2016 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
2017 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
2018 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2019 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
2020 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2021 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2022 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n"
2023 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2024 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2025 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2026 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2027 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2028 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2029 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2030 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2031 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n"
2032 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2033 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2034 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2035 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2036 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2037 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2038 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2039 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2040 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n"
2041 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2042 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2043 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2044 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2045 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2046 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2047 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2048 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2049 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n"
2050 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2051 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2052 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2053 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2054 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2055 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2056 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2057 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2058 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n"
2059 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2060 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2061 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2062 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2063 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2064 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2065 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2066 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2067 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n"
2068 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2069 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2070 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2071 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2072 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2073 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2074 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2075 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
2076 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2077 + "INSTRUMENTATION_CODE: 0\r\n";
2079 final TestIdentifier[] testIds = {
2080 new TestIdentifier("dEQP-GLES3.info", "vendor"),
2081 new TestIdentifier("dEQP-GLES3.info", "renderer"),
2082 new TestIdentifier("dEQP-GLES3.info", "version"),
2083 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
2084 new TestIdentifier("dEQP-GLES3.info", "extensions"),
2085 new TestIdentifier("dEQP-GLES3.info", "render_target")
2088 final String[] testPaths = {
2089 "dEQP-GLES3.info.vendor",
2090 "dEQP-GLES3.info.renderer",
2091 "dEQP-GLES3.info.version",
2092 "dEQP-GLES3.info.shading_language_version",
2093 "dEQP-GLES3.info.extensions",
2094 "dEQP-GLES3.info.render_target"
2097 final String testTrie
2098 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}";
2100 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
2101 ITestInvocationListener mockListener
2102 = EasyMock.createStrictMock(ITestInvocationListener.class);
2103 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
2105 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
2107 for (TestIdentifier id : testIds) {
2111 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
2112 OptionSetter setter = new OptionSetter(deqpTest);
2113 final long runtimeMs = 123456;
2114 setter.setOptionValue("runtime-hint", String.valueOf(runtimeMs));
2115 assertEquals("Wrong expected runtime - option not passed cleanly", runtimeMs, deqpTest.getRuntimeHint());
2117 // Try running the tests as well. The unit tests do not set the hint be default,
2118 // so that case is covered.
2120 int version = 3 << 16;
2121 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
2122 .andReturn(Integer.toString(version)).atLeastOnce();
2124 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
2126 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
2127 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
2128 .andReturn(null).once();
2130 expectRenderConfigQuery(mockDevice, 3, 0);
2132 String commandLine = String.format(
2133 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
2134 + "--deqp-screen-rotation=unspecified "
2135 + "--deqp-surface-type=window "
2136 + "--deqp-log-images=disable "
2137 + "--deqp-watchdog=enable",
2138 CASE_LIST_FILE_NAME);
2140 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
2142 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
2143 EasyMock.expectLastCall().once();
2145 for (int i = 0; i < testPaths.length; i++) {
2146 mockListener.testStarted(EasyMock.eq(testIds[i]));
2147 EasyMock.expectLastCall().once();
2149 mockListener.testEnded(EasyMock.eq(testIds[i]),
2150 EasyMock.<Map<String, String>>notNull());
2152 EasyMock.expectLastCall().once();
2155 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
2156 EasyMock.expectLastCall().once();
2158 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
2161 EasyMock.replay(mockDevice, mockIDevice);
2162 EasyMock.replay(mockListener);
2164 deqpTest.setDevice(mockDevice);
2165 deqpTest.run(mockListener);
2167 EasyMock.verify(mockListener);
2168 EasyMock.verify(mockDevice, mockIDevice);
2171 public void testRuntimeHint_optionSetSharded() throws Exception {
2172 final int TEST_COUNT = 1237;
2173 final int SHARD_SIZE = 1000;
2175 ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT);
2176 for (int i = 0; i < TEST_COUNT; i++) {
2177 testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i)));
2180 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds);
2181 OptionSetter setter = new OptionSetter(deqpTest);
2182 final long fullRuntimeMs = testIds.size()*100;
2183 setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs));
2185 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)deqpTest.split();
2186 assertEquals("First shard's time not proportional to test count",
2187 (fullRuntimeMs*SHARD_SIZE)/TEST_COUNT,
2188 ((IRuntimeHintProvider)shards.get(0)).getRuntimeHint());
2189 assertEquals("Second shard's time not proportional to test count",
2190 (fullRuntimeMs*(TEST_COUNT-SHARD_SIZE))/TEST_COUNT,
2191 ((IRuntimeHintProvider)shards.get(1)).getRuntimeHint());
2194 public void testRuntimeHint_optionNotSet() throws Exception {
2195 final TestIdentifier[] testIds = {
2196 new TestIdentifier("dEQP-GLES3.info", "vendor"),
2197 new TestIdentifier("dEQP-GLES3.info", "renderer"),
2198 new TestIdentifier("dEQP-GLES3.info", "version"),
2199 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
2200 new TestIdentifier("dEQP-GLES3.info", "extensions"),
2201 new TestIdentifier("dEQP-GLES3.info", "render_target")
2203 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
2205 for (TestIdentifier id : testIds) {
2209 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
2211 long runtime = deqpTest.getRuntimeHint();
2212 assertTrue("Runtime for tests must be positive", runtime > 0);
2213 assertTrue("Runtime for tests must be reasonable", runtime < (1000 * 10)); // Must be done in 10s
2217 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice,
2218 final String output) throws Exception {
2219 String cmd = String.format(
2220 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
2221 + "--deqp-screen-rotation=unspecified "
2222 + "--deqp-surface-type=window "
2223 + "--deqp-log-images=disable "
2224 + "--deqp-watchdog=enable",
2225 CASE_LIST_FILE_NAME);
2226 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, null, cmd, output);
2229 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice,
2230 final String testTrie, final String cmd, final String output) throws Exception {
2231 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME)))
2232 .andReturn("").once();
2234 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME)))
2235 .andReturn("").once();
2237 if (testTrie == null) {
2238 mockDevice.pushString((String)EasyMock.anyObject(), EasyMock.eq(CASE_LIST_FILE_NAME));
2241 mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME);
2243 EasyMock.expectLastCall().andReturn(true).once();
2245 String command = String.format(
2246 "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\" "
2247 + "-e deqpLogData \"%s\" %s",
2248 AbiUtils.createAbiFlag(ABI.getName()), LOG_FILE_NAME, cmd, false,
2249 INSTRUMENTATION_NAME);
2251 EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice);
2252 mockIDevice.executeShellCommand(EasyMock.eq(command),
2253 EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(),
2254 EasyMock.isA(TimeUnit.class));
2256 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
2258 public Object answer() {
2259 IShellOutputReceiver receiver
2260 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
2262 receiver.addOutput(output.getBytes(), 0, output.length());