2 * Copyright (C) 2014 The Android Open Source Project
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
16 package com.drawelements.deqp.runner;
18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
19 import com.android.ddmlib.IDevice;
20 import com.android.ddmlib.IShellOutputReceiver;
21 import com.android.ddmlib.testrunner.TestIdentifier;
22 import com.android.tradefed.build.IFolderBuildInfo;
23 import com.android.tradefed.config.ConfigurationException;
24 import com.android.tradefed.config.OptionSetter;
25 import com.android.tradefed.device.DeviceNotAvailableException;
26 import com.android.tradefed.device.ITestDevice;
27 import com.android.tradefed.result.ITestInvocationListener;
28 import com.android.tradefed.testtype.Abi;
29 import com.android.tradefed.testtype.IAbi;
30 import com.android.tradefed.testtype.IRemoteTest;
31 import com.android.tradefed.testtype.IRuntimeHintProvider;
32 import com.android.tradefed.util.AbiUtils;
33 import com.android.tradefed.util.FileUtil;
34 import com.android.tradefed.util.IRunUtil;
35 import com.android.tradefed.util.RunInterruptedException;
37 import junit.framework.TestCase;
39 import org.easymock.EasyMock;
40 import org.easymock.IAnswer;
41 import org.easymock.IMocksControl;
44 import java.io.FileNotFoundException;
45 import java.io.FileWriter;
46 import java.io.IOException;
47 import java.io.PrintWriter;
48 import java.io.StringReader;
49 import java.io.StringWriter;
50 import java.util.ArrayList;
51 import java.util.Collection;
52 import java.util.HashMap;
53 import java.util.HashSet;
54 import java.util.List;
57 import java.util.concurrent.TimeUnit;
60 * Unit tests for {@link DeqpTestRunner}.
62 public class DeqpTestRunnerTest extends TestCase {
63 private static final String NAME = "dEQP-GLES3";
64 private static final IAbi ABI = new Abi("armeabi-v7a", "32");
65 private static final String CASE_LIST_FILE_NAME = "/sdcard/dEQP-TestCaseList.txt";
66 private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa";
67 private static final String INSTRUMENTATION_NAME =
68 "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
69 private static final String QUERY_INSTRUMENTATION_NAME =
70 "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation";
71 private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk";
72 private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp";
73 private static final String ONLY_LANDSCAPE_FEATURES =
74 "feature:"+DeqpTestRunner.FEATURE_LANDSCAPE;
75 private static final String ALL_FEATURES =
76 ONLY_LANDSCAPE_FEATURES + "\nfeature:"+DeqpTestRunner.FEATURE_PORTRAIT;
77 private static List<Map<String,String>> DEFAULT_INSTANCE_ARGS;
80 DEFAULT_INSTANCE_ARGS = new ArrayList<>(1);
81 DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>());
82 DEFAULT_INSTANCE_ARGS.iterator().next().put("glconfig", "rgba8888d24s8");
83 DEFAULT_INSTANCE_ARGS.iterator().next().put("rotation", "unspecified");
84 DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window");
87 private File mTestsDir = null;
89 public static class BuildHelperMock extends CompatibilityBuildHelper {
90 private File mTestsDir = null;
91 public BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir) {
96 public File getTestsDir() throws FileNotFoundException {
106 protected void setUp() throws Exception {
108 mTestsDir = FileUtil.createTempDir("deqp-test-cases");
115 protected void tearDown() throws Exception {
116 FileUtil.recursiveDelete(mTestsDir);
120 private static DeqpTestRunner buildGlesTestRunner(int majorVersion,
122 Collection<TestIdentifier> tests,
123 File testsDir) throws ConfigurationException, FileNotFoundException {
124 StringWriter testlist = new StringWriter();
125 for (TestIdentifier test : tests) {
126 testlist.write(test.getClassName() + "." + test.getTestName() + "\n");
128 return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString(), testsDir);
131 private static CompatibilityBuildHelper getMockBuildHelper(File testsDir) {
132 IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class);
133 EasyMock.replay(mockIFolderBuildInfo);
134 return new BuildHelperMock(mockIFolderBuildInfo, testsDir);
137 private static DeqpTestRunner buildGlesTestRunner(int majorVersion,
140 File testsDir) throws ConfigurationException, FileNotFoundException {
141 DeqpTestRunner runner = new DeqpTestRunner();
142 OptionSetter setter = new OptionSetter(runner);
144 String deqpPackage = "dEQP-GLES" + Integer.toString(majorVersion)
145 + (minorVersion > 0 ? Integer.toString(minorVersion) : "");
147 setter.setOptionValue("deqp-package", deqpPackage);
148 setter.setOptionValue("deqp-gl-config-name", "rgba8888d24s8");
149 setter.setOptionValue("deqp-caselist-file", "dummyfile.txt");
150 setter.setOptionValue("deqp-screen-rotation", "unspecified");
151 setter.setOptionValue("deqp-surface-type", "window");
153 runner.setCaselistReader(new StringReader(testlist));
155 runner.setBuildHelper(getMockBuildHelper(testsDir));
160 private static String getTestId(DeqpTestRunner runner) {
161 return AbiUtils.createId(ABI.getName(), runner.getPackageName());
165 * Test version of OpenGL ES.
167 private void testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion) throws Exception {
168 final TestIdentifier testId = new TestIdentifier("dEQP-GLES"
169 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
170 + ".info", "version");
172 final String testPath = "dEQP-GLES"
173 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
176 final String testTrie = "{dEQP-GLES"
177 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
178 + "{info{version}}}";
180 final String resultCode = "Pass";
182 /* MultiLineReceiver expects "\r\n" line ending. */
183 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
184 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
185 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
186 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
187 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
188 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
189 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
190 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
191 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
192 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
193 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
194 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
195 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
196 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
197 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
198 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
199 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
200 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n"
201 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n"
202 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
203 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
204 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
205 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
206 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
207 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
208 + "INSTRUMENTATION_CODE: 0\r\n";
210 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
211 ITestInvocationListener mockListener
212 = EasyMock.createStrictMock(ITestInvocationListener.class);
213 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
214 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
217 DeqpTestRunner deqpTest = buildGlesTestRunner(requiredMajorVersion, requiredMinorVersion, tests, mTestsDir);
219 int version = (majorVersion << 16) | minorVersion;
220 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
221 .andReturn(Integer.toString(version)).atLeastOnce();
223 if (majorVersion > requiredMajorVersion
224 || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion)) {
226 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
227 .andReturn("").once();
228 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
230 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
231 .andReturn(null).once();
233 expectRenderConfigQuery(mockDevice, requiredMajorVersion,
234 requiredMinorVersion);
236 String commandLine = String.format(
237 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
238 + "--deqp-screen-rotation=unspecified "
239 + "--deqp-surface-type=window "
240 + "--deqp-log-images=disable "
241 + "--deqp-watchdog=enable",
242 CASE_LIST_FILE_NAME);
244 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
247 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
248 .andReturn("").once();
251 mockListener.testRunStarted(getTestId(deqpTest), 1);
252 EasyMock.expectLastCall().once();
254 mockListener.testStarted(EasyMock.eq(testId));
255 EasyMock.expectLastCall().once();
257 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
258 EasyMock.expectLastCall().once();
260 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
261 EasyMock.expectLastCall().once();
263 EasyMock.replay(mockDevice, mockIDevice);
264 EasyMock.replay(mockListener);
266 deqpTest.setDevice(mockDevice);
267 deqpTest.run(mockListener);
269 EasyMock.verify(mockListener);
270 EasyMock.verify(mockDevice, mockIDevice);
273 private void expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion,
274 int minorVersion) throws Exception {
275 expectRenderConfigQuery(mockDevice,
276 String.format("--deqp-gl-config-name=rgba8888d24s8 "
277 + "--deqp-screen-rotation=unspecified "
278 + "--deqp-surface-type=window "
279 + "--deqp-gl-major-version=%d "
280 + "--deqp-gl-minor-version=%d", majorVersion, minorVersion));
283 private void expectRenderConfigQuery(ITestDevice mockDevice, String commandLine)
285 expectRenderConfigQueryAndReturn(mockDevice, commandLine, "Yes");
288 private void expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine,
289 String output) throws Exception {
290 final String queryOutput = "INSTRUMENTATION_RESULT: Supported=" + output + "\r\n"
291 + "INSTRUMENTATION_CODE: 0\r\n";
292 final String command = String.format(
293 "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine "
295 AbiUtils.createAbiFlag(ABI.getName()), commandLine,
296 QUERY_INSTRUMENTATION_NAME);
298 mockDevice.executeShellCommand(EasyMock.eq(command),
299 EasyMock.<IShellOutputReceiver>notNull());
301 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
303 public Object answer() {
304 IShellOutputReceiver receiver
305 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
307 receiver.addOutput(queryOutput.getBytes(), 0, queryOutput.length());
316 * Test that result code produces correctly pass or fail.
318 private void testResultCode(final String resultCode, boolean pass) throws Exception {
319 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.info", "version");
320 final String testPath = "dEQP-GLES3.info.version";
321 final String testTrie = "{dEQP-GLES3{info{version}}}";
323 /* MultiLineReceiver expects "\r\n" line ending. */
324 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
325 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
326 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
327 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
328 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
329 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
330 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
331 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
332 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
333 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
334 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
335 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
336 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
337 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
338 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
339 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
340 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
341 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n"
342 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n"
343 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
344 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
345 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
346 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
347 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
348 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
349 + "INSTRUMENTATION_CODE: 0\r\n";
351 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
352 ITestInvocationListener mockListener
353 = EasyMock.createStrictMock(ITestInvocationListener.class);
354 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
356 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
359 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
361 int version = 3 << 16;
362 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
363 .andReturn(Integer.toString(version)).atLeastOnce();
365 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
368 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
369 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
370 .andReturn(null).once();
372 expectRenderConfigQuery(mockDevice, 3, 0);
374 String commandLine = String.format(
375 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
376 + "--deqp-screen-rotation=unspecified "
377 + "--deqp-surface-type=window "
378 + "--deqp-log-images=disable "
379 + "--deqp-watchdog=enable",
380 CASE_LIST_FILE_NAME);
382 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
384 mockListener.testRunStarted(getTestId(deqpTest), 1);
385 EasyMock.expectLastCall().once();
387 mockListener.testStarted(EasyMock.eq(testId));
388 EasyMock.expectLastCall().once();
391 mockListener.testFailed(testId,
392 "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n"
393 + resultCode + ": Detail" + resultCode);
395 EasyMock.expectLastCall().once();
398 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
399 EasyMock.expectLastCall().once();
401 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
402 EasyMock.expectLastCall().once();
404 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
407 EasyMock.replay(mockDevice, mockIDevice);
408 EasyMock.replay(mockListener);
410 deqpTest.setDevice(mockDevice);
411 deqpTest.run(mockListener);
413 EasyMock.verify(mockListener);
414 EasyMock.verify(mockDevice, mockIDevice);
418 * Test running multiple test cases.
420 public void testRun_multipleTests() throws Exception {
421 /* MultiLineReceiver expects "\r\n" line ending. */
422 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
423 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
424 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
425 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
426 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
427 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
428 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
429 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
430 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
431 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
432 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
433 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
434 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
435 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
436 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
437 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n"
438 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
439 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
440 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
441 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
442 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
443 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
444 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
445 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
446 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n"
447 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
448 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
449 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
450 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
451 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
452 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
453 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
454 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
455 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n"
456 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
457 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
458 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
459 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
460 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
461 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
462 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
463 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
464 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n"
465 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
466 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
467 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
468 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
469 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
470 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
471 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
472 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
473 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n"
474 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
475 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
476 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
477 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
478 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
479 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
480 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
481 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
482 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n"
483 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
484 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
485 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
486 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
487 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
488 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
489 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
490 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
491 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
492 + "INSTRUMENTATION_CODE: 0\r\n";
494 final TestIdentifier[] testIds = {
495 new TestIdentifier("dEQP-GLES3.info", "vendor"),
496 new TestIdentifier("dEQP-GLES3.info", "renderer"),
497 new TestIdentifier("dEQP-GLES3.info", "version"),
498 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
499 new TestIdentifier("dEQP-GLES3.info", "extensions"),
500 new TestIdentifier("dEQP-GLES3.info", "render_target")
503 final String[] testPaths = {
504 "dEQP-GLES3.info.vendor",
505 "dEQP-GLES3.info.renderer",
506 "dEQP-GLES3.info.version",
507 "dEQP-GLES3.info.shading_language_version",
508 "dEQP-GLES3.info.extensions",
509 "dEQP-GLES3.info.render_target"
512 final String testTrie
513 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}";
515 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
516 ITestInvocationListener mockListener
517 = EasyMock.createStrictMock(ITestInvocationListener.class);
518 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
520 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
522 for (TestIdentifier id : testIds) {
526 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
528 int version = 3 << 16;
529 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
530 .andReturn(Integer.toString(version)).atLeastOnce();
532 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
534 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
535 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
536 .andReturn(null).once();
538 expectRenderConfigQuery(mockDevice, 3, 0);
540 String commandLine = String.format(
541 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
542 + "--deqp-screen-rotation=unspecified "
543 + "--deqp-surface-type=window "
544 + "--deqp-log-images=disable "
545 + "--deqp-watchdog=enable",
546 CASE_LIST_FILE_NAME);
548 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
550 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
551 EasyMock.expectLastCall().once();
553 for (int i = 0; i < testPaths.length; i++) {
554 mockListener.testStarted(EasyMock.eq(testIds[i]));
555 EasyMock.expectLastCall().once();
557 mockListener.testEnded(EasyMock.eq(testIds[i]),
558 EasyMock.<Map<String, String>>notNull());
560 EasyMock.expectLastCall().once();
563 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
564 EasyMock.expectLastCall().once();
566 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
569 EasyMock.replay(mockDevice, mockIDevice);
570 EasyMock.replay(mockListener);
572 deqpTest.setDevice(mockDevice);
573 deqpTest.run(mockListener);
575 EasyMock.verify(mockListener);
576 EasyMock.verify(mockDevice, mockIDevice);
579 static private String buildTestProcessOutput(List<TestIdentifier> tests) {
580 /* MultiLineReceiver expects "\r\n" line ending. */
581 final String outputHeader = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
582 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
583 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
584 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
585 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
586 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
587 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
588 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
589 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
590 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
591 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
592 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
593 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
594 + "INSTRUMENTATION_STATUS_CODE: 0\r\n";
596 final String outputEnd = "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
597 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
598 + "INSTRUMENTATION_CODE: 0\r\n";
600 StringWriter output = new StringWriter();
601 output.write(outputHeader);
602 for (TestIdentifier test : tests) {
603 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n");
604 output.write("INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=");
605 output.write(test.getClassName());
607 output.write(test.getTestName());
608 output.write("\r\n");
609 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
610 output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n");
611 output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n");
612 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n");
613 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
614 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n");
615 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
617 output.write(outputEnd);
618 return output.toString();
621 private void testFiltering(DeqpTestRunner deqpTest,
623 List<TestIdentifier> expectedTests) throws Exception {
624 int version = 3 << 16;
625 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
626 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
627 .andReturn(Integer.toString(version)).atLeastOnce();
629 boolean thereAreTests = !expectedTests.isEmpty();
632 // only expect to install/uninstall packages if there are any tests
633 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
635 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
636 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
637 .andReturn(null).once();
640 ITestInvocationListener mockListener
641 = EasyMock.createStrictMock(ITestInvocationListener.class);
642 mockListener.testRunStarted(getTestId(deqpTest), expectedTests.size());
643 EasyMock.expectLastCall().once();
645 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
648 expectRenderConfigQuery(mockDevice, 3, 0);
650 String testOut = buildTestProcessOutput(expectedTests);
651 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut);
653 for (int i = 0; i < expectedTests.size(); i++) {
654 mockListener.testStarted(EasyMock.eq(expectedTests.get(i)));
655 EasyMock.expectLastCall().once();
657 mockListener.testEnded(EasyMock.eq(expectedTests.get(i)),
658 EasyMock.<Map<String, String>>notNull());
660 EasyMock.expectLastCall().once();
664 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
665 EasyMock.expectLastCall().once();
669 // package will only be installed if there are tests to run
670 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
674 EasyMock.replay(mockDevice, mockIDevice);
675 EasyMock.replay(mockListener);
677 deqpTest.setDevice(mockDevice);
678 deqpTest.run(mockListener);
680 EasyMock.verify(mockListener);
681 EasyMock.verify(mockDevice, mockIDevice);
684 public void testRun_trivialIncludeFilter() throws Exception {
685 final TestIdentifier[] testIds = {
686 new TestIdentifier("dEQP-GLES3.missing", "no"),
687 new TestIdentifier("dEQP-GLES3.missing", "nope"),
688 new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
689 new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
690 new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
691 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
694 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
695 for (TestIdentifier id : testIds) {
699 List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
700 activeTests.add(testIds[3]);
701 activeTests.add(testIds[4]);
702 activeTests.add(testIds[5]);
704 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
706 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
707 deqpTest.addIncludeFilter("dEQP-GLES3.pick_me#*");
708 testFiltering(deqpTest, expectedTrie, activeTests);
711 public void testRun_trivialExcludeFilter() throws Exception {
712 final TestIdentifier[] testIds = {
713 new TestIdentifier("dEQP-GLES3.missing", "no"),
714 new TestIdentifier("dEQP-GLES3.missing", "nope"),
715 new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
716 new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
717 new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
718 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
721 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
722 for (TestIdentifier id : testIds) {
726 List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
727 activeTests.add(testIds[3]);
728 activeTests.add(testIds[4]);
729 activeTests.add(testIds[5]);
731 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
733 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
734 deqpTest.addExcludeFilter("dEQP-GLES3.missing#*");
735 testFiltering(deqpTest, expectedTrie, activeTests);
738 public void testRun_includeAndExcludeFilter() throws Exception {
739 final TestIdentifier[] testIds = {
740 new TestIdentifier("dEQP-GLES3.group1", "foo"),
741 new TestIdentifier("dEQP-GLES3.group1", "nope"),
742 new TestIdentifier("dEQP-GLES3.group1", "donotwant"),
743 new TestIdentifier("dEQP-GLES3.group2", "foo"),
744 new TestIdentifier("dEQP-GLES3.group2", "yes"),
745 new TestIdentifier("dEQP-GLES3.group2", "thoushallnotpass"),
748 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
749 for (TestIdentifier id : testIds) {
753 List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
754 activeTests.add(testIds[4]);
756 String expectedTrie = "{dEQP-GLES3{group2{yes}}}";
758 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
760 Set<String> includes = new HashSet<>();
761 includes.add("dEQP-GLES3.group2#*");
762 deqpTest.addAllIncludeFilters(includes);
764 Set<String> excludes = new HashSet<>();
765 excludes.add("*foo");
766 excludes.add("*thoushallnotpass");
767 deqpTest.addAllExcludeFilters(excludes);
768 testFiltering(deqpTest, expectedTrie, activeTests);
771 public void testRun_includeAll() throws Exception {
772 final TestIdentifier[] testIds = {
773 new TestIdentifier("dEQP-GLES3.group1", "mememe"),
774 new TestIdentifier("dEQP-GLES3.group1", "yeah"),
775 new TestIdentifier("dEQP-GLES3.group1", "takeitall"),
776 new TestIdentifier("dEQP-GLES3.group2", "jeba"),
777 new TestIdentifier("dEQP-GLES3.group2", "yes"),
778 new TestIdentifier("dEQP-GLES3.group2", "granted"),
781 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
782 for (TestIdentifier id : testIds) {
786 String expectedTrie = "{dEQP-GLES3{group1{mememe,yeah,takeitall},group2{jeba,yes,granted}}}";
788 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
789 deqpTest.addIncludeFilter("*");
790 testFiltering(deqpTest, expectedTrie, allTests);
793 public void testRun_excludeAll() throws Exception {
794 final TestIdentifier[] testIds = {
795 new TestIdentifier("dEQP-GLES3.group1", "no"),
796 new TestIdentifier("dEQP-GLES3.group1", "nope"),
797 new TestIdentifier("dEQP-GLES3.group1", "nottoday"),
798 new TestIdentifier("dEQP-GLES3.group2", "banned"),
799 new TestIdentifier("dEQP-GLES3.group2", "notrecognized"),
800 new TestIdentifier("dEQP-GLES3.group2", "-2"),
803 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
804 for (TestIdentifier id : testIds) {
808 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
809 deqpTest.addExcludeFilter("*");
810 ITestInvocationListener mockListener
811 = EasyMock.createStrictMock(ITestInvocationListener.class);
812 mockListener.testRunStarted(getTestId(deqpTest), 0);
813 EasyMock.expectLastCall().once();
814 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
815 EasyMock.expectLastCall().once();
817 EasyMock.replay(mockListener);
818 deqpTest.run(mockListener);
819 EasyMock.verify(mockListener);
823 * Test running a unexecutable test.
825 public void testRun_unexecutableTests() throws Exception {
826 final String instrumentationAnswerNoExecs =
827 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
828 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
829 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
830 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
831 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
832 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
833 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
834 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
835 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
836 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
837 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
838 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
839 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
840 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
841 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
842 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
843 + "INSTRUMENTATION_CODE: 0\r\n";
845 final TestIdentifier[] testIds = {
846 new TestIdentifier("dEQP-GLES3.missing", "no"),
847 new TestIdentifier("dEQP-GLES3.missing", "nope"),
848 new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
851 final String[] testPaths = {
852 "dEQP-GLES3.missing.no",
853 "dEQP-GLES3.missing.nope",
854 "dEQP-GLES3.missing.donotwant",
857 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
858 ITestInvocationListener mockListener
859 = EasyMock.createStrictMock(ITestInvocationListener.class);
860 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
862 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
864 for (TestIdentifier id : testIds) {
868 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
870 int version = 3 << 16;
871 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
872 .andReturn(Integer.toString(version)).atLeastOnce();
874 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
876 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
877 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
878 .andReturn(null).once();
880 expectRenderConfigQuery(mockDevice, 3, 0);
882 String commandLine = String.format(
883 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
884 + "--deqp-screen-rotation=unspecified "
885 + "--deqp-surface-type=window "
886 + "--deqp-log-images=disable "
887 + "--deqp-watchdog=enable",
888 CASE_LIST_FILE_NAME);
891 runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
892 "{dEQP-GLES3{missing{no,nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs);
895 runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
896 "{dEQP-GLES3{missing{no}}}", commandLine, instrumentationAnswerNoExecs);
897 runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
898 "{dEQP-GLES3{missing{nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs);
899 runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
900 "{dEQP-GLES3{missing{nope}}}", commandLine, instrumentationAnswerNoExecs);
901 runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
902 "{dEQP-GLES3{missing{donotwant}}}", commandLine, instrumentationAnswerNoExecs);
904 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
905 EasyMock.expectLastCall().once();
907 for (int i = 0; i < testPaths.length; i++) {
908 mockListener.testStarted(EasyMock.eq(testIds[i]));
909 EasyMock.expectLastCall().once();
911 mockListener.testFailed(EasyMock.eq(testIds[i]),
912 EasyMock.eq("=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n"
913 + "Abort: Test cannot be executed"));
914 EasyMock.expectLastCall().once();
916 mockListener.testEnded(EasyMock.eq(testIds[i]),
917 EasyMock.<Map<String, String>>notNull());
918 EasyMock.expectLastCall().once();
921 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
922 EasyMock.expectLastCall().once();
924 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
927 EasyMock.replay(mockDevice, mockIDevice);
928 EasyMock.replay(mockListener);
930 deqpTest.setDevice(mockDevice);
931 deqpTest.run(mockListener);
933 EasyMock.verify(mockListener);
934 EasyMock.verify(mockDevice, mockIDevice);
938 * Test that test are left unexecuted if pm list query fails
940 public void testRun_queryPmListFailure()
942 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
944 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
945 ITestInvocationListener mockListener
946 = EasyMock.createStrictMock(ITestInvocationListener.class);
947 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
950 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
951 OptionSetter setter = new OptionSetter(deqpTest);
952 // Note: If the rotation is the default unspecified, features are not queried at all
953 setter.setOptionValue("deqp-screen-rotation", "90");
955 deqpTest.setDevice(mockDevice);
957 int version = 3 << 16;
958 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
959 .andReturn(Integer.toString(version)).atLeastOnce();
961 EasyMock.expect(mockDevice.executeShellCommand("pm list features"))
962 .andReturn("not a valid format");
964 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
965 andReturn("").once();
967 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
969 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
972 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
973 .andReturn("").once();
975 mockListener.testRunStarted(getTestId(deqpTest), 1);
976 EasyMock.expectLastCall().once();
978 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
979 EasyMock.expectLastCall().once();
981 EasyMock.replay(mockDevice);
982 EasyMock.replay(mockListener);
983 deqpTest.run(mockListener);
984 EasyMock.verify(mockListener);
985 EasyMock.verify(mockDevice);
989 * Test that test are left unexecuted if renderablity query fails
991 public void testRun_queryRenderabilityFailure()
993 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
995 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
996 ITestInvocationListener mockListener
997 = EasyMock.createStrictMock(ITestInvocationListener.class);
999 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1002 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
1004 deqpTest.setDevice(mockDevice);
1006 int version = 3 << 16;
1007 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1008 .andReturn(Integer.toString(version)).atLeastOnce();
1010 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1011 andReturn("").once();
1013 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1015 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1018 expectRenderConfigQueryAndReturn(mockDevice,
1019 "--deqp-gl-config-name=rgba8888d24s8 "
1020 + "--deqp-screen-rotation=unspecified "
1021 + "--deqp-surface-type=window "
1022 + "--deqp-gl-major-version=3 "
1023 + "--deqp-gl-minor-version=0", "Maybe?");
1025 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
1026 .andReturn("").once();
1028 mockListener.testRunStarted(getTestId(deqpTest), 1);
1029 EasyMock.expectLastCall().once();
1031 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1032 EasyMock.expectLastCall().once();
1034 EasyMock.replay(mockDevice);
1035 EasyMock.replay(mockListener);
1036 deqpTest.run(mockListener);
1037 EasyMock.verify(mockListener);
1038 EasyMock.verify(mockDevice);
1042 * Test that orientation is supplied to runner correctly
1044 private void testOrientation(final String rotation, final String featureString)
1046 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
1047 final String testPath = "dEQP-GLES3.orientation.test";
1048 final String testTrie = "{dEQP-GLES3{orientation{test}}}";
1049 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
1050 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1051 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
1052 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1053 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
1054 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1055 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
1056 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1057 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
1058 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1059 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
1060 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1061 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
1062 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1063 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
1064 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
1065 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1066 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
1067 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
1068 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
1069 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1070 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
1071 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1072 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
1073 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1074 + "INSTRUMENTATION_CODE: 0\r\n";
1076 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1077 ITestInvocationListener mockListener
1078 = EasyMock.createStrictMock(ITestInvocationListener.class);
1079 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
1081 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1084 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
1085 OptionSetter setter = new OptionSetter(deqpTest);
1086 setter.setOptionValue("deqp-screen-rotation", rotation);
1088 deqpTest.setDevice(mockDevice);
1090 int version = 3 << 16;
1091 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1092 .andReturn(Integer.toString(version)).atLeastOnce();
1094 if (!rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED)) {
1095 EasyMock.expect(mockDevice.executeShellCommand("pm list features"))
1096 .andReturn(featureString);
1099 final boolean isPortraitOrientation =
1100 rotation.equals(BatchRunConfiguration.ROTATION_PORTRAIT) ||
1101 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT);
1102 final boolean isLandscapeOrientation =
1103 rotation.equals(BatchRunConfiguration.ROTATION_LANDSCAPE) ||
1104 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE);
1105 final boolean executable =
1106 rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED) ||
1107 (isPortraitOrientation &&
1108 featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) ||
1109 (isLandscapeOrientation &&
1110 featureString.contains(DeqpTestRunner.FEATURE_LANDSCAPE));
1112 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1113 andReturn("").once();
1115 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1117 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1121 expectRenderConfigQuery(mockDevice, String.format(
1122 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=%s "
1123 + "--deqp-surface-type=window --deqp-gl-major-version=3 "
1124 + "--deqp-gl-minor-version=0", rotation));
1126 String commandLine = String.format(
1127 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
1128 + "--deqp-screen-rotation=%s "
1129 + "--deqp-surface-type=window "
1130 + "--deqp-log-images=disable "
1131 + "--deqp-watchdog=enable",
1132 CASE_LIST_FILE_NAME, rotation);
1134 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
1138 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
1139 .andReturn("").once();
1141 mockListener.testRunStarted(getTestId(deqpTest), 1);
1142 EasyMock.expectLastCall().once();
1144 mockListener.testStarted(EasyMock.eq(testId));
1145 EasyMock.expectLastCall().once();
1147 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
1148 EasyMock.expectLastCall().once();
1150 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1151 EasyMock.expectLastCall().once();
1153 EasyMock.replay(mockDevice, mockIDevice);
1154 EasyMock.replay(mockListener);
1155 deqpTest.run(mockListener);
1156 EasyMock.verify(mockListener);
1157 EasyMock.verify(mockDevice, mockIDevice);
1161 * Test OpeGL ES3 tests on device with OpenGL ES2.
1163 public void testRun_require30DeviceVersion20() throws Exception {
1164 testGlesVersion(3, 0, 2, 0);
1168 * Test OpeGL ES3.1 tests on device with OpenGL ES2.
1170 public void testRun_require31DeviceVersion20() throws Exception {
1171 testGlesVersion(3, 1, 2, 0);
1175 * Test OpeGL ES3 tests on device with OpenGL ES3.
1177 public void testRun_require30DeviceVersion30() throws Exception {
1178 testGlesVersion(3, 0, 3, 0);
1182 * Test OpeGL ES3.1 tests on device with OpenGL ES3.
1184 public void testRun_require31DeviceVersion30() throws Exception {
1185 testGlesVersion(3, 1, 3, 0);
1189 * Test OpeGL ES3 tests on device with OpenGL ES3.1.
1191 public void testRun_require30DeviceVersion31() throws Exception {
1192 testGlesVersion(3, 0, 3, 1);
1196 * Test OpeGL ES3.1 tests on device with OpenGL ES3.1.
1198 public void testRun_require31DeviceVersion31() throws Exception {
1199 testGlesVersion(3, 1, 3, 1);
1203 * Test dEQP Pass result code.
1205 public void testRun_resultPass() throws Exception {
1206 testResultCode("Pass", true);
1210 * Test dEQP Fail result code.
1212 public void testRun_resultFail() throws Exception {
1213 testResultCode("Fail", false);
1217 * Test dEQP NotSupported result code.
1219 public void testRun_resultNotSupported() throws Exception {
1220 testResultCode("NotSupported", true);
1224 * Test dEQP QualityWarning result code.
1226 public void testRun_resultQualityWarning() throws Exception {
1227 testResultCode("QualityWarning", true);
1231 * Test dEQP CompatibilityWarning result code.
1233 public void testRun_resultCompatibilityWarning() throws Exception {
1234 testResultCode("CompatibilityWarning", true);
1238 * Test dEQP ResourceError result code.
1240 public void testRun_resultResourceError() throws Exception {
1241 testResultCode("ResourceError", false);
1245 * Test dEQP InternalError result code.
1247 public void testRun_resultInternalError() throws Exception {
1248 testResultCode("InternalError", false);
1252 * Test dEQP Crash result code.
1254 public void testRun_resultCrash() throws Exception {
1255 testResultCode("Crash", false);
1259 * Test dEQP Timeout result code.
1261 public void testRun_resultTimeout() throws Exception {
1262 testResultCode("Timeout", false);
1265 * Test dEQP Orientation
1267 public void testRun_orientationLandscape() throws Exception {
1268 testOrientation("90", ALL_FEATURES);
1272 * Test dEQP Orientation
1274 public void testRun_orientationPortrait() throws Exception {
1275 testOrientation("0", ALL_FEATURES);
1279 * Test dEQP Orientation
1281 public void testRun_orientationReverseLandscape() throws Exception {
1282 testOrientation("270", ALL_FEATURES);
1286 * Test dEQP Orientation
1288 public void testRun_orientationReversePortrait() throws Exception {
1289 testOrientation("180", ALL_FEATURES);
1293 * Test dEQP Orientation
1295 public void testRun_orientationUnspecified() throws Exception {
1296 testOrientation("unspecified", ALL_FEATURES);
1300 * Test dEQP Orientation with limited features
1302 public void testRun_orientationUnspecifiedLimitedFeatures() throws Exception {
1303 testOrientation("unspecified", ONLY_LANDSCAPE_FEATURES);
1307 * Test dEQP Orientation with limited features
1309 public void testRun_orientationLandscapeLimitedFeatures() throws Exception {
1310 testOrientation("90", ONLY_LANDSCAPE_FEATURES);
1314 * Test dEQP Orientation with limited features
1316 public void testRun_orientationPortraitLimitedFeatures() throws Exception {
1317 testOrientation("0", ONLY_LANDSCAPE_FEATURES);
1321 * Test dEQP unsupported pixel format
1323 public void testRun_unsupportedPixelFormat() throws Exception {
1324 final String pixelFormat = "rgba5658d16m4";
1325 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.pixelformat", "test");
1327 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1328 ITestInvocationListener mockListener
1329 = EasyMock.createStrictMock(ITestInvocationListener.class);
1331 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1334 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
1335 OptionSetter setter = new OptionSetter(deqpTest);
1336 setter.setOptionValue("deqp-gl-config-name", pixelFormat);
1338 deqpTest.setDevice(mockDevice);
1340 int version = 3 << 16;
1341 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1342 .andReturn(Integer.toString(version)).atLeastOnce();
1344 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1345 andReturn("").once();
1347 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1349 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1352 expectRenderConfigQueryAndReturn(mockDevice, String.format(
1353 "--deqp-gl-config-name=%s --deqp-screen-rotation=unspecified "
1354 + "--deqp-surface-type=window "
1355 + "--deqp-gl-major-version=3 "
1356 + "--deqp-gl-minor-version=0", pixelFormat), "No");
1358 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
1359 .andReturn("").once();
1361 mockListener.testRunStarted(getTestId(deqpTest), 1);
1362 EasyMock.expectLastCall().once();
1364 mockListener.testStarted(EasyMock.eq(testId));
1365 EasyMock.expectLastCall().once();
1367 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
1368 EasyMock.expectLastCall().once();
1370 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1371 EasyMock.expectLastCall().once();
1373 EasyMock.replay(mockDevice);
1374 EasyMock.replay(mockListener);
1375 deqpTest.run(mockListener);
1376 EasyMock.verify(mockListener);
1377 EasyMock.verify(mockDevice);
1380 public static interface RecoverableTestDevice extends ITestDevice {
1381 public void recoverDevice() throws DeviceNotAvailableException;
1384 private static enum RecoveryEvent {
1386 FAIL_CONNECTION_REFUSED,
1390 private void runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events)
1391 throws DeviceNotAvailableException {
1392 for (RecoveryEvent event : events) {
1395 recovery.onExecutionProgressed();
1397 case FAIL_CONNECTION_REFUSED:
1398 recovery.recoverConnectionRefused();
1400 case FAIL_LINK_KILLED:
1401 recovery.recoverComLinkKilled();
1407 private void setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider) {
1408 mockSleepProvider.sleep(EasyMock.gt(0));
1409 EasyMock.expectLastCall().once();
1412 private void setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice,
1413 DeqpTestRunner.ISleepProvider mockSleepProvider) throws DeviceNotAvailableException {
1414 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1415 andReturn("root 1234 com.drawelement.deqp").once();
1417 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
1418 andReturn("").once();
1420 // Recovery checks if kill failed
1421 mockSleepProvider.sleep(EasyMock.gt(0));
1422 EasyMock.expectLastCall().once();
1423 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1424 andReturn("").once();
1427 private void setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice)
1428 throws DeviceNotAvailableException {
1429 mockDevice.recoverDevice();
1430 EasyMock.expectLastCall().once();
1433 private void setRecoveryExpectationReboot(RecoverableTestDevice mockDevice)
1434 throws DeviceNotAvailableException {
1435 mockDevice.reboot();
1436 EasyMock.expectLastCall().once();
1439 private int setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice,
1440 DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)
1441 throws DeviceNotAvailableException {
1442 switch (numConsecutiveErrors) {
1445 setRecoveryExpectationRecovery(mockDevice);
1448 setRecoveryExpectationReboot(mockDevice);
1455 private int setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice,
1456 DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)
1457 throws DeviceNotAvailableException {
1458 switch (numConsecutiveErrors) {
1460 setRecoveryExpectationWait(mockSleepProvider);
1461 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider);
1464 setRecoveryExpectationRecovery(mockDevice);
1465 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider);
1468 setRecoveryExpectationReboot(mockDevice);
1475 private void setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice,
1476 DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events)
1477 throws DeviceNotAvailableException {
1478 int numConsecutiveErrors = 0;
1479 for (RecoveryEvent event : events) {
1482 numConsecutiveErrors = 0;
1484 case FAIL_CONNECTION_REFUSED:
1485 numConsecutiveErrors = setRecoveryExpectationOfAConnFailure(mockDevice,
1486 mockSleepProvider, numConsecutiveErrors);
1488 case FAIL_LINK_KILLED:
1489 numConsecutiveErrors = setRecoveryExpectationOfAComKilled(mockDevice,
1490 mockSleepProvider, numConsecutiveErrors);
1497 * Test dEQP runner recovery state machine.
1499 private void testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern)
1501 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery();
1502 IMocksControl orderedControl = EasyMock.createStrictControl();
1503 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class);
1504 EasyMock.expect(mockDevice.getSerialNumber()).andStubReturn("SERIAL");
1505 DeqpTestRunner.ISleepProvider mockSleepProvider =
1506 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class);
1508 setRecoveryExpectationsOfAPattern(mockDevice, mockSleepProvider, pattern);
1510 orderedControl.replay();
1512 recovery.setDevice(mockDevice);
1513 recovery.setSleepProvider(mockSleepProvider);
1515 runRecoveryWithPattern(recovery, pattern);
1516 if (!expectSuccess) {
1517 fail("Expected DeviceNotAvailableException");
1519 } catch (DeviceNotAvailableException ex) {
1520 if (expectSuccess) {
1521 fail("Did not expect DeviceNotAvailableException");
1525 orderedControl.verify();
1530 public void testRecovery_NoEvents() throws Exception {
1531 testRecoveryWithPattern(true);
1534 public void testRecovery_AllOk() throws Exception {
1535 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, RecoveryEvent.PROGRESS);
1538 // conn fail patterns
1540 public void testRecovery_OneConnectionFailureBegin() throws Exception {
1541 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1542 RecoveryEvent.PROGRESS);
1545 public void testRecovery_TwoConnectionFailuresBegin() throws Exception {
1546 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1547 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS);
1550 public void testRecovery_ThreeConnectionFailuresBegin() throws Exception {
1551 testRecoveryWithPattern(false, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1552 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED);
1555 public void testRecovery_OneConnectionFailureMid() throws Exception {
1556 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1557 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS);
1560 public void testRecovery_TwoConnectionFailuresMid() throws Exception {
1561 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1562 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1563 RecoveryEvent.PROGRESS);
1566 public void testRecovery_ThreeConnectionFailuresMid() throws Exception {
1567 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS,
1568 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1569 RecoveryEvent.FAIL_CONNECTION_REFUSED);
1572 // link fail patterns
1574 public void testRecovery_OneLinkFailureBegin() throws Exception {
1575 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
1576 RecoveryEvent.PROGRESS);
1579 public void testRecovery_TwoLinkFailuresBegin() throws Exception {
1580 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
1581 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
1584 public void testRecovery_ThreeLinkFailuresBegin() throws Exception {
1585 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
1586 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1587 RecoveryEvent.PROGRESS);
1590 public void testRecovery_FourLinkFailuresBegin() throws Exception {
1591 testRecoveryWithPattern(false, RecoveryEvent.FAIL_LINK_KILLED,
1592 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1593 RecoveryEvent.FAIL_LINK_KILLED);
1596 public void testRecovery_OneLinkFailureMid() throws Exception {
1597 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1598 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
1601 public void testRecovery_TwoLinkFailuresMid() throws Exception {
1602 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1603 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1604 RecoveryEvent.PROGRESS);
1607 public void testRecovery_ThreeLinkFailuresMid() throws Exception {
1608 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1609 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1610 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
1613 public void testRecovery_FourLinkFailuresMid() throws Exception {
1614 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
1615 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1616 RecoveryEvent.FAIL_LINK_KILLED);
1621 public void testRecovery_MixedFailuresProgressBetween() throws Exception {
1622 testRecoveryWithPattern(true,
1623 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
1624 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1625 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
1626 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1627 RecoveryEvent.PROGRESS);
1630 public void testRecovery_MixedFailuresNoProgressBetween() throws Exception {
1631 testRecoveryWithPattern(true,
1632 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
1633 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_LINK_KILLED,
1634 RecoveryEvent.PROGRESS);
1638 * Test recovery if process cannot be killed
1640 public void testRecovery_unkillableProcess () throws Exception {
1641 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery();
1642 IMocksControl orderedControl = EasyMock.createStrictControl();
1643 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class);
1644 DeqpTestRunner.ISleepProvider mockSleepProvider =
1645 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class);
1647 // recovery attempts to kill the process after a timeout
1648 mockSleepProvider.sleep(EasyMock.gt(0));
1649 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1650 andReturn("root 1234 com.drawelement.deqp").once();
1651 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
1652 andReturn("").once();
1654 // Recovery checks if kill failed
1655 mockSleepProvider.sleep(EasyMock.gt(0));
1656 EasyMock.expectLastCall().once();
1657 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1658 andReturn("root 1234 com.drawelement.deqp").once();
1660 // Recovery resets the connection
1661 mockDevice.recoverDevice();
1662 EasyMock.expectLastCall().once();
1664 // and attempts to kill the process again
1665 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1666 andReturn("root 1234 com.drawelement.deqp").once();
1667 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
1668 andReturn("").once();
1670 // Recovery checks if kill failed
1671 mockSleepProvider.sleep(EasyMock.gt(0));
1672 EasyMock.expectLastCall().once();
1673 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1674 andReturn("root 1234 com.drawelement.deqp").once();
1676 // recovery reboots the device
1677 mockDevice.reboot();
1678 EasyMock.expectLastCall().once();
1680 orderedControl.replay();
1681 recovery.setDevice(mockDevice);
1682 recovery.setSleepProvider(mockSleepProvider);
1683 recovery.recoverComLinkKilled();
1684 orderedControl.verify();
1688 * Test external interruption before batch run.
1690 public void testInterrupt_killBeforeBatch() throws Exception {
1691 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.interrupt", "test");
1693 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1696 ITestInvocationListener mockListener
1697 = EasyMock.createStrictMock(ITestInvocationListener.class);
1698 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1699 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
1700 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class);
1702 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
1704 deqpTest.setDevice(mockDevice);
1705 deqpTest.setRunUtil(mockRunUtil);
1707 int version = 3 << 16;
1708 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1709 .andReturn(Integer.toString(version)).atLeastOnce();
1711 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1712 andReturn("").once();
1714 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1716 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1719 expectRenderConfigQuery(mockDevice,
1720 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified "
1721 + "--deqp-surface-type=window --deqp-gl-major-version=3 "
1722 + "--deqp-gl-minor-version=0");
1724 mockRunUtil.sleep(0);
1725 EasyMock.expectLastCall().andThrow(new RunInterruptedException());
1727 mockListener.testRunStarted(getTestId(deqpTest), 1);
1728 EasyMock.expectLastCall().once();
1729 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.anyObject());
1730 EasyMock.expectLastCall().once();
1732 EasyMock.replay(mockDevice, mockIDevice);
1733 EasyMock.replay(mockListener);
1734 EasyMock.replay(mockRunUtil);
1736 deqpTest.run(mockListener);
1737 fail("expected RunInterruptedException");
1738 } catch (RunInterruptedException ex) {
1741 EasyMock.verify(mockRunUtil);
1742 EasyMock.verify(mockListener);
1743 EasyMock.verify(mockDevice, mockIDevice);
1746 private void runShardedTest(TestIdentifier[] testIds,
1747 ArrayList<ArrayList<TestIdentifier>> testsForShard) throws Exception {
1748 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1749 for (TestIdentifier id : testIds) tests.add(id);
1751 DeqpTestRunner runner = buildGlesTestRunner(3, 0, tests, mTestsDir);
1752 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split();
1754 for (int shardIndex = 0; shardIndex < shards.size(); shardIndex++) {
1755 DeqpTestRunner shard = (DeqpTestRunner)shards.get(shardIndex);
1756 shard.setBuildHelper(getMockBuildHelper(mTestsDir));
1758 ArrayList<TestIdentifier> shardTests = testsForShard.get(shardIndex);
1760 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1761 ITestInvocationListener mockListener
1762 = EasyMock.createStrictMock(ITestInvocationListener.class);
1763 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
1764 int version = 3 << 16;
1765 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1766 .andReturn(Integer.toString(version)).atLeastOnce();
1768 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
1770 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1771 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
1772 .andReturn(null).once();
1774 mockListener.testRunStarted(getTestId(shard), shardTests.size());
1775 EasyMock.expectLastCall().once();
1777 expectRenderConfigQuery(mockDevice, 3, 0);
1779 String testOut = buildTestProcessOutput(shardTests);
1780 // NOTE: This assumes that there won't be multiple batches per shard!
1781 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut);
1783 for (int i = 0; i < shardTests.size(); i++) {
1784 mockListener.testStarted(EasyMock.eq(shardTests.get(i)));
1785 EasyMock.expectLastCall().once();
1787 mockListener.testEnded(EasyMock.eq(shardTests.get(i)),
1788 EasyMock.<Map<String, String>>notNull());
1790 EasyMock.expectLastCall().once();
1793 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1794 EasyMock.expectLastCall().once();
1796 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
1799 EasyMock.replay(mockDevice, mockIDevice);
1800 EasyMock.replay(mockListener);
1802 shard.setDevice(mockDevice);
1803 shard.run(mockListener);
1805 EasyMock.verify(mockListener);
1806 EasyMock.verify(mockDevice, mockIDevice);
1810 public void testSharding_smallTrivial() throws Exception {
1811 final TestIdentifier[] testIds = {
1812 new TestIdentifier("dEQP-GLES3.info", "vendor"),
1813 new TestIdentifier("dEQP-GLES3.info", "renderer"),
1814 new TestIdentifier("dEQP-GLES3.info", "version"),
1815 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
1816 new TestIdentifier("dEQP-GLES3.info", "extensions"),
1817 new TestIdentifier("dEQP-GLES3.info", "render_target")
1819 ArrayList<ArrayList<TestIdentifier>> shardedTests = new ArrayList<>();
1820 ArrayList<TestIdentifier> shardOne = new ArrayList<>();
1821 for (int i = 0; i < testIds.length; i++) {
1822 shardOne.add(testIds[i]);
1824 shardedTests.add(shardOne);
1825 runShardedTest(testIds, shardedTests);
1828 public void testSharding_twoShards() throws Exception {
1829 final int TEST_COUNT = 1237;
1830 final int SHARD_SIZE = 1000;
1832 ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT);
1833 for (int i = 0; i < TEST_COUNT; i++) {
1834 testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i)));
1837 ArrayList<ArrayList<TestIdentifier>> shardedTests = new ArrayList<>();
1838 ArrayList<TestIdentifier> shard = new ArrayList<>();
1839 for (int i = 0; i < testIds.size(); i++) {
1840 if (i == SHARD_SIZE) {
1841 shardedTests.add(shard);
1842 shard = new ArrayList<>();
1844 shard.add(testIds.get(i));
1846 shardedTests.add(shard);
1847 runShardedTest(testIds.toArray(new TestIdentifier[testIds.size()]), shardedTests);
1850 public void testSharding_empty() throws Exception {
1851 DeqpTestRunner runner = buildGlesTestRunner(3, 0, new ArrayList<TestIdentifier>(), mTestsDir);
1852 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split();
1853 // Returns null when cannot be sharded.
1858 * Test external interruption in testFailed().
1860 public void testInterrupt_killReportTestFailed() throws Exception {
1861 final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.interrupt", "test");
1862 final String testPath = "dEQP-GLES3.interrupt.test";
1863 final String testTrie = "{dEQP-GLES3{interrupt{test}}}";
1864 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
1865 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1866 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
1867 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1868 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
1869 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1870 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
1871 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1872 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
1873 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1874 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
1875 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1876 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
1877 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1878 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
1879 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
1880 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1881 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Fail\r\n"
1882 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Fail\r\n"
1883 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
1884 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1885 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
1886 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1887 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
1888 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1889 + "INSTRUMENTATION_CODE: 0\r\n";
1891 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1894 ITestInvocationListener mockListener
1895 = EasyMock.createStrictMock(ITestInvocationListener.class);
1896 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1897 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
1898 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class);
1900 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
1902 deqpTest.setDevice(mockDevice);
1903 deqpTest.setRunUtil(mockRunUtil);
1905 int version = 3 << 16;
1906 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1907 .andReturn(Integer.toString(version)).atLeastOnce();
1909 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1910 andReturn("").once();
1912 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1914 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1917 expectRenderConfigQuery(mockDevice,
1918 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified "
1919 + "--deqp-surface-type=window --deqp-gl-major-version=3 "
1920 + "--deqp-gl-minor-version=0");
1922 mockRunUtil.sleep(0);
1923 EasyMock.expectLastCall().once();
1925 String commandLine = String.format(
1926 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
1927 + "--deqp-screen-rotation=unspecified "
1928 + "--deqp-surface-type=window "
1929 + "--deqp-log-images=disable "
1930 + "--deqp-watchdog=enable",
1931 CASE_LIST_FILE_NAME);
1933 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
1936 mockListener.testRunStarted(getTestId(deqpTest), 1);
1937 EasyMock.expectLastCall().once();
1939 mockListener.testStarted(EasyMock.eq(testId));
1940 EasyMock.expectLastCall().once();
1942 mockListener.testFailed(EasyMock.eq(testId), EasyMock.<String>notNull());
1943 EasyMock.expectLastCall().andThrow(new RunInterruptedException());
1945 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.anyObject());
1946 EasyMock.expectLastCall().once();
1947 EasyMock.replay(mockDevice, mockIDevice);
1948 EasyMock.replay(mockListener);
1949 EasyMock.replay(mockRunUtil);
1951 deqpTest.run(mockListener);
1952 fail("expected RunInterruptedException");
1953 } catch (RunInterruptedException ex) {
1956 EasyMock.verify(mockRunUtil);
1957 EasyMock.verify(mockListener);
1958 EasyMock.verify(mockDevice, mockIDevice);
1961 public void testRuntimeHint_optionSet() throws Exception {
1962 /* MultiLineReceiver expects "\r\n" line ending. */
1963 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
1964 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1965 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
1966 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1967 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
1968 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1969 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
1970 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1971 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
1972 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1973 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
1974 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1975 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
1976 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1977 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
1978 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n"
1979 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1980 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
1981 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
1982 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
1983 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1984 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
1985 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1986 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
1987 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n"
1988 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1989 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
1990 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
1991 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
1992 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1993 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
1994 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1995 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
1996 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n"
1997 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1998 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
1999 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2000 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2001 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2002 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2003 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2004 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2005 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n"
2006 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2007 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2008 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2009 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2010 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2011 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2012 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2013 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2014 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n"
2015 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2016 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2017 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2018 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2019 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2020 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2021 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2022 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2023 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n"
2024 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2025 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2026 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2027 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2028 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2029 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2030 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2031 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
2032 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2033 + "INSTRUMENTATION_CODE: 0\r\n";
2035 final TestIdentifier[] testIds = {
2036 new TestIdentifier("dEQP-GLES3.info", "vendor"),
2037 new TestIdentifier("dEQP-GLES3.info", "renderer"),
2038 new TestIdentifier("dEQP-GLES3.info", "version"),
2039 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
2040 new TestIdentifier("dEQP-GLES3.info", "extensions"),
2041 new TestIdentifier("dEQP-GLES3.info", "render_target")
2044 final String[] testPaths = {
2045 "dEQP-GLES3.info.vendor",
2046 "dEQP-GLES3.info.renderer",
2047 "dEQP-GLES3.info.version",
2048 "dEQP-GLES3.info.shading_language_version",
2049 "dEQP-GLES3.info.extensions",
2050 "dEQP-GLES3.info.render_target"
2053 final String testTrie
2054 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}";
2056 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
2057 ITestInvocationListener mockListener
2058 = EasyMock.createStrictMock(ITestInvocationListener.class);
2059 IDevice mockIDevice = EasyMock.createMock(IDevice.class);
2061 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
2063 for (TestIdentifier id : testIds) {
2067 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
2068 OptionSetter setter = new OptionSetter(deqpTest);
2069 final long runtimeMs = 123456;
2070 setter.setOptionValue("runtime-hint", String.valueOf(runtimeMs));
2071 assertEquals("Wrong expected runtime - option not passed cleanly", runtimeMs, deqpTest.getRuntimeHint());
2073 // Try running the tests as well. The unit tests do not set the hint be default,
2074 // so that case is covered.
2076 int version = 3 << 16;
2077 EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
2078 .andReturn(Integer.toString(version)).atLeastOnce();
2080 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
2082 EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
2083 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
2084 .andReturn(null).once();
2086 expectRenderConfigQuery(mockDevice, 3, 0);
2088 String commandLine = String.format(
2089 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
2090 + "--deqp-screen-rotation=unspecified "
2091 + "--deqp-surface-type=window "
2092 + "--deqp-log-images=disable "
2093 + "--deqp-watchdog=enable",
2094 CASE_LIST_FILE_NAME);
2096 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
2098 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
2099 EasyMock.expectLastCall().once();
2101 for (int i = 0; i < testPaths.length; i++) {
2102 mockListener.testStarted(EasyMock.eq(testIds[i]));
2103 EasyMock.expectLastCall().once();
2105 mockListener.testEnded(EasyMock.eq(testIds[i]),
2106 EasyMock.<Map<String, String>>notNull());
2108 EasyMock.expectLastCall().once();
2111 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
2112 EasyMock.expectLastCall().once();
2114 EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
2117 EasyMock.replay(mockDevice, mockIDevice);
2118 EasyMock.replay(mockListener);
2120 deqpTest.setDevice(mockDevice);
2121 deqpTest.run(mockListener);
2123 EasyMock.verify(mockListener);
2124 EasyMock.verify(mockDevice, mockIDevice);
2127 public void testRuntimeHint_optionSetSharded() throws Exception {
2128 final int TEST_COUNT = 1237;
2129 final int SHARD_SIZE = 1000;
2131 ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT);
2132 for (int i = 0; i < TEST_COUNT; i++) {
2133 testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i)));
2136 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir);
2137 OptionSetter setter = new OptionSetter(deqpTest);
2138 final long fullRuntimeMs = testIds.size()*100;
2139 setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs));
2141 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)deqpTest.split();
2142 assertEquals("First shard's time not proportional to test count",
2143 (fullRuntimeMs*SHARD_SIZE)/TEST_COUNT,
2144 ((IRuntimeHintProvider)shards.get(0)).getRuntimeHint());
2145 assertEquals("Second shard's time not proportional to test count",
2146 (fullRuntimeMs*(TEST_COUNT-SHARD_SIZE))/TEST_COUNT,
2147 ((IRuntimeHintProvider)shards.get(1)).getRuntimeHint());
2151 * Test that strict shardable is able to split deterministically the set of tests.
2153 public void testGetTestShard() throws Exception {
2154 final int TEST_COUNT = 2237;
2155 final int SHARD_COUNT = 4;
2157 ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT);
2158 for (int i = 0; i < TEST_COUNT; i++) {
2159 testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i)));
2162 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir);
2163 OptionSetter setter = new OptionSetter(deqpTest);
2164 final long fullRuntimeMs = testIds.size()*100;
2165 setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs));
2167 DeqpTestRunner shard1 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 0);
2168 assertEquals(559, shard1.getTestInstance().size());
2170 // Ensure numbers, and that order is stable
2171 for (TestIdentifier t : shard1.getTestInstance().keySet()) {
2172 assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
2173 String.format("%s#%s", t.getClassName(), t.getTestName()));
2176 DeqpTestRunner shard2 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 1);
2177 assertEquals(559, shard2.getTestInstance().size());
2178 for (TestIdentifier t : shard2.getTestInstance().keySet()) {
2179 assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
2180 String.format("%s#%s", t.getClassName(), t.getTestName()));
2183 DeqpTestRunner shard3 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 2);
2184 assertEquals(559, shard3.getTestInstance().size());
2185 for (TestIdentifier t : shard3.getTestInstance().keySet()) {
2186 assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
2187 String.format("%s#%s", t.getClassName(), t.getTestName()));
2190 DeqpTestRunner shard4 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 3);
2191 assertEquals(560, shard4.getTestInstance().size());
2192 for (TestIdentifier t : shard4.getTestInstance().keySet()) {
2193 assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
2194 String.format("%s#%s", t.getClassName(), t.getTestName()));
2197 assertEquals(TEST_COUNT, j);
2201 * Test that strict shardable is creating an empty shard of the runner when too many shards
2204 public void testGetTestShard_tooManyShardRequested() throws Exception {
2205 final int TEST_COUNT = 2;
2206 final int SHARD_COUNT = 3;
2208 ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT);
2209 for (int i = 0; i < TEST_COUNT; i++) {
2210 testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i)));
2212 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir);
2213 OptionSetter setter = new OptionSetter(deqpTest);
2214 final long fullRuntimeMs = testIds.size()*100;
2215 setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs));
2216 DeqpTestRunner shard1 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 0);
2217 assertEquals(1, shard1.getTestInstance().size());
2219 // Ensure numbers, and that order is stable
2220 for (TestIdentifier t : shard1.getTestInstance().keySet()) {
2221 assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
2222 String.format("%s#%s", t.getClassName(), t.getTestName()));
2225 DeqpTestRunner shard2 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 1);
2226 assertEquals(1, shard2.getTestInstance().size());
2227 for (TestIdentifier t : shard2.getTestInstance().keySet()) {
2228 assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
2229 String.format("%s#%s", t.getClassName(), t.getTestName()));
2232 DeqpTestRunner shard3 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 2);
2233 assertTrue(shard3.getTestInstance().isEmpty());
2234 assertEquals(TEST_COUNT, j);
2235 ITestInvocationListener mockListener
2236 = EasyMock.createStrictMock(ITestInvocationListener.class);
2237 mockListener.testRunStarted(EasyMock.anyObject(), EasyMock.eq(0));
2238 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.anyObject());
2239 EasyMock.replay(mockListener);
2240 shard3.run(mockListener);
2241 EasyMock.verify(mockListener);
2244 public void testRuntimeHint_optionNotSet() throws Exception {
2245 final TestIdentifier[] testIds = {
2246 new TestIdentifier("dEQP-GLES3.info", "vendor"),
2247 new TestIdentifier("dEQP-GLES3.info", "renderer"),
2248 new TestIdentifier("dEQP-GLES3.info", "version"),
2249 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
2250 new TestIdentifier("dEQP-GLES3.info", "extensions"),
2251 new TestIdentifier("dEQP-GLES3.info", "render_target")
2253 Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
2255 for (TestIdentifier id : testIds) {
2259 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
2261 long runtime = deqpTest.getRuntimeHint();
2262 assertTrue("Runtime for tests must be positive", runtime > 0);
2263 assertTrue("Runtime for tests must be reasonable", runtime < (1000 * 10)); // Must be done in 10s
2267 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice,
2268 final String output) throws Exception {
2269 String cmd = String.format(
2270 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
2271 + "--deqp-screen-rotation=unspecified "
2272 + "--deqp-surface-type=window "
2273 + "--deqp-log-images=disable "
2274 + "--deqp-watchdog=enable",
2275 CASE_LIST_FILE_NAME);
2276 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, null, cmd, output);
2279 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice,
2280 final String testTrie, final String cmd, final String output) throws Exception {
2281 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME)))
2282 .andReturn("").once();
2284 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME)))
2285 .andReturn("").once();
2287 if (testTrie == null) {
2288 mockDevice.pushString((String)EasyMock.anyObject(), EasyMock.eq(CASE_LIST_FILE_NAME));
2291 mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME);
2293 EasyMock.expectLastCall().andReturn(true).once();
2295 String command = String.format(
2296 "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\" "
2297 + "-e deqpLogData \"%s\" %s",
2298 AbiUtils.createAbiFlag(ABI.getName()), LOG_FILE_NAME, cmd, false,
2299 INSTRUMENTATION_NAME);
2301 EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice);
2302 mockIDevice.executeShellCommand(EasyMock.eq(command),
2303 EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(),
2304 EasyMock.isA(TimeUnit.class));
2306 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
2308 public Object answer() {
2309 IShellOutputReceiver receiver
2310 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
2312 receiver.addOutput(output.getBytes(), 0, output.length());
2320 static private void writeStringsToFile(File target, Set<String> strings) throws IOException {
2321 try (PrintWriter out = new PrintWriter(new FileWriter(target))) {
2322 out.print(String.join(System.lineSeparator(), strings));
2327 private void addFilterFileForOption(DeqpTestRunner test, Set<String> filters, String option)
2328 throws IOException, ConfigurationException {
2329 String filterFile = option + ".txt";
2330 writeStringsToFile(new File(mTestsDir, filterFile), filters);
2331 OptionSetter setter = new OptionSetter(test);
2332 setter.setOptionValue(option, filterFile);
2335 public void testIncludeFilterFile() throws Exception {
2336 final TestIdentifier[] testIds = {
2337 new TestIdentifier("dEQP-GLES3.missing", "no"),
2338 new TestIdentifier("dEQP-GLES3.missing", "nope"),
2339 new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
2340 new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
2341 new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
2342 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
2345 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
2346 for (TestIdentifier id : testIds) {
2350 List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
2351 activeTests.add(testIds[3]);
2352 activeTests.add(testIds[4]);
2353 activeTests.add(testIds[5]);
2355 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
2357 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
2358 Set<String> includes = new HashSet<>();
2359 includes.add("dEQP-GLES3.pick_me#*");
2360 addFilterFileForOption(deqpTest, includes, "include-filter-file");
2361 testFiltering(deqpTest, expectedTrie, activeTests);
2364 public void testMissingIncludeFilterFile() throws Exception {
2365 final TestIdentifier[] testIds = {
2366 new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
2367 new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
2368 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
2371 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
2372 for (TestIdentifier id : testIds) {
2376 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
2378 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
2379 OptionSetter setter = new OptionSetter(deqpTest);
2380 setter.setOptionValue("include-filter-file", "not-a-file.txt");
2382 testFiltering(deqpTest, expectedTrie, allTests);
2383 fail("Test execution should have aborted with exception.");
2384 } catch (RuntimeException e) {
2388 public void testExcludeFilterFile() throws Exception {
2389 final TestIdentifier[] testIds = {
2390 new TestIdentifier("dEQP-GLES3.missing", "no"),
2391 new TestIdentifier("dEQP-GLES3.missing", "nope"),
2392 new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
2393 new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
2394 new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
2395 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
2398 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
2399 for (TestIdentifier id : testIds) {
2403 List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
2404 activeTests.add(testIds[3]);
2405 activeTests.add(testIds[4]);
2406 activeTests.add(testIds[5]);
2408 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
2410 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
2411 Set<String> excludes = new HashSet<>();
2412 excludes.add("dEQP-GLES3.missing#*");
2413 addFilterFileForOption(deqpTest, excludes, "exclude-filter-file");
2414 testFiltering(deqpTest, expectedTrie, activeTests);
2417 public void testFilterComboWithFiles() throws Exception {
2418 final TestIdentifier[] testIds = {
2419 new TestIdentifier("dEQP-GLES3.group1", "footah"),
2420 new TestIdentifier("dEQP-GLES3.group1", "foo"),
2421 new TestIdentifier("dEQP-GLES3.group1", "nope"),
2422 new TestIdentifier("dEQP-GLES3.group1", "nonotwant"),
2423 new TestIdentifier("dEQP-GLES3.group2", "foo"),
2424 new TestIdentifier("dEQP-GLES3.group2", "yes"),
2425 new TestIdentifier("dEQP-GLES3.group2", "thoushallnotpass"),
2428 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
2429 for (TestIdentifier id : testIds) {
2433 List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
2434 activeTests.add(testIds[0]);
2435 activeTests.add(testIds[5]);
2437 String expectedTrie = "{dEQP-GLES3{group1{footah}group2{yes}}}";
2439 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
2441 Set<String> includes = new HashSet<>();
2442 includes.add("dEQP-GLES3.group2#*");
2443 deqpTest.addAllIncludeFilters(includes);
2445 Set<String> fileIncludes = new HashSet<>();
2446 fileIncludes.add("dEQP-GLES3.group1#no*");
2447 fileIncludes.add("dEQP-GLES3.group1#foo*");
2448 addFilterFileForOption(deqpTest, fileIncludes, "include-filter-file");
2450 Set<String> fileExcludes = new HashSet<>();
2451 fileExcludes.add("*foo");
2452 fileExcludes.add("*thoushallnotpass");
2453 addFilterFileForOption(deqpTest, fileExcludes, "exclude-filter-file");
2455 deqpTest.addExcludeFilter("dEQP-GLES3.group1#no*");
2457 testFiltering(deqpTest, expectedTrie, activeTests);
2460 public void testDotToHashConversionInFilters() throws Exception {
2461 final TestIdentifier[] testIds = {
2462 new TestIdentifier("dEQP-GLES3.missing", "no"),
2463 new TestIdentifier("dEQP-GLES3.pick_me", "donotwant"),
2464 new TestIdentifier("dEQP-GLES3.pick_me", "yes")
2467 List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
2468 for (TestIdentifier id : testIds) {
2472 List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
2473 activeTests.add(testIds[2]);
2475 String expectedTrie = "{dEQP-GLES3{pick_me{yes}}}";
2477 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
2478 deqpTest.addIncludeFilter("dEQP-GLES3.pick_me.yes");
2479 testFiltering(deqpTest, expectedTrie, activeTests);