android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java

   1 /*
   2  * Copyright (C) 2014 The Android Open Source Project
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package com.drawelements.deqp.runner;
  17
  18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
  19 import com.android.ddmlib.IDevice;
  20 import com.android.ddmlib.IShellOutputReceiver;
  21 import com.android.ddmlib.testrunner.TestIdentifier;
  22 import com.android.tradefed.build.IFolderBuildInfo;
  23 import com.android.tradefed.config.ConfigurationException;
  24 import com.android.tradefed.config.OptionSetter;
  25 import com.android.tradefed.device.DeviceNotAvailableException;
  26 import com.android.tradefed.device.ITestDevice;
  27 import com.android.tradefed.result.ITestInvocationListener;
  28 import com.android.tradefed.testtype.Abi;
  29 import com.android.tradefed.testtype.IAbi;
  30 import com.android.tradefed.testtype.IRemoteTest;
  31 import com.android.tradefed.testtype.IRuntimeHintProvider;
  32 import com.android.tradefed.util.AbiUtils;
  33 import com.android.tradefed.util.FileUtil;
  34 import com.android.tradefed.util.IRunUtil;
  35 import com.android.tradefed.util.RunInterruptedException;
  36
  37 import junit.framework.TestCase;
  38
  39 import org.easymock.EasyMock;
  40 import org.easymock.IAnswer;
  41 import org.easymock.IMocksControl;
  42
  43 import java.io.File;
  44 import java.io.FileNotFoundException;
  45 import java.io.FileWriter;
  46 import java.io.IOException;
  47 import java.io.PrintWriter;
  48 import java.io.StringReader;
  49 import java.io.StringWriter;
  50 import java.util.ArrayList;
  51 import java.util.Collection;
  52 import java.util.HashMap;
  53 import java.util.HashSet;
  54 import java.util.List;
  55 import java.util.Map;
  56 import java.util.Set;
  57 import java.util.concurrent.TimeUnit;
  58
  59 /**
  60  * Unit tests for {@link DeqpTestRunner}.
  61  */
  62 public class DeqpTestRunnerTest extends TestCase {
  63     private static final String NAME = "dEQP-GLES3";
  64     private static final IAbi ABI = new Abi("armeabi-v7a", "32");
  65     private static final String CASE_LIST_FILE_NAME = "/sdcard/dEQP-TestCaseList.txt";
  66     private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa";
  67     private static final String INSTRUMENTATION_NAME =
  68             "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
  69     private static final String QUERY_INSTRUMENTATION_NAME =
  70             "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation";
  71     private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk";
  72     private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp";
  73     private static final String ONLY_LANDSCAPE_FEATURES =
  74             "feature:"+DeqpTestRunner.FEATURE_LANDSCAPE;
  75     private static final String ALL_FEATURES =
  76             ONLY_LANDSCAPE_FEATURES + "\nfeature:"+DeqpTestRunner.FEATURE_PORTRAIT;
  77     private static List<Map<String,String>> DEFAULT_INSTANCE_ARGS;
  78
  79     static {
  80         DEFAULT_INSTANCE_ARGS = new ArrayList<>(1);
  81         DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>());
  82         DEFAULT_INSTANCE_ARGS.iterator().next().put("glconfig", "rgba8888d24s8");
  83         DEFAULT_INSTANCE_ARGS.iterator().next().put("rotation", "unspecified");
  84         DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window");
  85     }
  86
  87     private File mTestsDir = null;
  88
  89     public static class BuildHelperMock extends CompatibilityBuildHelper {
  90         private File mTestsDir = null;
  91         public BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir) {
  92             super(buildInfo);
  93             mTestsDir = testsDir;
  94         }
  95         @Override
  96         public File getTestsDir() throws FileNotFoundException {
  97             return mTestsDir;
  98         }
  99     }
 100
 101
 102     /**
 103      * {@inheritDoc}
 104      */
 105     @Override
 106     protected void setUp() throws Exception {
 107         super.setUp();
 108         mTestsDir = FileUtil.createTempDir("deqp-test-cases");
 109     }
 110
 111     /**
 112      * {@inheritDoc}
 113      */
 114     @Override
 115     protected void tearDown() throws Exception {
 116         FileUtil.recursiveDelete(mTestsDir);
 117         super.tearDown();
 118     }
 119
 120     private static DeqpTestRunner buildGlesTestRunner(int majorVersion,
 121                                                       int minorVersion,
 122                                                       Collection<TestIdentifier> tests,
 123                                                       File testsDir) throws ConfigurationException, FileNotFoundException {
 124         StringWriter testlist = new StringWriter();
 125         for (TestIdentifier test : tests) {
 126             testlist.write(test.getClassName() + "." + test.getTestName() + "\n");
 127         }
 128         return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString(), testsDir);
 129     }
 130
 131     private static CompatibilityBuildHelper getMockBuildHelper(File testsDir) {
 132         IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class);
 133         EasyMock.replay(mockIFolderBuildInfo);
 134         return new BuildHelperMock(mockIFolderBuildInfo, testsDir);
 135     }
 136
 137     private static DeqpTestRunner buildGlesTestRunner(int majorVersion,
 138                                                       int minorVersion,
 139                                                       String testlist,
 140                                                       File testsDir) throws ConfigurationException, FileNotFoundException {
 141         DeqpTestRunner runner = new DeqpTestRunner();
 142         OptionSetter setter = new OptionSetter(runner);
 143
 144         String deqpPackage = "dEQP-GLES" + Integer.toString(majorVersion)
 145                 + (minorVersion > 0 ? Integer.toString(minorVersion) : "");
 146
 147         setter.setOptionValue("deqp-package", deqpPackage);
 148         setter.setOptionValue("deqp-gl-config-name", "rgba8888d24s8");
 149         setter.setOptionValue("deqp-caselist-file", "dummyfile.txt");
 150         setter.setOptionValue("deqp-screen-rotation", "unspecified");
 151         setter.setOptionValue("deqp-surface-type", "window");
 152
 153         runner.setCaselistReader(new StringReader(testlist));
 154         runner.setAbi(ABI);
 155         runner.setBuildHelper(getMockBuildHelper(testsDir));
 156
 157         return runner;
 158     }
 159
 160     private static String getTestId(DeqpTestRunner runner) {
 161         return AbiUtils.createId(ABI.getName(), runner.getPackageName());
 162     }
 163
 164     /**
 165      * Test version of OpenGL ES.
 166      */
 167     private void testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion) throws Exception {
 168         final TestIdentifier testId = new TestIdentifier("dEQP-GLES"
 169                 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
 170                 + ".info", "version");
 171
 172         final String testPath = "dEQP-GLES"
 173                 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
 174                 +".info.version";
 175
 176         final String testTrie = "{dEQP-GLES"
 177                 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
 178                 + "{info{version}}}";
 179
 180         final String resultCode = "Pass";
 181
 182         /* MultiLineReceiver expects "\r\n" line ending. */
 183         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
 184                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 185                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
 186                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 187                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
 188                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 189                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
 190                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 191                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
 192                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 193                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
 194                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 195                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
 196                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 197                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
 198                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
 199                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 200                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n"
 201                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n"
 202                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
 203                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 204                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
 205                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 206                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
 207                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 208                 + "INSTRUMENTATION_CODE: 0\r\n";
 209
 210         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
 211         ITestInvocationListener mockListener
 212                 = EasyMock.createStrictMock(ITestInvocationListener.class);
 213         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
 214         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
 215         tests.add(testId);
 216
 217         DeqpTestRunner deqpTest = buildGlesTestRunner(requiredMajorVersion, requiredMinorVersion, tests, mTestsDir);
 218
 219         int version = (majorVersion << 16) | minorVersion;
 220         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
 221             .andReturn(Integer.toString(version)).atLeastOnce();
 222
 223         if (majorVersion > requiredMajorVersion
 224                 || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion)) {
 225
 226             EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
 227                     .andReturn("").once();
 228             EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
 229                     EasyMock.eq(true),
 230                     EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
 231                     .andReturn(null).once();
 232
 233             expectRenderConfigQuery(mockDevice, requiredMajorVersion,
 234                     requiredMinorVersion);
 235
 236             String commandLine = String.format(
 237                     "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
 238                     + "--deqp-screen-rotation=unspecified "
 239                     + "--deqp-surface-type=window "
 240                     + "--deqp-log-images=disable "
 241                     + "--deqp-watchdog=enable",
 242                     CASE_LIST_FILE_NAME);
 243
 244             runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
 245                     output);
 246
 247             EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
 248                     .andReturn("").once();
 249         }
 250
 251         mockListener.testRunStarted(getTestId(deqpTest), 1);
 252         EasyMock.expectLastCall().once();
 253
 254         mockListener.testStarted(EasyMock.eq(testId));
 255         EasyMock.expectLastCall().once();
 256
 257         mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
 258         EasyMock.expectLastCall().once();
 259
 260         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
 261         EasyMock.expectLastCall().once();
 262
 263         EasyMock.replay(mockDevice, mockIDevice);
 264         EasyMock.replay(mockListener);
 265
 266         deqpTest.setDevice(mockDevice);
 267         deqpTest.run(mockListener);
 268
 269         EasyMock.verify(mockListener);
 270         EasyMock.verify(mockDevice, mockIDevice);
 271     }
 272
 273     private void expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion,
 274             int minorVersion) throws Exception {
 275         expectRenderConfigQuery(mockDevice,
 276                 String.format("--deqp-gl-config-name=rgba8888d24s8 "
 277                 + "--deqp-screen-rotation=unspecified "
 278                 + "--deqp-surface-type=window "
 279                 + "--deqp-gl-major-version=%d "
 280                 + "--deqp-gl-minor-version=%d", majorVersion, minorVersion));
 281     }
 282
 283     private void expectRenderConfigQuery(ITestDevice mockDevice, String commandLine)
 284             throws Exception {
 285         expectRenderConfigQueryAndReturn(mockDevice, commandLine, "Yes");
 286     }
 287
 288     private void expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine,
 289             String output) throws Exception {
 290         final String queryOutput = "INSTRUMENTATION_RESULT: Supported=" + output + "\r\n"
 291                 + "INSTRUMENTATION_CODE: 0\r\n";
 292         final String command = String.format(
 293                 "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine "
 294                     + "\"%s\" %s",
 295                 AbiUtils.createAbiFlag(ABI.getName()), commandLine,
 296                 QUERY_INSTRUMENTATION_NAME);
 297
 298         mockDevice.executeShellCommand(EasyMock.eq(command),
 299                 EasyMock.<IShellOutputReceiver>notNull());
 300
 301         EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
 302             @Override
 303             public Object answer() {
 304                 IShellOutputReceiver receiver
 305                         = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
 306
 307                 receiver.addOutput(queryOutput.getBytes(), 0, queryOutput.length());
 308                 receiver.flush();
 309
 310                 return null;
 311             }
 312         });
 313     }
 314
 315     /**
 316      * Test that result code produces correctly pass or fail.
 317      */
 318     private void testResultCode(final String resultCode, boolean pass) throws Exception {
 319         final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.info", "version");
 320         final String testPath = "dEQP-GLES3.info.version";
 321         final String testTrie = "{dEQP-GLES3{info{version}}}";
 322
 323         /* MultiLineReceiver expects "\r\n" line ending. */
 324         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
 325                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 326                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
 327                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 328                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
 329                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 330                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
 331                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 332                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
 333                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 334                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
 335                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 336                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
 337                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 338                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
 339                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
 340                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 341                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n"
 342                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n"
 343                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
 344                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 345                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
 346                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 347                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
 348                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 349                 + "INSTRUMENTATION_CODE: 0\r\n";
 350
 351         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
 352         ITestInvocationListener mockListener
 353                 = EasyMock.createStrictMock(ITestInvocationListener.class);
 354         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
 355
 356         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
 357         tests.add(testId);
 358
 359         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
 360
 361         int version = 3 << 16;
 362         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
 363                 .andReturn(Integer.toString(version)).atLeastOnce();
 364
 365         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
 366                 .once();
 367
 368         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
 369                 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
 370                 .andReturn(null).once();
 371
 372         expectRenderConfigQuery(mockDevice, 3, 0);
 373
 374         String commandLine = String.format(
 375                 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
 376                 + "--deqp-screen-rotation=unspecified "
 377                 + "--deqp-surface-type=window "
 378                 + "--deqp-log-images=disable "
 379                 + "--deqp-watchdog=enable",
 380                 CASE_LIST_FILE_NAME);
 381
 382         runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
 383
 384         mockListener.testRunStarted(getTestId(deqpTest), 1);
 385         EasyMock.expectLastCall().once();
 386
 387         mockListener.testStarted(EasyMock.eq(testId));
 388         EasyMock.expectLastCall().once();
 389
 390         if (!pass) {
 391             mockListener.testFailed(testId,
 392                     "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n"
 393                     + resultCode + ": Detail" + resultCode);
 394
 395             EasyMock.expectLastCall().once();
 396         }
 397
 398         mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
 399         EasyMock.expectLastCall().once();
 400
 401         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
 402         EasyMock.expectLastCall().once();
 403
 404         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
 405                 .once();
 406
 407         EasyMock.replay(mockDevice, mockIDevice);
 408         EasyMock.replay(mockListener);
 409
 410         deqpTest.setDevice(mockDevice);
 411         deqpTest.run(mockListener);
 412
 413         EasyMock.verify(mockListener);
 414         EasyMock.verify(mockDevice, mockIDevice);
 415     }
 416
 417     /**
 418      * Test running multiple test cases.
 419      */
 420     public void testRun_multipleTests() throws Exception {
 421         /* MultiLineReceiver expects "\r\n" line ending. */
 422         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
 423                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 424                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
 425                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 426                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
 427                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 428                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
 429                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 430                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
 431                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 432                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
 433                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 434                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
 435                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 436                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
 437                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n"
 438                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 439                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
 440                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
 441                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
 442                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 443                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
 444                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 445                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
 446                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n"
 447                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 448                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
 449                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
 450                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
 451                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 452                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
 453                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 454                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
 455                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n"
 456                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 457                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
 458                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
 459                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
 460                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 461                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
 462                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 463                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
 464                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n"
 465                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 466                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
 467                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
 468                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
 469                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 470                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
 471                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 472                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
 473                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n"
 474                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 475                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
 476                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
 477                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
 478                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 479                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
 480                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 481                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
 482                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n"
 483                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 484                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
 485                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
 486                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
 487                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 488                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
 489                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 490                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
 491                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 492                 + "INSTRUMENTATION_CODE: 0\r\n";
 493
 494         final TestIdentifier[] testIds = {
 495                 new TestIdentifier("dEQP-GLES3.info", "vendor"),
 496                 new TestIdentifier("dEQP-GLES3.info", "renderer"),
 497                 new TestIdentifier("dEQP-GLES3.info", "version"),
 498                 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
 499                 new TestIdentifier("dEQP-GLES3.info", "extensions"),
 500                 new TestIdentifier("dEQP-GLES3.info", "render_target")
 501         };
 502
 503         final String[] testPaths = {
 504                 "dEQP-GLES3.info.vendor",
 505                 "dEQP-GLES3.info.renderer",
 506                 "dEQP-GLES3.info.version",
 507                 "dEQP-GLES3.info.shading_language_version",
 508                 "dEQP-GLES3.info.extensions",
 509                 "dEQP-GLES3.info.render_target"
 510         };
 511
 512         final String testTrie
 513                 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}";
 514
 515         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
 516         ITestInvocationListener mockListener
 517                 = EasyMock.createStrictMock(ITestInvocationListener.class);
 518         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
 519
 520         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
 521
 522         for (TestIdentifier id : testIds) {
 523             tests.add(id);
 524         }
 525
 526         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
 527
 528         int version = 3 << 16;
 529         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
 530                 .andReturn(Integer.toString(version)).atLeastOnce();
 531
 532         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
 533                 .once();
 534         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
 535                 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
 536                 .andReturn(null).once();
 537
 538         expectRenderConfigQuery(mockDevice, 3, 0);
 539
 540         String commandLine = String.format(
 541                 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
 542                 + "--deqp-screen-rotation=unspecified "
 543                 + "--deqp-surface-type=window "
 544                 + "--deqp-log-images=disable "
 545                 + "--deqp-watchdog=enable",
 546                 CASE_LIST_FILE_NAME);
 547
 548         runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
 549
 550         mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
 551         EasyMock.expectLastCall().once();
 552
 553         for (int i = 0; i < testPaths.length; i++) {
 554             mockListener.testStarted(EasyMock.eq(testIds[i]));
 555             EasyMock.expectLastCall().once();
 556
 557             mockListener.testEnded(EasyMock.eq(testIds[i]),
 558                     EasyMock.<Map<String, String>>notNull());
 559
 560             EasyMock.expectLastCall().once();
 561         }
 562
 563         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
 564         EasyMock.expectLastCall().once();
 565
 566         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
 567                 .once();
 568
 569         EasyMock.replay(mockDevice, mockIDevice);
 570         EasyMock.replay(mockListener);
 571
 572         deqpTest.setDevice(mockDevice);
 573         deqpTest.run(mockListener);
 574
 575         EasyMock.verify(mockListener);
 576         EasyMock.verify(mockDevice, mockIDevice);
 577     }
 578
 579     static private String buildTestProcessOutput(List<TestIdentifier> tests) {
 580         /* MultiLineReceiver expects "\r\n" line ending. */
 581         final String outputHeader = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
 582                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 583                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
 584                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 585                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
 586                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 587                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
 588                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 589                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
 590                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 591                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
 592                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 593                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
 594                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n";
 595
 596         final String outputEnd = "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
 597                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 598                 + "INSTRUMENTATION_CODE: 0\r\n";
 599
 600         StringWriter output = new StringWriter();
 601         output.write(outputHeader);
 602         for (TestIdentifier test : tests) {
 603             output.write("INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n");
 604             output.write("INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=");
 605             output.write(test.getClassName());
 606             output.write(".");
 607             output.write(test.getTestName());
 608             output.write("\r\n");
 609             output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
 610             output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n");
 611             output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n");
 612             output.write("INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n");
 613             output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
 614             output.write("INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n");
 615             output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
 616         }
 617         output.write(outputEnd);
 618         return output.toString();
 619     }
 620
 621     private void testFiltering(DeqpTestRunner deqpTest,
 622                                String expectedTrie,
 623                                List<TestIdentifier> expectedTests) throws Exception {
 624         int version = 3 << 16;
 625         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
 626         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
 627                 .andReturn(Integer.toString(version)).atLeastOnce();
 628
 629         boolean thereAreTests = !expectedTests.isEmpty();
 630         if (thereAreTests)
 631         {
 632             // only expect to install/uninstall packages if there are any tests
 633             EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
 634                 .once();
 635             EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
 636                 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
 637                 .andReturn(null).once();
 638         }
 639
 640         ITestInvocationListener mockListener
 641                 = EasyMock.createStrictMock(ITestInvocationListener.class);
 642         mockListener.testRunStarted(getTestId(deqpTest), expectedTests.size());
 643         EasyMock.expectLastCall().once();
 644
 645         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
 646         if (thereAreTests)
 647         {
 648             expectRenderConfigQuery(mockDevice, 3, 0);
 649
 650             String testOut = buildTestProcessOutput(expectedTests);
 651             runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut);
 652
 653             for (int i = 0; i < expectedTests.size(); i++) {
 654                 mockListener.testStarted(EasyMock.eq(expectedTests.get(i)));
 655                 EasyMock.expectLastCall().once();
 656
 657                 mockListener.testEnded(EasyMock.eq(expectedTests.get(i)),
 658                                        EasyMock.<Map<String, String>>notNull());
 659
 660                 EasyMock.expectLastCall().once();
 661             }
 662         }
 663
 664         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
 665         EasyMock.expectLastCall().once();
 666
 667         if (thereAreTests)
 668         {
 669             // package will only be installed if there are tests to run
 670             EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
 671                 .once();
 672         }
 673
 674         EasyMock.replay(mockDevice, mockIDevice);
 675         EasyMock.replay(mockListener);
 676
 677         deqpTest.setDevice(mockDevice);
 678         deqpTest.run(mockListener);
 679
 680         EasyMock.verify(mockListener);
 681         EasyMock.verify(mockDevice, mockIDevice);
 682     }
 683
 684     public void testRun_trivialIncludeFilter() throws Exception {
 685         final TestIdentifier[] testIds = {
 686                 new TestIdentifier("dEQP-GLES3.missing", "no"),
 687                 new TestIdentifier("dEQP-GLES3.missing", "nope"),
 688                 new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
 689                 new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
 690                 new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
 691                 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
 692         };
 693
 694         List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
 695         for (TestIdentifier id : testIds) {
 696             allTests.add(id);
 697         }
 698
 699         List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
 700         activeTests.add(testIds[3]);
 701         activeTests.add(testIds[4]);
 702         activeTests.add(testIds[5]);
 703
 704         String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
 705
 706         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
 707         deqpTest.addIncludeFilter("dEQP-GLES3.pick_me#*");
 708         testFiltering(deqpTest, expectedTrie, activeTests);
 709     }
 710
 711     public void testRun_trivialExcludeFilter() throws Exception {
 712         final TestIdentifier[] testIds = {
 713                 new TestIdentifier("dEQP-GLES3.missing", "no"),
 714                 new TestIdentifier("dEQP-GLES3.missing", "nope"),
 715                 new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
 716                 new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
 717                 new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
 718                 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
 719         };
 720
 721         List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
 722         for (TestIdentifier id : testIds) {
 723             allTests.add(id);
 724         }
 725
 726         List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
 727         activeTests.add(testIds[3]);
 728         activeTests.add(testIds[4]);
 729         activeTests.add(testIds[5]);
 730
 731         String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
 732
 733         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
 734         deqpTest.addExcludeFilter("dEQP-GLES3.missing#*");
 735         testFiltering(deqpTest, expectedTrie, activeTests);
 736     }
 737
 738     public void testRun_includeAndExcludeFilter() throws Exception {
 739         final TestIdentifier[] testIds = {
 740                 new TestIdentifier("dEQP-GLES3.group1", "foo"),
 741                 new TestIdentifier("dEQP-GLES3.group1", "nope"),
 742                 new TestIdentifier("dEQP-GLES3.group1", "donotwant"),
 743                 new TestIdentifier("dEQP-GLES3.group2", "foo"),
 744                 new TestIdentifier("dEQP-GLES3.group2", "yes"),
 745                 new TestIdentifier("dEQP-GLES3.group2", "thoushallnotpass"),
 746         };
 747
 748         List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
 749         for (TestIdentifier id : testIds) {
 750             allTests.add(id);
 751         }
 752
 753         List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
 754         activeTests.add(testIds[4]);
 755
 756         String expectedTrie = "{dEQP-GLES3{group2{yes}}}";
 757
 758         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
 759
 760         Set<String> includes = new HashSet<>();
 761         includes.add("dEQP-GLES3.group2#*");
 762         deqpTest.addAllIncludeFilters(includes);
 763
 764         Set<String> excludes = new HashSet<>();
 765         excludes.add("*foo");
 766         excludes.add("*thoushallnotpass");
 767         deqpTest.addAllExcludeFilters(excludes);
 768         testFiltering(deqpTest, expectedTrie, activeTests);
 769     }
 770
 771     public void testRun_includeAll() throws Exception {
 772         final TestIdentifier[] testIds = {
 773                 new TestIdentifier("dEQP-GLES3.group1", "mememe"),
 774                 new TestIdentifier("dEQP-GLES3.group1", "yeah"),
 775                 new TestIdentifier("dEQP-GLES3.group1", "takeitall"),
 776                 new TestIdentifier("dEQP-GLES3.group2", "jeba"),
 777                 new TestIdentifier("dEQP-GLES3.group2", "yes"),
 778                 new TestIdentifier("dEQP-GLES3.group2", "granted"),
 779         };
 780
 781         List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
 782         for (TestIdentifier id : testIds) {
 783             allTests.add(id);
 784         }
 785
 786         String expectedTrie = "{dEQP-GLES3{group1{mememe,yeah,takeitall},group2{jeba,yes,granted}}}";
 787
 788         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
 789         deqpTest.addIncludeFilter("*");
 790         testFiltering(deqpTest, expectedTrie, allTests);
 791     }
 792
 793     public void testRun_excludeAll() throws Exception {
 794         final TestIdentifier[] testIds = {
 795                 new TestIdentifier("dEQP-GLES3.group1", "no"),
 796                 new TestIdentifier("dEQP-GLES3.group1", "nope"),
 797                 new TestIdentifier("dEQP-GLES3.group1", "nottoday"),
 798                 new TestIdentifier("dEQP-GLES3.group2", "banned"),
 799                 new TestIdentifier("dEQP-GLES3.group2", "notrecognized"),
 800                 new TestIdentifier("dEQP-GLES3.group2", "-2"),
 801         };
 802
 803         List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
 804         for (TestIdentifier id : testIds) {
 805             allTests.add(id);
 806         }
 807
 808         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
 809         deqpTest.addExcludeFilter("*");
 810         ITestInvocationListener mockListener
 811                 = EasyMock.createStrictMock(ITestInvocationListener.class);
 812         mockListener.testRunStarted(getTestId(deqpTest), 0);
 813         EasyMock.expectLastCall().once();
 814         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
 815         EasyMock.expectLastCall().once();
 816
 817         EasyMock.replay(mockListener);
 818         deqpTest.run(mockListener);
 819         EasyMock.verify(mockListener);
 820     }
 821
 822     /**
 823      * Test running a unexecutable test.
 824      */
 825     public void testRun_unexecutableTests() throws Exception {
 826         final String instrumentationAnswerNoExecs =
 827                 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
 828                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 829                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
 830                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 831                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
 832                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 833                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
 834                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 835                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
 836                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 837                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
 838                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 839                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
 840                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 841                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
 842                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 843                 + "INSTRUMENTATION_CODE: 0\r\n";
 844
 845         final TestIdentifier[] testIds = {
 846                 new TestIdentifier("dEQP-GLES3.missing", "no"),
 847                 new TestIdentifier("dEQP-GLES3.missing", "nope"),
 848                 new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
 849         };
 850
 851         final String[] testPaths = {
 852                 "dEQP-GLES3.missing.no",
 853                 "dEQP-GLES3.missing.nope",
 854                 "dEQP-GLES3.missing.donotwant",
 855         };
 856
 857         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
 858         ITestInvocationListener mockListener
 859                 = EasyMock.createStrictMock(ITestInvocationListener.class);
 860         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
 861
 862         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
 863
 864         for (TestIdentifier id : testIds) {
 865             tests.add(id);
 866         }
 867
 868         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
 869
 870         int version = 3 << 16;
 871         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
 872                 .andReturn(Integer.toString(version)).atLeastOnce();
 873
 874         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
 875                 .once();
 876         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
 877                 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
 878                 .andReturn(null).once();
 879
 880         expectRenderConfigQuery(mockDevice, 3, 0);
 881
 882         String commandLine = String.format(
 883                 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
 884                 + "--deqp-screen-rotation=unspecified "
 885                 + "--deqp-surface-type=window "
 886                 + "--deqp-log-images=disable "
 887                 + "--deqp-watchdog=enable",
 888                 CASE_LIST_FILE_NAME);
 889
 890         // first try
 891         runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
 892                 "{dEQP-GLES3{missing{no,nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs);
 893
 894         // splitting begins
 895         runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
 896                 "{dEQP-GLES3{missing{no}}}", commandLine, instrumentationAnswerNoExecs);
 897         runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
 898                 "{dEQP-GLES3{missing{nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs);
 899         runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
 900                 "{dEQP-GLES3{missing{nope}}}", commandLine, instrumentationAnswerNoExecs);
 901         runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
 902                 "{dEQP-GLES3{missing{donotwant}}}", commandLine, instrumentationAnswerNoExecs);
 903
 904         mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
 905         EasyMock.expectLastCall().once();
 906
 907         for (int i = 0; i < testPaths.length; i++) {
 908             mockListener.testStarted(EasyMock.eq(testIds[i]));
 909             EasyMock.expectLastCall().once();
 910
 911             mockListener.testFailed(EasyMock.eq(testIds[i]),
 912                     EasyMock.eq("=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n"
 913                     + "Abort: Test cannot be executed"));
 914             EasyMock.expectLastCall().once();
 915
 916             mockListener.testEnded(EasyMock.eq(testIds[i]),
 917                     EasyMock.<Map<String, String>>notNull());
 918             EasyMock.expectLastCall().once();
 919         }
 920
 921         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
 922         EasyMock.expectLastCall().once();
 923
 924         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
 925                 .once();
 926
 927         EasyMock.replay(mockDevice, mockIDevice);
 928         EasyMock.replay(mockListener);
 929
 930         deqpTest.setDevice(mockDevice);
 931         deqpTest.run(mockListener);
 932
 933         EasyMock.verify(mockListener);
 934         EasyMock.verify(mockDevice, mockIDevice);
 935     }
 936
 937     /**
 938      * Test that test are left unexecuted if pm list query fails
 939      */
 940     public void testRun_queryPmListFailure()
 941             throws Exception {
 942         final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
 943
 944         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
 945         ITestInvocationListener mockListener
 946                 = EasyMock.createStrictMock(ITestInvocationListener.class);
 947         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
 948         tests.add(testId);
 949
 950         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
 951         OptionSetter setter = new OptionSetter(deqpTest);
 952         // Note: If the rotation is the default unspecified, features are not queried at all
 953         setter.setOptionValue("deqp-screen-rotation", "90");
 954
 955         deqpTest.setDevice(mockDevice);
 956
 957         int version = 3 << 16;
 958         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
 959                 .andReturn(Integer.toString(version)).atLeastOnce();
 960
 961         EasyMock.expect(mockDevice.executeShellCommand("pm list features"))
 962                 .andReturn("not a valid format");
 963
 964         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
 965             andReturn("").once();
 966
 967         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
 968                 EasyMock.eq(true),
 969                 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
 970                 .once();
 971
 972         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
 973                 .andReturn("").once();
 974
 975         mockListener.testRunStarted(getTestId(deqpTest), 1);
 976         EasyMock.expectLastCall().once();
 977
 978         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
 979         EasyMock.expectLastCall().once();
 980
 981         EasyMock.replay(mockDevice);
 982         EasyMock.replay(mockListener);
 983         deqpTest.run(mockListener);
 984         EasyMock.verify(mockListener);
 985         EasyMock.verify(mockDevice);
 986     }
 987
 988     /**
 989      * Test that test are left unexecuted if renderablity query fails
 990      */
 991     public void testRun_queryRenderabilityFailure()
 992             throws Exception {
 993         final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
 994
 995         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
 996         ITestInvocationListener mockListener
 997                 = EasyMock.createStrictMock(ITestInvocationListener.class);
 998
 999         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1000         tests.add(testId);
1001
1002         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
1003
1004         deqpTest.setDevice(mockDevice);
1005
1006         int version = 3 << 16;
1007         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1008                 .andReturn(Integer.toString(version)).atLeastOnce();
1009
1010         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1011             andReturn("").once();
1012
1013         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1014                 EasyMock.eq(true),
1015                 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1016                 .once();
1017
1018         expectRenderConfigQueryAndReturn(mockDevice,
1019                 "--deqp-gl-config-name=rgba8888d24s8 "
1020                 + "--deqp-screen-rotation=unspecified "
1021                 + "--deqp-surface-type=window "
1022                 + "--deqp-gl-major-version=3 "
1023                 + "--deqp-gl-minor-version=0", "Maybe?");
1024
1025         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
1026                 .andReturn("").once();
1027
1028         mockListener.testRunStarted(getTestId(deqpTest), 1);
1029         EasyMock.expectLastCall().once();
1030
1031         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1032         EasyMock.expectLastCall().once();
1033
1034         EasyMock.replay(mockDevice);
1035         EasyMock.replay(mockListener);
1036         deqpTest.run(mockListener);
1037         EasyMock.verify(mockListener);
1038         EasyMock.verify(mockDevice);
1039     }
1040
1041     /**
1042      * Test that orientation is supplied to runner correctly
1043      */
1044     private void testOrientation(final String rotation, final String featureString)
1045             throws Exception {
1046         final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
1047         final String testPath = "dEQP-GLES3.orientation.test";
1048         final String testTrie = "{dEQP-GLES3{orientation{test}}}";
1049         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
1050                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1051                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
1052                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1053                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
1054                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1055                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
1056                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1057                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
1058                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1059                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
1060                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1061                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
1062                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1063                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
1064                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
1065                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1066                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
1067                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
1068                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
1069                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1070                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
1071                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1072                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
1073                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1074                 + "INSTRUMENTATION_CODE: 0\r\n";
1075
1076         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1077         ITestInvocationListener mockListener
1078                 = EasyMock.createStrictMock(ITestInvocationListener.class);
1079         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
1080
1081         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1082         tests.add(testId);
1083
1084         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
1085         OptionSetter setter = new OptionSetter(deqpTest);
1086         setter.setOptionValue("deqp-screen-rotation", rotation);
1087
1088         deqpTest.setDevice(mockDevice);
1089
1090         int version = 3 << 16;
1091         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1092                 .andReturn(Integer.toString(version)).atLeastOnce();
1093
1094         if (!rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED)) {
1095             EasyMock.expect(mockDevice.executeShellCommand("pm list features"))
1096                     .andReturn(featureString);
1097         }
1098
1099         final boolean isPortraitOrientation =
1100                 rotation.equals(BatchRunConfiguration.ROTATION_PORTRAIT) ||
1101                 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT);
1102         final boolean isLandscapeOrientation =
1103                 rotation.equals(BatchRunConfiguration.ROTATION_LANDSCAPE) ||
1104                 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE);
1105         final boolean executable =
1106                 rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED) ||
1107                 (isPortraitOrientation &&
1108                 featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) ||
1109                 (isLandscapeOrientation &&
1110                 featureString.contains(DeqpTestRunner.FEATURE_LANDSCAPE));
1111
1112         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1113             andReturn("").once();
1114
1115         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1116                 EasyMock.eq(true),
1117                 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1118                 .once();
1119
1120         if (executable) {
1121             expectRenderConfigQuery(mockDevice, String.format(
1122                     "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=%s "
1123                     + "--deqp-surface-type=window --deqp-gl-major-version=3 "
1124                     + "--deqp-gl-minor-version=0", rotation));
1125
1126             String commandLine = String.format(
1127                     "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
1128                     + "--deqp-screen-rotation=%s "
1129                     + "--deqp-surface-type=window "
1130                     + "--deqp-log-images=disable "
1131                     + "--deqp-watchdog=enable",
1132                     CASE_LIST_FILE_NAME, rotation);
1133
1134             runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
1135                     output);
1136         }
1137
1138         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
1139                 .andReturn("").once();
1140
1141         mockListener.testRunStarted(getTestId(deqpTest), 1);
1142         EasyMock.expectLastCall().once();
1143
1144         mockListener.testStarted(EasyMock.eq(testId));
1145         EasyMock.expectLastCall().once();
1146
1147         mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
1148         EasyMock.expectLastCall().once();
1149
1150         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1151         EasyMock.expectLastCall().once();
1152
1153         EasyMock.replay(mockDevice, mockIDevice);
1154         EasyMock.replay(mockListener);
1155         deqpTest.run(mockListener);
1156         EasyMock.verify(mockListener);
1157         EasyMock.verify(mockDevice, mockIDevice);
1158     }
1159
1160     /**
1161      * Test OpeGL ES3 tests on device with OpenGL ES2.
1162      */
1163     public void testRun_require30DeviceVersion20() throws Exception {
1164         testGlesVersion(3, 0, 2, 0);
1165     }
1166
1167     /**
1168      * Test OpeGL ES3.1 tests on device with OpenGL ES2.
1169      */
1170     public void testRun_require31DeviceVersion20() throws Exception {
1171         testGlesVersion(3, 1, 2, 0);
1172     }
1173
1174     /**
1175      * Test OpeGL ES3 tests on device with OpenGL ES3.
1176      */
1177     public void testRun_require30DeviceVersion30() throws Exception {
1178         testGlesVersion(3, 0, 3, 0);
1179     }
1180
1181     /**
1182      * Test OpeGL ES3.1 tests on device with OpenGL ES3.
1183      */
1184     public void testRun_require31DeviceVersion30() throws Exception {
1185         testGlesVersion(3, 1, 3, 0);
1186     }
1187
1188     /**
1189      * Test OpeGL ES3 tests on device with OpenGL ES3.1.
1190      */
1191     public void testRun_require30DeviceVersion31() throws Exception {
1192         testGlesVersion(3, 0, 3, 1);
1193     }
1194
1195     /**
1196      * Test OpeGL ES3.1 tests on device with OpenGL ES3.1.
1197      */
1198     public void testRun_require31DeviceVersion31() throws Exception {
1199         testGlesVersion(3, 1, 3, 1);
1200     }
1201
1202     /**
1203      * Test dEQP Pass result code.
1204      */
1205     public void testRun_resultPass() throws Exception {
1206         testResultCode("Pass", true);
1207     }
1208
1209     /**
1210      * Test dEQP Fail result code.
1211      */
1212     public void testRun_resultFail() throws Exception {
1213         testResultCode("Fail", false);
1214     }
1215
1216     /**
1217      * Test dEQP NotSupported result code.
1218      */
1219     public void testRun_resultNotSupported() throws Exception {
1220         testResultCode("NotSupported", true);
1221     }
1222
1223     /**
1224      * Test dEQP QualityWarning result code.
1225      */
1226     public void testRun_resultQualityWarning() throws Exception {
1227         testResultCode("QualityWarning", true);
1228     }
1229
1230     /**
1231      * Test dEQP CompatibilityWarning result code.
1232      */
1233     public void testRun_resultCompatibilityWarning() throws Exception {
1234         testResultCode("CompatibilityWarning", true);
1235     }
1236
1237     /**
1238      * Test dEQP ResourceError result code.
1239      */
1240     public void testRun_resultResourceError() throws Exception {
1241         testResultCode("ResourceError", false);
1242     }
1243
1244     /**
1245      * Test dEQP InternalError result code.
1246      */
1247     public void testRun_resultInternalError() throws Exception {
1248         testResultCode("InternalError", false);
1249     }
1250
1251     /**
1252      * Test dEQP Crash result code.
1253      */
1254     public void testRun_resultCrash() throws Exception {
1255         testResultCode("Crash", false);
1256     }
1257
1258     /**
1259      * Test dEQP Timeout result code.
1260      */
1261     public void testRun_resultTimeout() throws Exception {
1262         testResultCode("Timeout", false);
1263     }
1264     /**
1265      * Test dEQP Orientation
1266      */
1267     public void testRun_orientationLandscape() throws Exception {
1268         testOrientation("90", ALL_FEATURES);
1269     }
1270
1271     /**
1272      * Test dEQP Orientation
1273      */
1274     public void testRun_orientationPortrait() throws Exception {
1275         testOrientation("0", ALL_FEATURES);
1276     }
1277
1278     /**
1279      * Test dEQP Orientation
1280      */
1281     public void testRun_orientationReverseLandscape() throws Exception {
1282         testOrientation("270", ALL_FEATURES);
1283     }
1284
1285     /**
1286      * Test dEQP Orientation
1287      */
1288     public void testRun_orientationReversePortrait() throws Exception {
1289         testOrientation("180", ALL_FEATURES);
1290     }
1291
1292     /**
1293      * Test dEQP Orientation
1294      */
1295     public void testRun_orientationUnspecified() throws Exception {
1296         testOrientation("unspecified", ALL_FEATURES);
1297     }
1298
1299     /**
1300      * Test dEQP Orientation with limited features
1301      */
1302     public void testRun_orientationUnspecifiedLimitedFeatures() throws Exception {
1303         testOrientation("unspecified", ONLY_LANDSCAPE_FEATURES);
1304     }
1305
1306     /**
1307      * Test dEQP Orientation with limited features
1308      */
1309     public void testRun_orientationLandscapeLimitedFeatures() throws Exception {
1310         testOrientation("90", ONLY_LANDSCAPE_FEATURES);
1311     }
1312
1313     /**
1314      * Test dEQP Orientation with limited features
1315      */
1316     public void testRun_orientationPortraitLimitedFeatures() throws Exception {
1317         testOrientation("0", ONLY_LANDSCAPE_FEATURES);
1318     }
1319
1320     /**
1321      * Test dEQP unsupported pixel format
1322      */
1323     public void testRun_unsupportedPixelFormat() throws Exception {
1324         final String pixelFormat = "rgba5658d16m4";
1325         final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.pixelformat", "test");
1326
1327         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1328         ITestInvocationListener mockListener
1329                 = EasyMock.createStrictMock(ITestInvocationListener.class);
1330
1331         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1332         tests.add(testId);
1333
1334         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
1335         OptionSetter setter = new OptionSetter(deqpTest);
1336         setter.setOptionValue("deqp-gl-config-name", pixelFormat);
1337
1338         deqpTest.setDevice(mockDevice);
1339
1340         int version = 3 << 16;
1341         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1342                 .andReturn(Integer.toString(version)).atLeastOnce();
1343
1344         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1345             andReturn("").once();
1346
1347         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1348                 EasyMock.eq(true),
1349                 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1350                 .once();
1351
1352         expectRenderConfigQueryAndReturn(mockDevice, String.format(
1353                 "--deqp-gl-config-name=%s --deqp-screen-rotation=unspecified "
1354                 + "--deqp-surface-type=window "
1355                 + "--deqp-gl-major-version=3 "
1356                 + "--deqp-gl-minor-version=0", pixelFormat), "No");
1357
1358         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
1359                 .andReturn("").once();
1360
1361         mockListener.testRunStarted(getTestId(deqpTest), 1);
1362         EasyMock.expectLastCall().once();
1363
1364         mockListener.testStarted(EasyMock.eq(testId));
1365         EasyMock.expectLastCall().once();
1366
1367         mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
1368         EasyMock.expectLastCall().once();
1369
1370         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1371         EasyMock.expectLastCall().once();
1372
1373         EasyMock.replay(mockDevice);
1374         EasyMock.replay(mockListener);
1375         deqpTest.run(mockListener);
1376         EasyMock.verify(mockListener);
1377         EasyMock.verify(mockDevice);
1378     }
1379
1380     public static interface RecoverableTestDevice extends ITestDevice {
1381         public void recoverDevice() throws DeviceNotAvailableException;
1382     }
1383
1384     private static enum RecoveryEvent {
1385         PROGRESS,
1386         FAIL_CONNECTION_REFUSED,
1387         FAIL_LINK_KILLED,
1388     }
1389
1390     private void runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events)
1391             throws DeviceNotAvailableException {
1392         for (RecoveryEvent event : events) {
1393             switch (event) {
1394                 case PROGRESS:
1395                     recovery.onExecutionProgressed();
1396                     break;
1397                 case FAIL_CONNECTION_REFUSED:
1398                     recovery.recoverConnectionRefused();
1399                     break;
1400                 case FAIL_LINK_KILLED:
1401                     recovery.recoverComLinkKilled();
1402                     break;
1403             }
1404         }
1405     }
1406
1407     private void setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider) {
1408         mockSleepProvider.sleep(EasyMock.gt(0));
1409         EasyMock.expectLastCall().once();
1410     }
1411
1412     private void setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice,
1413             DeqpTestRunner.ISleepProvider mockSleepProvider) throws DeviceNotAvailableException {
1414         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1415                 andReturn("root 1234 com.drawelement.deqp").once();
1416
1417         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
1418                 andReturn("").once();
1419
1420         // Recovery checks if kill failed
1421         mockSleepProvider.sleep(EasyMock.gt(0));
1422         EasyMock.expectLastCall().once();
1423         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1424                 andReturn("").once();
1425     }
1426
1427     private void setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice)
1428             throws DeviceNotAvailableException {
1429         mockDevice.recoverDevice();
1430         EasyMock.expectLastCall().once();
1431     }
1432
1433     private void setRecoveryExpectationReboot(RecoverableTestDevice mockDevice)
1434             throws DeviceNotAvailableException {
1435         mockDevice.reboot();
1436         EasyMock.expectLastCall().once();
1437     }
1438
1439     private int setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice,
1440             DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)
1441             throws DeviceNotAvailableException {
1442         switch (numConsecutiveErrors) {
1443             case 0:
1444             case 1:
1445                 setRecoveryExpectationRecovery(mockDevice);
1446                 return 2;
1447             case 2:
1448                 setRecoveryExpectationReboot(mockDevice);
1449                 return 3;
1450             default:
1451                 return 4;
1452         }
1453     }
1454
1455     private int setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice,
1456             DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)
1457             throws DeviceNotAvailableException {
1458         switch (numConsecutiveErrors) {
1459             case 0:
1460                 setRecoveryExpectationWait(mockSleepProvider);
1461                 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider);
1462                 return 1;
1463             case 1:
1464                 setRecoveryExpectationRecovery(mockDevice);
1465                 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider);
1466                 return 2;
1467             case 2:
1468                 setRecoveryExpectationReboot(mockDevice);
1469                 return 3;
1470             default:
1471                 return 4;
1472         }
1473     }
1474
1475     private void setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice,
1476             DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events)
1477             throws DeviceNotAvailableException {
1478         int numConsecutiveErrors = 0;
1479         for (RecoveryEvent event : events) {
1480             switch (event) {
1481                 case PROGRESS:
1482                     numConsecutiveErrors = 0;
1483                     break;
1484                 case FAIL_CONNECTION_REFUSED:
1485                     numConsecutiveErrors = setRecoveryExpectationOfAConnFailure(mockDevice,
1486                             mockSleepProvider, numConsecutiveErrors);
1487                     break;
1488                 case FAIL_LINK_KILLED:
1489                     numConsecutiveErrors = setRecoveryExpectationOfAComKilled(mockDevice,
1490                             mockSleepProvider, numConsecutiveErrors);
1491                     break;
1492             }
1493         }
1494     }
1495
1496     /**
1497      * Test dEQP runner recovery state machine.
1498      */
1499     private void testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern)
1500             throws Exception {
1501         DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery();
1502         IMocksControl orderedControl = EasyMock.createStrictControl();
1503         RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class);
1504         EasyMock.expect(mockDevice.getSerialNumber()).andStubReturn("SERIAL");
1505         DeqpTestRunner.ISleepProvider mockSleepProvider =
1506                 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class);
1507
1508         setRecoveryExpectationsOfAPattern(mockDevice, mockSleepProvider, pattern);
1509
1510         orderedControl.replay();
1511
1512         recovery.setDevice(mockDevice);
1513         recovery.setSleepProvider(mockSleepProvider);
1514         try {
1515             runRecoveryWithPattern(recovery, pattern);
1516             if (!expectSuccess) {
1517                 fail("Expected DeviceNotAvailableException");
1518             }
1519         } catch (DeviceNotAvailableException ex) {
1520             if (expectSuccess) {
1521                 fail("Did not expect DeviceNotAvailableException");
1522             }
1523         }
1524
1525         orderedControl.verify();
1526     }
1527
1528     // basic patterns
1529
1530     public void testRecovery_NoEvents() throws Exception {
1531         testRecoveryWithPattern(true);
1532     }
1533
1534     public void testRecovery_AllOk() throws Exception {
1535         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, RecoveryEvent.PROGRESS);
1536     }
1537
1538     // conn fail patterns
1539
1540     public void testRecovery_OneConnectionFailureBegin() throws Exception {
1541         testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1542                 RecoveryEvent.PROGRESS);
1543     }
1544
1545     public void testRecovery_TwoConnectionFailuresBegin() throws Exception {
1546         testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1547                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS);
1548     }
1549
1550     public void testRecovery_ThreeConnectionFailuresBegin() throws Exception {
1551         testRecoveryWithPattern(false, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1552                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED);
1553     }
1554
1555     public void testRecovery_OneConnectionFailureMid() throws Exception {
1556         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1557                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS);
1558     }
1559
1560     public void testRecovery_TwoConnectionFailuresMid() throws Exception {
1561         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1562                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1563                 RecoveryEvent.PROGRESS);
1564     }
1565
1566     public void testRecovery_ThreeConnectionFailuresMid() throws Exception {
1567         testRecoveryWithPattern(false, RecoveryEvent.PROGRESS,
1568                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1569                 RecoveryEvent.FAIL_CONNECTION_REFUSED);
1570     }
1571
1572     // link fail patterns
1573
1574     public void testRecovery_OneLinkFailureBegin() throws Exception {
1575         testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
1576                 RecoveryEvent.PROGRESS);
1577     }
1578
1579     public void testRecovery_TwoLinkFailuresBegin() throws Exception {
1580         testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
1581                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
1582     }
1583
1584     public void testRecovery_ThreeLinkFailuresBegin() throws Exception {
1585         testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
1586                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1587                 RecoveryEvent.PROGRESS);
1588     }
1589
1590     public void testRecovery_FourLinkFailuresBegin() throws Exception {
1591         testRecoveryWithPattern(false, RecoveryEvent.FAIL_LINK_KILLED,
1592                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1593                 RecoveryEvent.FAIL_LINK_KILLED);
1594     }
1595
1596     public void testRecovery_OneLinkFailureMid() throws Exception {
1597         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1598                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
1599     }
1600
1601     public void testRecovery_TwoLinkFailuresMid() throws Exception {
1602         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1603                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1604                 RecoveryEvent.PROGRESS);
1605     }
1606
1607     public void testRecovery_ThreeLinkFailuresMid() throws Exception {
1608         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1609                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1610                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
1611     }
1612
1613     public void testRecovery_FourLinkFailuresMid() throws Exception {
1614         testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
1615                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1616                 RecoveryEvent.FAIL_LINK_KILLED);
1617     }
1618
1619     // mixed patterns
1620
1621     public void testRecovery_MixedFailuresProgressBetween() throws Exception {
1622         testRecoveryWithPattern(true,
1623                 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
1624                 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1625                 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
1626                 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1627                 RecoveryEvent.PROGRESS);
1628     }
1629
1630     public void testRecovery_MixedFailuresNoProgressBetween() throws Exception {
1631         testRecoveryWithPattern(true,
1632                 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
1633                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_LINK_KILLED,
1634                 RecoveryEvent.PROGRESS);
1635     }
1636
1637     /**
1638      * Test recovery if process cannot be killed
1639      */
1640     public void testRecovery_unkillableProcess () throws Exception {
1641         DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery();
1642         IMocksControl orderedControl = EasyMock.createStrictControl();
1643         RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class);
1644         DeqpTestRunner.ISleepProvider mockSleepProvider =
1645                 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class);
1646
1647         // recovery attempts to kill the process after a timeout
1648         mockSleepProvider.sleep(EasyMock.gt(0));
1649         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1650                 andReturn("root 1234 com.drawelement.deqp").once();
1651         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
1652                 andReturn("").once();
1653
1654         // Recovery checks if kill failed
1655         mockSleepProvider.sleep(EasyMock.gt(0));
1656         EasyMock.expectLastCall().once();
1657         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1658                 andReturn("root 1234 com.drawelement.deqp").once();
1659
1660         // Recovery resets the connection
1661         mockDevice.recoverDevice();
1662         EasyMock.expectLastCall().once();
1663
1664         // and attempts to kill the process again
1665         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1666                 andReturn("root 1234 com.drawelement.deqp").once();
1667         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
1668                 andReturn("").once();
1669
1670         // Recovery checks if kill failed
1671         mockSleepProvider.sleep(EasyMock.gt(0));
1672         EasyMock.expectLastCall().once();
1673         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1674                 andReturn("root 1234 com.drawelement.deqp").once();
1675
1676         // recovery reboots the device
1677         mockDevice.reboot();
1678         EasyMock.expectLastCall().once();
1679
1680         orderedControl.replay();
1681         recovery.setDevice(mockDevice);
1682         recovery.setSleepProvider(mockSleepProvider);
1683         recovery.recoverComLinkKilled();
1684         orderedControl.verify();
1685     }
1686
1687     /**
1688      * Test external interruption before batch run.
1689      */
1690     public void testInterrupt_killBeforeBatch() throws Exception {
1691         final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.interrupt", "test");
1692
1693         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1694         tests.add(testId);
1695
1696         ITestInvocationListener mockListener
1697                 = EasyMock.createStrictMock(ITestInvocationListener.class);
1698         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1699         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
1700         IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class);
1701
1702         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
1703
1704         deqpTest.setDevice(mockDevice);
1705         deqpTest.setRunUtil(mockRunUtil);
1706
1707         int version = 3 << 16;
1708         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1709                 .andReturn(Integer.toString(version)).atLeastOnce();
1710
1711         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1712             andReturn("").once();
1713
1714         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1715                 EasyMock.eq(true),
1716                 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1717                 .once();
1718
1719         expectRenderConfigQuery(mockDevice,
1720                 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified "
1721                 + "--deqp-surface-type=window --deqp-gl-major-version=3 "
1722                 + "--deqp-gl-minor-version=0");
1723
1724         mockRunUtil.sleep(0);
1725         EasyMock.expectLastCall().andThrow(new RunInterruptedException());
1726
1727         mockListener.testRunStarted(getTestId(deqpTest), 1);
1728         EasyMock.expectLastCall().once();
1729         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.anyObject());
1730         EasyMock.expectLastCall().once();
1731
1732         EasyMock.replay(mockDevice, mockIDevice);
1733         EasyMock.replay(mockListener);
1734         EasyMock.replay(mockRunUtil);
1735         try {
1736             deqpTest.run(mockListener);
1737             fail("expected RunInterruptedException");
1738         } catch (RunInterruptedException ex) {
1739             // expected
1740         }
1741         EasyMock.verify(mockRunUtil);
1742         EasyMock.verify(mockListener);
1743         EasyMock.verify(mockDevice, mockIDevice);
1744     }
1745
1746     private void runShardedTest(TestIdentifier[] testIds,
1747             ArrayList<ArrayList<TestIdentifier>> testsForShard) throws Exception {
1748         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1749         for (TestIdentifier id : testIds) tests.add(id);
1750
1751         DeqpTestRunner runner = buildGlesTestRunner(3, 0, tests, mTestsDir);
1752         ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split();
1753
1754         for (int shardIndex = 0; shardIndex < shards.size(); shardIndex++) {
1755             DeqpTestRunner shard = (DeqpTestRunner)shards.get(shardIndex);
1756             shard.setBuildHelper(getMockBuildHelper(mTestsDir));
1757
1758             ArrayList<TestIdentifier> shardTests = testsForShard.get(shardIndex);
1759
1760             ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1761             ITestInvocationListener mockListener
1762                     = EasyMock.createStrictMock(ITestInvocationListener.class);
1763             IDevice mockIDevice = EasyMock.createMock(IDevice.class);
1764             int version = 3 << 16;
1765             EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1766                     .andReturn(Integer.toString(version)).atLeastOnce();
1767
1768             EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
1769                     .once();
1770             EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1771                     EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
1772                     .andReturn(null).once();
1773
1774             mockListener.testRunStarted(getTestId(shard), shardTests.size());
1775             EasyMock.expectLastCall().once();
1776
1777             expectRenderConfigQuery(mockDevice, 3, 0);
1778
1779             String testOut = buildTestProcessOutput(shardTests);
1780             // NOTE: This assumes that there won't be multiple batches per shard!
1781             runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut);
1782
1783             for (int i = 0; i < shardTests.size(); i++) {
1784                 mockListener.testStarted(EasyMock.eq(shardTests.get(i)));
1785                 EasyMock.expectLastCall().once();
1786
1787                 mockListener.testEnded(EasyMock.eq(shardTests.get(i)),
1788                                        EasyMock.<Map<String, String>>notNull());
1789
1790                 EasyMock.expectLastCall().once();
1791             }
1792
1793             mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1794             EasyMock.expectLastCall().once();
1795
1796             EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
1797                     .once();
1798
1799             EasyMock.replay(mockDevice, mockIDevice);
1800             EasyMock.replay(mockListener);
1801
1802             shard.setDevice(mockDevice);
1803             shard.run(mockListener);
1804
1805             EasyMock.verify(mockListener);
1806             EasyMock.verify(mockDevice, mockIDevice);
1807         }
1808     }
1809
1810     public void testSharding_smallTrivial() throws Exception {
1811         final TestIdentifier[] testIds = {
1812                 new TestIdentifier("dEQP-GLES3.info", "vendor"),
1813                 new TestIdentifier("dEQP-GLES3.info", "renderer"),
1814                 new TestIdentifier("dEQP-GLES3.info", "version"),
1815                 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
1816                 new TestIdentifier("dEQP-GLES3.info", "extensions"),
1817                 new TestIdentifier("dEQP-GLES3.info", "render_target")
1818         };
1819         ArrayList<ArrayList<TestIdentifier>> shardedTests = new ArrayList<>();
1820         ArrayList<TestIdentifier> shardOne = new ArrayList<>();
1821         for (int i = 0; i < testIds.length; i++) {
1822             shardOne.add(testIds[i]);
1823         }
1824         shardedTests.add(shardOne);
1825         runShardedTest(testIds, shardedTests);
1826     }
1827
1828     public void testSharding_twoShards() throws Exception {
1829         final int TEST_COUNT = 1237;
1830         final int SHARD_SIZE = 1000;
1831
1832         ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT);
1833         for (int i = 0; i < TEST_COUNT; i++) {
1834             testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i)));
1835         }
1836
1837         ArrayList<ArrayList<TestIdentifier>> shardedTests = new ArrayList<>();
1838         ArrayList<TestIdentifier> shard = new ArrayList<>();
1839         for (int i = 0; i < testIds.size(); i++) {
1840             if (i == SHARD_SIZE) {
1841                 shardedTests.add(shard);
1842                 shard = new ArrayList<>();
1843             }
1844             shard.add(testIds.get(i));
1845         }
1846         shardedTests.add(shard);
1847         runShardedTest(testIds.toArray(new TestIdentifier[testIds.size()]), shardedTests);
1848     }
1849
1850     public void testSharding_empty() throws Exception {
1851         DeqpTestRunner runner = buildGlesTestRunner(3, 0, new ArrayList<TestIdentifier>(), mTestsDir);
1852         ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split();
1853         // Returns null when cannot be sharded.
1854         assertNull(shards);
1855     }
1856
1857     /**
1858      * Test external interruption in testFailed().
1859      */
1860     public void testInterrupt_killReportTestFailed() throws Exception {
1861         final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.interrupt", "test");
1862         final String testPath = "dEQP-GLES3.interrupt.test";
1863         final String testTrie = "{dEQP-GLES3{interrupt{test}}}";
1864         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
1865                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1866                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
1867                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1868                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
1869                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1870                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
1871                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1872                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
1873                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1874                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
1875                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1876                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
1877                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1878                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
1879                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
1880                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1881                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Fail\r\n"
1882                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Fail\r\n"
1883                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
1884                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1885                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
1886                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1887                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
1888                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1889                 + "INSTRUMENTATION_CODE: 0\r\n";
1890
1891         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1892         tests.add(testId);
1893
1894         ITestInvocationListener mockListener
1895                 = EasyMock.createStrictMock(ITestInvocationListener.class);
1896         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1897         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
1898         IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class);
1899
1900         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
1901
1902         deqpTest.setDevice(mockDevice);
1903         deqpTest.setRunUtil(mockRunUtil);
1904
1905         int version = 3 << 16;
1906         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1907                 .andReturn(Integer.toString(version)).atLeastOnce();
1908
1909         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1910             andReturn("").once();
1911
1912         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1913                 EasyMock.eq(true),
1914                 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1915                 .once();
1916
1917         expectRenderConfigQuery(mockDevice,
1918                 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified "
1919                 + "--deqp-surface-type=window --deqp-gl-major-version=3 "
1920                 + "--deqp-gl-minor-version=0");
1921
1922         mockRunUtil.sleep(0);
1923         EasyMock.expectLastCall().once();
1924
1925         String commandLine = String.format(
1926                 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
1927                 + "--deqp-screen-rotation=unspecified "
1928                 + "--deqp-surface-type=window "
1929                 + "--deqp-log-images=disable "
1930                 + "--deqp-watchdog=enable",
1931                 CASE_LIST_FILE_NAME);
1932
1933         runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
1934                 output);
1935
1936         mockListener.testRunStarted(getTestId(deqpTest), 1);
1937         EasyMock.expectLastCall().once();
1938
1939         mockListener.testStarted(EasyMock.eq(testId));
1940         EasyMock.expectLastCall().once();
1941
1942         mockListener.testFailed(EasyMock.eq(testId), EasyMock.<String>notNull());
1943         EasyMock.expectLastCall().andThrow(new RunInterruptedException());
1944
1945         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.anyObject());
1946         EasyMock.expectLastCall().once();
1947         EasyMock.replay(mockDevice, mockIDevice);
1948         EasyMock.replay(mockListener);
1949         EasyMock.replay(mockRunUtil);
1950         try {
1951             deqpTest.run(mockListener);
1952             fail("expected RunInterruptedException");
1953         } catch (RunInterruptedException ex) {
1954             // expected
1955         }
1956         EasyMock.verify(mockRunUtil);
1957         EasyMock.verify(mockListener);
1958         EasyMock.verify(mockDevice, mockIDevice);
1959     }
1960
1961     public void testRuntimeHint_optionSet() throws Exception {
1962         /* MultiLineReceiver expects "\r\n" line ending. */
1963         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
1964                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1965                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
1966                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1967                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
1968                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1969                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
1970                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1971                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
1972                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1973                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
1974                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1975                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
1976                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1977                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
1978                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n"
1979                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1980                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
1981                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
1982                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
1983                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1984                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
1985                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1986                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
1987                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n"
1988                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1989                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
1990                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
1991                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
1992                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1993                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
1994                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1995                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
1996                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n"
1997                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1998                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
1999                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2000                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2001                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2002                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2003                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2004                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2005                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n"
2006                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2007                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2008                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2009                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2010                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2011                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2012                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2013                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2014                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n"
2015                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2016                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2017                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2018                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2019                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2020                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2021                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2022                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2023                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n"
2024                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2025                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2026                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2027                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2028                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2029                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2030                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2031                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
2032                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2033                 + "INSTRUMENTATION_CODE: 0\r\n";
2034
2035         final TestIdentifier[] testIds = {
2036                 new TestIdentifier("dEQP-GLES3.info", "vendor"),
2037                 new TestIdentifier("dEQP-GLES3.info", "renderer"),
2038                 new TestIdentifier("dEQP-GLES3.info", "version"),
2039                 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
2040                 new TestIdentifier("dEQP-GLES3.info", "extensions"),
2041                 new TestIdentifier("dEQP-GLES3.info", "render_target")
2042         };
2043
2044         final String[] testPaths = {
2045                 "dEQP-GLES3.info.vendor",
2046                 "dEQP-GLES3.info.renderer",
2047                 "dEQP-GLES3.info.version",
2048                 "dEQP-GLES3.info.shading_language_version",
2049                 "dEQP-GLES3.info.extensions",
2050                 "dEQP-GLES3.info.render_target"
2051         };
2052
2053         final String testTrie
2054                 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}";
2055
2056         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
2057         ITestInvocationListener mockListener
2058                 = EasyMock.createStrictMock(ITestInvocationListener.class);
2059         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
2060
2061         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
2062
2063         for (TestIdentifier id : testIds) {
2064             tests.add(id);
2065         }
2066
2067         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
2068         OptionSetter setter = new OptionSetter(deqpTest);
2069         final long runtimeMs = 123456;
2070         setter.setOptionValue("runtime-hint", String.valueOf(runtimeMs));
2071         assertEquals("Wrong expected runtime - option not passed cleanly", runtimeMs, deqpTest.getRuntimeHint());
2072
2073         // Try running the tests as well. The unit tests do not set the hint be default,
2074         // so that case is covered.
2075
2076         int version = 3 << 16;
2077         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
2078                 .andReturn(Integer.toString(version)).atLeastOnce();
2079
2080         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
2081                 .once();
2082         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
2083                 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
2084                 .andReturn(null).once();
2085
2086         expectRenderConfigQuery(mockDevice, 3, 0);
2087
2088         String commandLine = String.format(
2089                 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
2090                 + "--deqp-screen-rotation=unspecified "
2091                 + "--deqp-surface-type=window "
2092                 + "--deqp-log-images=disable "
2093                 + "--deqp-watchdog=enable",
2094                 CASE_LIST_FILE_NAME);
2095
2096         runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
2097
2098         mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
2099         EasyMock.expectLastCall().once();
2100
2101         for (int i = 0; i < testPaths.length; i++) {
2102             mockListener.testStarted(EasyMock.eq(testIds[i]));
2103             EasyMock.expectLastCall().once();
2104
2105             mockListener.testEnded(EasyMock.eq(testIds[i]),
2106                     EasyMock.<Map<String, String>>notNull());
2107
2108             EasyMock.expectLastCall().once();
2109         }
2110
2111         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
2112         EasyMock.expectLastCall().once();
2113
2114         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
2115                 .once();
2116
2117         EasyMock.replay(mockDevice, mockIDevice);
2118         EasyMock.replay(mockListener);
2119
2120         deqpTest.setDevice(mockDevice);
2121         deqpTest.run(mockListener);
2122
2123         EasyMock.verify(mockListener);
2124         EasyMock.verify(mockDevice, mockIDevice);
2125     }
2126
2127     public void testRuntimeHint_optionSetSharded() throws Exception {
2128         final int TEST_COUNT = 1237;
2129         final int SHARD_SIZE = 1000;
2130
2131         ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT);
2132         for (int i = 0; i < TEST_COUNT; i++) {
2133             testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i)));
2134         }
2135
2136         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir);
2137         OptionSetter setter = new OptionSetter(deqpTest);
2138         final long fullRuntimeMs = testIds.size()*100;
2139         setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs));
2140
2141         ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)deqpTest.split();
2142         assertEquals("First shard's time not proportional to test count",
2143                  (fullRuntimeMs*SHARD_SIZE)/TEST_COUNT,
2144                  ((IRuntimeHintProvider)shards.get(0)).getRuntimeHint());
2145         assertEquals("Second shard's time not proportional to test count",
2146                  (fullRuntimeMs*(TEST_COUNT-SHARD_SIZE))/TEST_COUNT,
2147                  ((IRuntimeHintProvider)shards.get(1)).getRuntimeHint());
2148     }
2149
2150     /**
2151      * Test that strict shardable is able to split deterministically the set of tests.
2152      */
2153     public void testGetTestShard() throws Exception {
2154         final int TEST_COUNT = 2237;
2155         final int SHARD_COUNT = 4;
2156
2157         ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT);
2158         for (int i = 0; i < TEST_COUNT; i++) {
2159             testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i)));
2160         }
2161
2162         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir);
2163         OptionSetter setter = new OptionSetter(deqpTest);
2164         final long fullRuntimeMs = testIds.size()*100;
2165         setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs));
2166
2167         DeqpTestRunner shard1 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 0);
2168         assertEquals(559, shard1.getTestInstance().size());
2169         int j = 0;
2170         // Ensure numbers, and that order is stable
2171         for (TestIdentifier t : shard1.getTestInstance().keySet()) {
2172             assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
2173                     String.format("%s#%s", t.getClassName(), t.getTestName()));
2174             j++;
2175         }
2176         DeqpTestRunner shard2 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 1);
2177         assertEquals(559, shard2.getTestInstance().size());
2178         for (TestIdentifier t : shard2.getTestInstance().keySet()) {
2179             assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
2180                     String.format("%s#%s", t.getClassName(), t.getTestName()));
2181             j++;
2182         }
2183         DeqpTestRunner shard3 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 2);
2184         assertEquals(559, shard3.getTestInstance().size());
2185         for (TestIdentifier t : shard3.getTestInstance().keySet()) {
2186             assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
2187                     String.format("%s#%s", t.getClassName(), t.getTestName()));
2188             j++;
2189         }
2190         DeqpTestRunner shard4 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 3);
2191         assertEquals(560, shard4.getTestInstance().size());
2192         for (TestIdentifier t : shard4.getTestInstance().keySet()) {
2193             assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
2194                     String.format("%s#%s", t.getClassName(), t.getTestName()));
2195             j++;
2196         }
2197         assertEquals(TEST_COUNT, j);
2198     }
2199
2200     /**
2201      * Test that strict shardable is creating an empty shard of the runner when too many shards
2202      * are requested.
2203      */
2204     public void testGetTestShard_tooManyShardRequested() throws Exception {
2205         final int TEST_COUNT = 2;
2206         final int SHARD_COUNT = 3;
2207
2208         ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT);
2209         for (int i = 0; i < TEST_COUNT; i++) {
2210             testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i)));
2211         }
2212         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir);
2213         OptionSetter setter = new OptionSetter(deqpTest);
2214         final long fullRuntimeMs = testIds.size()*100;
2215         setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs));
2216         DeqpTestRunner shard1 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 0);
2217         assertEquals(1, shard1.getTestInstance().size());
2218         int j = 0;
2219         // Ensure numbers, and that order is stable
2220         for (TestIdentifier t : shard1.getTestInstance().keySet()) {
2221             assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
2222                     String.format("%s#%s", t.getClassName(), t.getTestName()));
2223             j++;
2224         }
2225         DeqpTestRunner shard2 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 1);
2226         assertEquals(1, shard2.getTestInstance().size());
2227         for (TestIdentifier t : shard2.getTestInstance().keySet()) {
2228             assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
2229                     String.format("%s#%s", t.getClassName(), t.getTestName()));
2230             j++;
2231         }
2232         DeqpTestRunner shard3 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 2);
2233         assertTrue(shard3.getTestInstance().isEmpty());
2234         assertEquals(TEST_COUNT, j);
2235         ITestInvocationListener mockListener
2236                 = EasyMock.createStrictMock(ITestInvocationListener.class);
2237         mockListener.testRunStarted(EasyMock.anyObject(), EasyMock.eq(0));
2238         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.anyObject());
2239         EasyMock.replay(mockListener);
2240         shard3.run(mockListener);
2241         EasyMock.verify(mockListener);
2242     }
2243
2244     public void testRuntimeHint_optionNotSet() throws Exception {
2245         final TestIdentifier[] testIds = {
2246                 new TestIdentifier("dEQP-GLES3.info", "vendor"),
2247                 new TestIdentifier("dEQP-GLES3.info", "renderer"),
2248                 new TestIdentifier("dEQP-GLES3.info", "version"),
2249                 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
2250                 new TestIdentifier("dEQP-GLES3.info", "extensions"),
2251                 new TestIdentifier("dEQP-GLES3.info", "render_target")
2252         };
2253         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
2254
2255         for (TestIdentifier id : testIds) {
2256             tests.add(id);
2257         }
2258
2259         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
2260
2261         long runtime = deqpTest.getRuntimeHint();
2262         assertTrue("Runtime for tests must be positive", runtime > 0);
2263         assertTrue("Runtime for tests must be reasonable", runtime < (1000 * 10)); // Must be done in 10s
2264     }
2265
2266
2267     private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice,
2268             final String output) throws Exception {
2269         String cmd = String.format(
2270             "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
2271             + "--deqp-screen-rotation=unspecified "
2272             + "--deqp-surface-type=window "
2273             + "--deqp-log-images=disable "
2274             + "--deqp-watchdog=enable",
2275             CASE_LIST_FILE_NAME);
2276         runInstrumentationLineAndAnswer(mockDevice, mockIDevice, null, cmd, output);
2277     }
2278
2279     private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice,
2280             final String testTrie, final String cmd, final String output) throws Exception {
2281         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME)))
2282                 .andReturn("").once();
2283
2284         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME)))
2285                 .andReturn("").once();
2286
2287         if (testTrie == null) {
2288             mockDevice.pushString((String)EasyMock.anyObject(), EasyMock.eq(CASE_LIST_FILE_NAME));
2289         }
2290         else {
2291             mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME);
2292         }
2293         EasyMock.expectLastCall().andReturn(true).once();
2294
2295         String command = String.format(
2296                 "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\" "
2297                     + "-e deqpLogData \"%s\" %s",
2298                 AbiUtils.createAbiFlag(ABI.getName()), LOG_FILE_NAME, cmd, false,
2299                 INSTRUMENTATION_NAME);
2300
2301         EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice);
2302         mockIDevice.executeShellCommand(EasyMock.eq(command),
2303                 EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(),
2304                 EasyMock.isA(TimeUnit.class));
2305
2306         EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
2307             @Override
2308             public Object answer() {
2309                 IShellOutputReceiver receiver
2310                         = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
2311
2312                 receiver.addOutput(output.getBytes(), 0, output.length());
2313                 receiver.flush();
2314
2315                 return null;
2316             }
2317         });
2318     }
2319
2320     static private void writeStringsToFile(File target, Set<String> strings) throws IOException {
2321         try (PrintWriter out = new PrintWriter(new FileWriter(target))) {
2322             out.print(String.join(System.lineSeparator(), strings));
2323             out.println();
2324         }
2325     }
2326
2327     private void addFilterFileForOption(DeqpTestRunner test, Set<String> filters, String option)
2328             throws IOException, ConfigurationException {
2329         String filterFile = option + ".txt";
2330         writeStringsToFile(new File(mTestsDir, filterFile), filters);
2331         OptionSetter setter = new OptionSetter(test);
2332         setter.setOptionValue(option, filterFile);
2333     }
2334
2335     public void testIncludeFilterFile() throws Exception {
2336         final TestIdentifier[] testIds = {
2337                 new TestIdentifier("dEQP-GLES3.missing", "no"),
2338                 new TestIdentifier("dEQP-GLES3.missing", "nope"),
2339                 new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
2340                 new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
2341                 new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
2342                 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
2343         };
2344
2345         List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
2346         for (TestIdentifier id : testIds) {
2347             allTests.add(id);
2348         }
2349
2350         List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
2351         activeTests.add(testIds[3]);
2352         activeTests.add(testIds[4]);
2353         activeTests.add(testIds[5]);
2354
2355         String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
2356
2357         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
2358         Set<String> includes = new HashSet<>();
2359         includes.add("dEQP-GLES3.pick_me#*");
2360         addFilterFileForOption(deqpTest, includes, "include-filter-file");
2361         testFiltering(deqpTest, expectedTrie, activeTests);
2362     }
2363
2364     public void testMissingIncludeFilterFile() throws Exception {
2365         final TestIdentifier[] testIds = {
2366                 new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
2367                 new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
2368                 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
2369         };
2370
2371         List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
2372         for (TestIdentifier id : testIds) {
2373             allTests.add(id);
2374         }
2375
2376         String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
2377
2378         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
2379         OptionSetter setter = new OptionSetter(deqpTest);
2380         setter.setOptionValue("include-filter-file", "not-a-file.txt");
2381         try {
2382             testFiltering(deqpTest, expectedTrie, allTests);
2383             fail("Test execution should have aborted with exception.");
2384         } catch (RuntimeException e) {
2385         }
2386     }
2387
2388     public void testExcludeFilterFile() throws Exception {
2389         final TestIdentifier[] testIds = {
2390                 new TestIdentifier("dEQP-GLES3.missing", "no"),
2391                 new TestIdentifier("dEQP-GLES3.missing", "nope"),
2392                 new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
2393                 new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
2394                 new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
2395                 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
2396         };
2397
2398         List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
2399         for (TestIdentifier id : testIds) {
2400             allTests.add(id);
2401         }
2402
2403         List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
2404         activeTests.add(testIds[3]);
2405         activeTests.add(testIds[4]);
2406         activeTests.add(testIds[5]);
2407
2408         String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
2409
2410         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
2411         Set<String> excludes = new HashSet<>();
2412         excludes.add("dEQP-GLES3.missing#*");
2413         addFilterFileForOption(deqpTest, excludes, "exclude-filter-file");
2414         testFiltering(deqpTest, expectedTrie, activeTests);
2415     }
2416
2417     public void testFilterComboWithFiles() throws Exception {
2418         final TestIdentifier[] testIds = {
2419                 new TestIdentifier("dEQP-GLES3.group1", "footah"),
2420                 new TestIdentifier("dEQP-GLES3.group1", "foo"),
2421                 new TestIdentifier("dEQP-GLES3.group1", "nope"),
2422                 new TestIdentifier("dEQP-GLES3.group1", "nonotwant"),
2423                 new TestIdentifier("dEQP-GLES3.group2", "foo"),
2424                 new TestIdentifier("dEQP-GLES3.group2", "yes"),
2425                 new TestIdentifier("dEQP-GLES3.group2", "thoushallnotpass"),
2426         };
2427
2428         List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
2429         for (TestIdentifier id : testIds) {
2430             allTests.add(id);
2431         }
2432
2433         List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
2434         activeTests.add(testIds[0]);
2435         activeTests.add(testIds[5]);
2436
2437         String expectedTrie = "{dEQP-GLES3{group1{footah}group2{yes}}}";
2438
2439         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
2440
2441         Set<String> includes = new HashSet<>();
2442         includes.add("dEQP-GLES3.group2#*");
2443         deqpTest.addAllIncludeFilters(includes);
2444
2445         Set<String> fileIncludes = new HashSet<>();
2446         fileIncludes.add("dEQP-GLES3.group1#no*");
2447         fileIncludes.add("dEQP-GLES3.group1#foo*");
2448         addFilterFileForOption(deqpTest, fileIncludes, "include-filter-file");
2449
2450         Set<String> fileExcludes = new HashSet<>();
2451         fileExcludes.add("*foo");
2452         fileExcludes.add("*thoushallnotpass");
2453         addFilterFileForOption(deqpTest, fileExcludes, "exclude-filter-file");
2454
2455         deqpTest.addExcludeFilter("dEQP-GLES3.group1#no*");
2456
2457         testFiltering(deqpTest, expectedTrie, activeTests);
2458     }
2459
2460     public void testDotToHashConversionInFilters() throws Exception {
2461         final TestIdentifier[] testIds = {
2462                 new TestIdentifier("dEQP-GLES3.missing", "no"),
2463                 new TestIdentifier("dEQP-GLES3.pick_me", "donotwant"),
2464                 new TestIdentifier("dEQP-GLES3.pick_me", "yes")
2465         };
2466
2467         List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
2468         for (TestIdentifier id : testIds) {
2469             allTests.add(id);
2470         }
2471
2472         List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
2473         activeTests.add(testIds[2]);
2474
2475         String expectedTrie = "{dEQP-GLES3{pick_me{yes}}}";
2476
2477         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
2478         deqpTest.addIncludeFilter("dEQP-GLES3.pick_me.yes");
2479         testFiltering(deqpTest, expectedTrie, activeTests);
2480     }
2481 }