android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java

   1 /*
   2  * Copyright (C) 2014 The Android Open Source Project
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package com.drawelements.deqp.runner;
  17
  18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
  19 import com.android.compatibility.common.util.AbiUtils;
  20 import com.android.ddmlib.IDevice;
  21 import com.android.ddmlib.IShellOutputReceiver;
  22 import com.android.ddmlib.ShellCommandUnresponsiveException;
  23 import com.android.ddmlib.testrunner.TestIdentifier;
  24 import com.android.tradefed.build.IFolderBuildInfo;
  25 import com.android.tradefed.config.ConfigurationException;
  26 import com.android.tradefed.config.OptionSetter;
  27 import com.android.tradefed.device.DeviceNotAvailableException;
  28 import com.android.tradefed.device.ITestDevice;
  29 import com.android.tradefed.result.ITestInvocationListener;
  30 import com.android.tradefed.testtype.Abi;
  31 import com.android.tradefed.testtype.IAbi;
  32 import com.android.tradefed.testtype.IRemoteTest;
  33 import com.android.tradefed.testtype.IRuntimeHintProvider;
  34 import com.android.tradefed.util.IRunUtil;
  35 import com.android.tradefed.util.RunInterruptedException;
  36
  37 import junit.framework.TestCase;
  38
  39 import org.easymock.EasyMock;
  40 import org.easymock.IAnswer;
  41 import org.easymock.IMocksControl;
  42
  43 import java.io.File;
  44 import java.io.FileNotFoundException;
  45 import java.io.StringReader;
  46 import java.io.StringWriter;
  47 import java.util.ArrayList;
  48 import java.util.Collection;
  49 import java.util.HashMap;
  50 import java.util.HashSet;
  51 import java.util.List;
  52 import java.util.Map;
  53 import java.util.Set;
  54 import java.util.concurrent.TimeUnit;
  55
  56 /**
  57  * Unit tests for {@link DeqpTestRunner}.
  58  */
  59 public class DeqpTestRunnerTest extends TestCase {
  60     private static final String NAME = "dEQP-GLES3";
  61     private static final IAbi ABI = new Abi("armeabi-v7a", "32");
  62     private static final String CASE_LIST_FILE_NAME = "/sdcard/dEQP-TestCaseList.txt";
  63     private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa";
  64     private static final String INSTRUMENTATION_NAME =
  65             "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
  66     private static final String QUERY_INSTRUMENTATION_NAME =
  67             "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation";
  68     private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk";
  69     private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp";
  70     private static final String ONLY_LANDSCAPE_FEATURES =
  71             "feature:"+DeqpTestRunner.FEATURE_LANDSCAPE;
  72     private static final String ALL_FEATURES =
  73             ONLY_LANDSCAPE_FEATURES + "\nfeature:"+DeqpTestRunner.FEATURE_PORTRAIT;
  74     private static List<Map<String,String>> DEFAULT_INSTANCE_ARGS;
  75
  76     static {
  77         DEFAULT_INSTANCE_ARGS = new ArrayList<>(1);
  78         DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>());
  79         DEFAULT_INSTANCE_ARGS.iterator().next().put("glconfig", "rgba8888d24s8");
  80         DEFAULT_INSTANCE_ARGS.iterator().next().put("rotation", "unspecified");
  81         DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window");
  82     }
  83
  84     private static class StubRecovery implements DeqpTestRunner.IRecovery {
  85         /**
  86          * {@inheritDoc}
  87          */
  88         @Override
  89         public void setSleepProvider(DeqpTestRunner.ISleepProvider sleepProvider) {
  90         }
  91
  92         /**
  93          * {@inheritDoc}
  94          */
  95         @Override
  96         public void setDevice(ITestDevice device) {
  97         }
  98
  99         /**
 100          * {@inheritDoc}
 101          */
 102         @Override
 103         public void onExecutionProgressed() {
 104         }
 105
 106         /**
 107          * {@inheritDoc}
 108          */
 109         @Override
 110         public void recoverConnectionRefused() throws DeviceNotAvailableException {
 111         }
 112
 113         /**
 114          * {@inheritDoc}
 115          */
 116         @Override
 117         public void recoverComLinkKilled() throws DeviceNotAvailableException {
 118         }
 119     };
 120
 121     public static class BuildHelperMock extends CompatibilityBuildHelper {
 122         public BuildHelperMock(IFolderBuildInfo buildInfo) {
 123             super(buildInfo);
 124         }
 125         @Override
 126         public File getTestsDir() throws FileNotFoundException {
 127             return new File("logs");
 128         }
 129     }
 130
 131
 132     /**
 133      * {@inheritDoc}
 134      */
 135     @Override
 136     protected void setUp() throws Exception {
 137         super.setUp();
 138     }
 139
 140     private static DeqpTestRunner buildGlesTestRunner(int majorVersion,
 141                                                       int minorVersion,
 142                                                       Collection<TestIdentifier> tests) throws ConfigurationException, FileNotFoundException {
 143         StringWriter testlist = new StringWriter();
 144         for (TestIdentifier test : tests) {
 145             testlist.write(test.getClassName() + "." + test.getTestName() + "\n");
 146         }
 147         return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString());
 148     }
 149
 150     private static CompatibilityBuildHelper getMockBuildHelper() {
 151         IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class);
 152         EasyMock.replay(mockIFolderBuildInfo);
 153         return new BuildHelperMock(mockIFolderBuildInfo);
 154     }
 155
 156     private static DeqpTestRunner buildGlesTestRunner(int majorVersion,
 157                                                       int minorVersion,
 158                                                       String testlist) throws ConfigurationException, FileNotFoundException {
 159         DeqpTestRunner runner = new DeqpTestRunner();
 160         OptionSetter setter = new OptionSetter(runner);
 161
 162         String deqpPackage = "dEQP-GLES" + Integer.toString(majorVersion)
 163                 + (minorVersion > 0 ? Integer.toString(minorVersion) : "");
 164
 165         setter.setOptionValue("deqp-package", deqpPackage);
 166         setter.setOptionValue("deqp-gl-config-name", "rgba8888d24s8");
 167         setter.setOptionValue("deqp-caselist-file", "dummyfile.txt");
 168         setter.setOptionValue("deqp-screen-rotation", "unspecified");
 169         setter.setOptionValue("deqp-surface-type", "window");
 170
 171         runner.setCaselistReader(new StringReader(testlist));
 172         runner.setAbi(ABI);
 173         runner.setBuildHelper(getMockBuildHelper());
 174
 175         return runner;
 176     }
 177
 178     private static String getTestId(DeqpTestRunner runner) {
 179         return AbiUtils.createId(ABI.getName(), runner.getPackageName());
 180     }
 181
 182     /**
 183      * Test version of OpenGL ES.
 184      */
 185     private void testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion) throws Exception {
 186         final TestIdentifier testId = new TestIdentifier("dEQP-GLES"
 187                 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
 188                 + ".info", "version");
 189
 190         final String testPath = "dEQP-GLES"
 191                 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
 192                 +".info.version";
 193
 194         final String testTrie = "{dEQP-GLES"
 195                 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
 196                 + "{info{version}}}";
 197
 198         final String resultCode = "Pass";
 199
 200         /* MultiLineReceiver expects "\r\n" line ending. */
 201         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
 202                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 203                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
 204                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 205                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
 206                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 207                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
 208                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 209                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
 210                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 211                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
 212                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 213                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
 214                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 215                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
 216                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
 217                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 218                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n"
 219                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n"
 220                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
 221                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 222                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
 223                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 224                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
 225                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 226                 + "INSTRUMENTATION_CODE: 0\r\n";
 227
 228         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
 229         ITestInvocationListener mockListener
 230                 = EasyMock.createStrictMock(ITestInvocationListener.class);
 231         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
 232         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
 233         tests.add(testId);
 234
 235         DeqpTestRunner deqpTest = buildGlesTestRunner(requiredMajorVersion, requiredMinorVersion, tests);
 236
 237         int version = (majorVersion << 16) | minorVersion;
 238         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
 239             .andReturn(Integer.toString(version)).atLeastOnce();
 240
 241         if (majorVersion > requiredMajorVersion
 242                 || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion)) {
 243
 244             EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
 245                     .andReturn("").once();
 246             EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
 247                     EasyMock.eq(true),
 248                     EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
 249                     .andReturn(null).once();
 250
 251             expectRenderConfigQuery(mockDevice, requiredMajorVersion,
 252                     requiredMinorVersion);
 253
 254             String commandLine = String.format(
 255                     "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
 256                     + "--deqp-screen-rotation=unspecified "
 257                     + "--deqp-surface-type=window "
 258                     + "--deqp-log-images=disable "
 259                     + "--deqp-watchdog=enable",
 260                     CASE_LIST_FILE_NAME);
 261
 262             runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
 263                     output);
 264
 265             EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
 266                     .andReturn("").once();
 267         }
 268
 269         mockListener.testRunStarted(getTestId(deqpTest), 1);
 270         EasyMock.expectLastCall().once();
 271
 272         mockListener.testStarted(EasyMock.eq(testId));
 273         EasyMock.expectLastCall().once();
 274
 275         mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
 276         EasyMock.expectLastCall().once();
 277
 278         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
 279         EasyMock.expectLastCall().once();
 280
 281         EasyMock.replay(mockDevice, mockIDevice);
 282         EasyMock.replay(mockListener);
 283
 284         deqpTest.setDevice(mockDevice);
 285         deqpTest.run(mockListener);
 286
 287         EasyMock.verify(mockListener);
 288         EasyMock.verify(mockDevice, mockIDevice);
 289     }
 290
 291     private void expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion,
 292             int minorVersion) throws Exception {
 293         expectRenderConfigQuery(mockDevice,
 294                 String.format("--deqp-gl-config-name=rgba8888d24s8 "
 295                 + "--deqp-screen-rotation=unspecified "
 296                 + "--deqp-surface-type=window "
 297                 + "--deqp-gl-major-version=%d "
 298                 + "--deqp-gl-minor-version=%d", majorVersion, minorVersion));
 299     }
 300
 301     private void expectRenderConfigQuery(ITestDevice mockDevice, String commandLine)
 302             throws Exception {
 303         expectRenderConfigQueryAndReturn(mockDevice, commandLine, "Yes");
 304     }
 305
 306     private void expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine,
 307             String output) throws Exception {
 308         final String queryOutput = "INSTRUMENTATION_RESULT: Supported=" + output + "\r\n"
 309                 + "INSTRUMENTATION_CODE: 0\r\n";
 310         final String command = String.format(
 311                 "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine "
 312                     + "\"%s\" %s",
 313                 AbiUtils.createAbiFlag(ABI.getName()), commandLine,
 314                 QUERY_INSTRUMENTATION_NAME);
 315
 316         mockDevice.executeShellCommand(EasyMock.eq(command),
 317                 EasyMock.<IShellOutputReceiver>notNull());
 318
 319         EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
 320             @Override
 321             public Object answer() {
 322                 IShellOutputReceiver receiver
 323                         = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
 324
 325                 receiver.addOutput(queryOutput.getBytes(), 0, queryOutput.length());
 326                 receiver.flush();
 327
 328                 return null;
 329             }
 330         });
 331     }
 332
 333     /**
 334      * Test that result code produces correctly pass or fail.
 335      */
 336     private void testResultCode(final String resultCode, boolean pass) throws Exception {
 337         final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.info", "version");
 338         final String testPath = "dEQP-GLES3.info.version";
 339         final String testTrie = "{dEQP-GLES3{info{version}}}";
 340
 341         /* MultiLineReceiver expects "\r\n" line ending. */
 342         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
 343                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 344                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
 345                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 346                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
 347                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 348                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
 349                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 350                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
 351                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 352                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
 353                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 354                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
 355                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 356                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
 357                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
 358                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 359                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n"
 360                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n"
 361                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
 362                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 363                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
 364                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 365                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
 366                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 367                 + "INSTRUMENTATION_CODE: 0\r\n";
 368
 369         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
 370         ITestInvocationListener mockListener
 371                 = EasyMock.createStrictMock(ITestInvocationListener.class);
 372         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
 373
 374         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
 375         tests.add(testId);
 376
 377         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
 378
 379         int version = 3 << 16;
 380         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
 381                 .andReturn(Integer.toString(version)).atLeastOnce();
 382
 383         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
 384                 .once();
 385
 386         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
 387                 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
 388                 .andReturn(null).once();
 389
 390         expectRenderConfigQuery(mockDevice, 3, 0);
 391
 392         String commandLine = String.format(
 393                 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
 394                 + "--deqp-screen-rotation=unspecified "
 395                 + "--deqp-surface-type=window "
 396                 + "--deqp-log-images=disable "
 397                 + "--deqp-watchdog=enable",
 398                 CASE_LIST_FILE_NAME);
 399
 400         runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
 401
 402         mockListener.testRunStarted(getTestId(deqpTest), 1);
 403         EasyMock.expectLastCall().once();
 404
 405         mockListener.testStarted(EasyMock.eq(testId));
 406         EasyMock.expectLastCall().once();
 407
 408         if (!pass) {
 409             mockListener.testFailed(testId,
 410                     "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n"
 411                     + resultCode + ": Detail" + resultCode);
 412
 413             EasyMock.expectLastCall().once();
 414         }
 415
 416         mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
 417         EasyMock.expectLastCall().once();
 418
 419         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
 420         EasyMock.expectLastCall().once();
 421
 422         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
 423                 .once();
 424
 425         EasyMock.replay(mockDevice, mockIDevice);
 426         EasyMock.replay(mockListener);
 427
 428         deqpTest.setDevice(mockDevice);
 429         deqpTest.run(mockListener);
 430
 431         EasyMock.verify(mockListener);
 432         EasyMock.verify(mockDevice, mockIDevice);
 433     }
 434
 435     /**
 436      * Test running multiple test cases.
 437      */
 438     public void testRun_multipleTests() throws Exception {
 439         /* MultiLineReceiver expects "\r\n" line ending. */
 440         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
 441                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 442                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
 443                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 444                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
 445                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 446                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
 447                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 448                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
 449                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 450                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
 451                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 452                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
 453                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 454                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
 455                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n"
 456                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 457                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
 458                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
 459                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
 460                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 461                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
 462                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 463                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
 464                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n"
 465                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 466                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
 467                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
 468                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
 469                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 470                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
 471                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 472                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
 473                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n"
 474                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 475                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
 476                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
 477                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
 478                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 479                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
 480                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 481                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
 482                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n"
 483                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 484                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
 485                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
 486                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
 487                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 488                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
 489                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 490                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
 491                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n"
 492                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 493                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
 494                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
 495                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
 496                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 497                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
 498                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 499                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
 500                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n"
 501                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 502                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
 503                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
 504                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
 505                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 506                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
 507                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 508                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
 509                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 510                 + "INSTRUMENTATION_CODE: 0\r\n";
 511
 512         final TestIdentifier[] testIds = {
 513                 new TestIdentifier("dEQP-GLES3.info", "vendor"),
 514                 new TestIdentifier("dEQP-GLES3.info", "renderer"),
 515                 new TestIdentifier("dEQP-GLES3.info", "version"),
 516                 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
 517                 new TestIdentifier("dEQP-GLES3.info", "extensions"),
 518                 new TestIdentifier("dEQP-GLES3.info", "render_target")
 519         };
 520
 521         final String[] testPaths = {
 522                 "dEQP-GLES3.info.vendor",
 523                 "dEQP-GLES3.info.renderer",
 524                 "dEQP-GLES3.info.version",
 525                 "dEQP-GLES3.info.shading_language_version",
 526                 "dEQP-GLES3.info.extensions",
 527                 "dEQP-GLES3.info.render_target"
 528         };
 529
 530         final String testTrie
 531                 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}";
 532
 533         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
 534         ITestInvocationListener mockListener
 535                 = EasyMock.createStrictMock(ITestInvocationListener.class);
 536         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
 537
 538         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
 539
 540         for (TestIdentifier id : testIds) {
 541             tests.add(id);
 542         }
 543
 544         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
 545
 546         int version = 3 << 16;
 547         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
 548                 .andReturn(Integer.toString(version)).atLeastOnce();
 549
 550         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
 551                 .once();
 552         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
 553                 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
 554                 .andReturn(null).once();
 555
 556         expectRenderConfigQuery(mockDevice, 3, 0);
 557
 558         String commandLine = String.format(
 559                 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
 560                 + "--deqp-screen-rotation=unspecified "
 561                 + "--deqp-surface-type=window "
 562                 + "--deqp-log-images=disable "
 563                 + "--deqp-watchdog=enable",
 564                 CASE_LIST_FILE_NAME);
 565
 566         runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
 567
 568         mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
 569         EasyMock.expectLastCall().once();
 570
 571         for (int i = 0; i < testPaths.length; i++) {
 572             mockListener.testStarted(EasyMock.eq(testIds[i]));
 573             EasyMock.expectLastCall().once();
 574
 575             mockListener.testEnded(EasyMock.eq(testIds[i]),
 576                     EasyMock.<Map<String, String>>notNull());
 577
 578             EasyMock.expectLastCall().once();
 579         }
 580
 581         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
 582         EasyMock.expectLastCall().once();
 583
 584         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
 585                 .once();
 586
 587         EasyMock.replay(mockDevice, mockIDevice);
 588         EasyMock.replay(mockListener);
 589
 590         deqpTest.setDevice(mockDevice);
 591         deqpTest.run(mockListener);
 592
 593         EasyMock.verify(mockListener);
 594         EasyMock.verify(mockDevice, mockIDevice);
 595     }
 596
 597     static private String buildTestProcessOutput(List<TestIdentifier> tests) {
 598         /* MultiLineReceiver expects "\r\n" line ending. */
 599         final String outputHeader = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
 600                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 601                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
 602                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 603                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
 604                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 605                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
 606                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 607                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
 608                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 609                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
 610                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 611                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
 612                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n";
 613
 614         final String outputEnd = "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
 615                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 616                 + "INSTRUMENTATION_CODE: 0\r\n";
 617
 618         StringWriter output = new StringWriter();
 619         output.write(outputHeader);
 620         for (TestIdentifier test : tests) {
 621             output.write("INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n");
 622             output.write("INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=");
 623             output.write(test.getClassName());
 624             output.write(".");
 625             output.write(test.getTestName());
 626             output.write("\r\n");
 627             output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
 628             output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n");
 629             output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n");
 630             output.write("INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n");
 631             output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
 632             output.write("INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n");
 633             output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
 634         }
 635         output.write(outputEnd);
 636         return output.toString();
 637     }
 638
 639     private void testFiltering(Set<String> includes,
 640                                Set<String> excludes,
 641                                List<TestIdentifier> fullTestList,
 642                                String expectedTrie,
 643                                List<TestIdentifier> expectedTests) throws Exception {
 644
 645         boolean thereAreTests = !expectedTests.isEmpty();
 646         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
 647         ITestInvocationListener mockListener
 648                 = EasyMock.createStrictMock(ITestInvocationListener.class);
 649         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
 650
 651         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, fullTestList);
 652         if (includes != null) {
 653             deqpTest.addAllIncludeFilters(includes);
 654         }
 655         if (excludes != null) {
 656             deqpTest.addAllExcludeFilters(excludes);
 657         }
 658
 659         int version = 3 << 16;
 660         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
 661                 .andReturn(Integer.toString(version)).atLeastOnce();
 662
 663         if (thereAreTests)
 664         {
 665             // only expect to install/uninstall packages if there are any tests
 666             EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
 667                 .once();
 668             EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
 669                 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
 670                 .andReturn(null).once();
 671         }
 672
 673
 674         mockListener.testRunStarted(getTestId(deqpTest), expectedTests.size());
 675         EasyMock.expectLastCall().once();
 676
 677         if (thereAreTests)
 678         {
 679             expectRenderConfigQuery(mockDevice, 3, 0);
 680
 681             String testOut = buildTestProcessOutput(expectedTests);
 682             runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut);
 683
 684             for (int i = 0; i < expectedTests.size(); i++) {
 685                 mockListener.testStarted(EasyMock.eq(expectedTests.get(i)));
 686                 EasyMock.expectLastCall().once();
 687
 688                 mockListener.testEnded(EasyMock.eq(expectedTests.get(i)),
 689                                        EasyMock.<Map<String, String>>notNull());
 690
 691                 EasyMock.expectLastCall().once();
 692             }
 693         }
 694
 695         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
 696         EasyMock.expectLastCall().once();
 697
 698         if (thereAreTests)
 699         {
 700             // package will only be installed if there are tests to run
 701             EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
 702                 .once();
 703         }
 704
 705         EasyMock.replay(mockDevice, mockIDevice);
 706         EasyMock.replay(mockListener);
 707
 708         deqpTest.setDevice(mockDevice);
 709         deqpTest.run(mockListener);
 710
 711         EasyMock.verify(mockListener);
 712         EasyMock.verify(mockDevice, mockIDevice);
 713     }
 714
 715     public void testRun_trivialIncludeFilter() throws Exception {
 716         final TestIdentifier[] testIds = {
 717                 new TestIdentifier("dEQP-GLES3.missing", "no"),
 718                 new TestIdentifier("dEQP-GLES3.missing", "nope"),
 719                 new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
 720                 new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
 721                 new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
 722                 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
 723         };
 724
 725         List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
 726         for (TestIdentifier id : testIds) {
 727             allTests.add(id);
 728         }
 729
 730         List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
 731         activeTests.add(testIds[3]);
 732         activeTests.add(testIds[4]);
 733         activeTests.add(testIds[5]);
 734
 735         String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
 736
 737         Set<String> includes = new HashSet();
 738         includes.add("dEQP-GLES3.pick_me#*");
 739         testFiltering(includes, null, allTests, expectedTrie, activeTests);
 740     }
 741
 742     public void testRun_trivialExcludeFilter() throws Exception {
 743         final TestIdentifier[] testIds = {
 744                 new TestIdentifier("dEQP-GLES3.missing", "no"),
 745                 new TestIdentifier("dEQP-GLES3.missing", "nope"),
 746                 new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
 747                 new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
 748                 new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
 749                 new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
 750         };
 751
 752         List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
 753         for (TestIdentifier id : testIds) {
 754             allTests.add(id);
 755         }
 756
 757         List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
 758         activeTests.add(testIds[3]);
 759         activeTests.add(testIds[4]);
 760         activeTests.add(testIds[5]);
 761
 762         String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
 763
 764         Set<String> excludes = new HashSet();
 765         excludes.add("dEQP-GLES3.missing#*");
 766         testFiltering(null, excludes, allTests, expectedTrie, activeTests);
 767     }
 768
 769     public void testRun_includeAndExcludeFilter() throws Exception {
 770         final TestIdentifier[] testIds = {
 771                 new TestIdentifier("dEQP-GLES3.group1", "foo"),
 772                 new TestIdentifier("dEQP-GLES3.group1", "nope"),
 773                 new TestIdentifier("dEQP-GLES3.group1", "donotwant"),
 774                 new TestIdentifier("dEQP-GLES3.group2", "foo"),
 775                 new TestIdentifier("dEQP-GLES3.group2", "yes"),
 776                 new TestIdentifier("dEQP-GLES3.group2", "thoushallnotpass"),
 777         };
 778
 779         List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
 780         for (TestIdentifier id : testIds) {
 781             allTests.add(id);
 782         }
 783
 784         List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
 785         activeTests.add(testIds[4]);
 786
 787         String expectedTrie = "{dEQP-GLES3{group2{yes}}}";
 788
 789         Set<String> includes = new HashSet();
 790         includes.add("dEQP-GLES3.group2#*");
 791
 792         Set<String> excludes = new HashSet();
 793         excludes.add("*foo");
 794         excludes.add("*thoushallnotpass");
 795         testFiltering(includes, excludes, allTests, expectedTrie, activeTests);
 796     }
 797
 798     public void testRun_includeAll() throws Exception {
 799         final TestIdentifier[] testIds = {
 800                 new TestIdentifier("dEQP-GLES3.group1", "mememe"),
 801                 new TestIdentifier("dEQP-GLES3.group1", "yeah"),
 802                 new TestIdentifier("dEQP-GLES3.group1", "takeitall"),
 803                 new TestIdentifier("dEQP-GLES3.group2", "jeba"),
 804                 new TestIdentifier("dEQP-GLES3.group2", "yes"),
 805                 new TestIdentifier("dEQP-GLES3.group2", "granted"),
 806         };
 807
 808         List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
 809         for (TestIdentifier id : testIds) {
 810             allTests.add(id);
 811         }
 812
 813         String expectedTrie = "{dEQP-GLES3{group1{mememe,yeah,takeitall},group2{jeba,yes,granted}}}";
 814
 815         Set<String> includes = new HashSet();
 816         includes.add("*");
 817
 818         testFiltering(includes, null, allTests, expectedTrie, allTests);
 819     }
 820
 821     public void testRun_excludeAll() throws Exception {
 822         final TestIdentifier[] testIds = {
 823                 new TestIdentifier("dEQP-GLES3.group1", "no"),
 824                 new TestIdentifier("dEQP-GLES3.group1", "nope"),
 825                 new TestIdentifier("dEQP-GLES3.group1", "nottoday"),
 826                 new TestIdentifier("dEQP-GLES3.group2", "banned"),
 827                 new TestIdentifier("dEQP-GLES3.group2", "notrecognized"),
 828                 new TestIdentifier("dEQP-GLES3.group2", "-2"),
 829         };
 830
 831         List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
 832         for (TestIdentifier id : testIds) {
 833             allTests.add(id);
 834         }
 835
 836         String expectedTrie = "";
 837
 838         Set<String> excludes = new HashSet();
 839         excludes.add("*");
 840
 841         testFiltering(null, excludes, allTests, expectedTrie, new ArrayList<TestIdentifier>());
 842     }
 843
 844     /**
 845      * Test running a unexecutable test.
 846      */
 847     public void testRun_unexecutableTests() throws Exception {
 848         final String instrumentationAnswerNoExecs =
 849                 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
 850                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 851                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
 852                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 853                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
 854                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 855                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
 856                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 857                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
 858                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
 859                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
 860                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 861                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
 862                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 863                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
 864                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
 865                 + "INSTRUMENTATION_CODE: 0\r\n";
 866
 867         final TestIdentifier[] testIds = {
 868                 new TestIdentifier("dEQP-GLES3.missing", "no"),
 869                 new TestIdentifier("dEQP-GLES3.missing", "nope"),
 870                 new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
 871         };
 872
 873         final String[] testPaths = {
 874                 "dEQP-GLES3.missing.no",
 875                 "dEQP-GLES3.missing.nope",
 876                 "dEQP-GLES3.missing.donotwant",
 877         };
 878
 879         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
 880         ITestInvocationListener mockListener
 881                 = EasyMock.createStrictMock(ITestInvocationListener.class);
 882         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
 883
 884         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
 885
 886         for (TestIdentifier id : testIds) {
 887             tests.add(id);
 888         }
 889
 890         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
 891
 892         int version = 3 << 16;
 893         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
 894                 .andReturn(Integer.toString(version)).atLeastOnce();
 895
 896         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
 897                 .once();
 898         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
 899                 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
 900                 .andReturn(null).once();
 901
 902         expectRenderConfigQuery(mockDevice, 3, 0);
 903
 904         String commandLine = String.format(
 905                 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
 906                 + "--deqp-screen-rotation=unspecified "
 907                 + "--deqp-surface-type=window "
 908                 + "--deqp-log-images=disable "
 909                 + "--deqp-watchdog=enable",
 910                 CASE_LIST_FILE_NAME);
 911
 912         // first try
 913         runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
 914                 "{dEQP-GLES3{missing{no,nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs);
 915
 916         // splitting begins
 917         runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
 918                 "{dEQP-GLES3{missing{no}}}", commandLine, instrumentationAnswerNoExecs);
 919         runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
 920                 "{dEQP-GLES3{missing{nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs);
 921         runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
 922                 "{dEQP-GLES3{missing{nope}}}", commandLine, instrumentationAnswerNoExecs);
 923         runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
 924                 "{dEQP-GLES3{missing{donotwant}}}", commandLine, instrumentationAnswerNoExecs);
 925
 926         mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
 927         EasyMock.expectLastCall().once();
 928
 929         for (int i = 0; i < testPaths.length; i++) {
 930             mockListener.testStarted(EasyMock.eq(testIds[i]));
 931             EasyMock.expectLastCall().once();
 932
 933             mockListener.testFailed(EasyMock.eq(testIds[i]),
 934                     EasyMock.eq("=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n"
 935                     + "Abort: Test cannot be executed"));
 936             EasyMock.expectLastCall().once();
 937
 938             mockListener.testEnded(EasyMock.eq(testIds[i]),
 939                     EasyMock.<Map<String, String>>notNull());
 940             EasyMock.expectLastCall().once();
 941         }
 942
 943         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
 944         EasyMock.expectLastCall().once();
 945
 946         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
 947                 .once();
 948
 949         EasyMock.replay(mockDevice, mockIDevice);
 950         EasyMock.replay(mockListener);
 951
 952         deqpTest.setDevice(mockDevice);
 953         deqpTest.run(mockListener);
 954
 955         EasyMock.verify(mockListener);
 956         EasyMock.verify(mockDevice, mockIDevice);
 957     }
 958
 959     /**
 960      * Test that test are left unexecuted if pm list query fails
 961      */
 962     public void testRun_queryPmListFailure()
 963             throws Exception {
 964         final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
 965
 966         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
 967         ITestInvocationListener mockListener
 968                 = EasyMock.createStrictMock(ITestInvocationListener.class);
 969         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
 970         tests.add(testId);
 971
 972         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
 973         OptionSetter setter = new OptionSetter(deqpTest);
 974         // Note: If the rotation is the default unspecified, features are not queried at all
 975         setter.setOptionValue("deqp-screen-rotation", "90");
 976
 977         deqpTest.setDevice(mockDevice);
 978
 979         int version = 3 << 16;
 980         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
 981                 .andReturn(Integer.toString(version)).atLeastOnce();
 982
 983         EasyMock.expect(mockDevice.executeShellCommand("pm list features"))
 984                 .andReturn("not a valid format");
 985
 986         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
 987             andReturn("").once();
 988
 989         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
 990                 EasyMock.eq(true),
 991                 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
 992                 .once();
 993
 994         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
 995                 .andReturn("").once();
 996
 997         mockListener.testRunStarted(getTestId(deqpTest), 1);
 998         EasyMock.expectLastCall().once();
 999
1000         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1001         EasyMock.expectLastCall().once();
1002
1003         EasyMock.replay(mockDevice);
1004         EasyMock.replay(mockListener);
1005         deqpTest.run(mockListener);
1006         EasyMock.verify(mockListener);
1007         EasyMock.verify(mockDevice);
1008     }
1009
1010     /**
1011      * Test that test are left unexecuted if renderablity query fails
1012      */
1013     public void testRun_queryRenderabilityFailure()
1014             throws Exception {
1015         final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
1016
1017         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1018         ITestInvocationListener mockListener
1019                 = EasyMock.createStrictMock(ITestInvocationListener.class);
1020
1021         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1022         tests.add(testId);
1023
1024         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
1025
1026         deqpTest.setDevice(mockDevice);
1027
1028         int version = 3 << 16;
1029         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1030                 .andReturn(Integer.toString(version)).atLeastOnce();
1031
1032         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1033             andReturn("").once();
1034
1035         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1036                 EasyMock.eq(true),
1037                 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1038                 .once();
1039
1040         expectRenderConfigQueryAndReturn(mockDevice,
1041                 "--deqp-gl-config-name=rgba8888d24s8 "
1042                 + "--deqp-screen-rotation=unspecified "
1043                 + "--deqp-surface-type=window "
1044                 + "--deqp-gl-major-version=3 "
1045                 + "--deqp-gl-minor-version=0", "Maybe?");
1046
1047         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
1048                 .andReturn("").once();
1049
1050         mockListener.testRunStarted(getTestId(deqpTest), 1);
1051         EasyMock.expectLastCall().once();
1052
1053         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1054         EasyMock.expectLastCall().once();
1055
1056         EasyMock.replay(mockDevice);
1057         EasyMock.replay(mockListener);
1058         deqpTest.run(mockListener);
1059         EasyMock.verify(mockListener);
1060         EasyMock.verify(mockDevice);
1061     }
1062
1063     /**
1064      * Test that orientation is supplied to runner correctly
1065      */
1066     private void testOrientation(final String rotation, final String featureString)
1067             throws Exception {
1068         final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.orientation", "test");
1069         final String testPath = "dEQP-GLES3.orientation.test";
1070         final String testTrie = "{dEQP-GLES3{orientation{test}}}";
1071         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
1072                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1073                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
1074                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1075                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
1076                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1077                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
1078                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1079                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
1080                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1081                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
1082                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1083                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
1084                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1085                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
1086                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
1087                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1088                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
1089                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
1090                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
1091                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1092                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
1093                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1094                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
1095                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1096                 + "INSTRUMENTATION_CODE: 0\r\n";
1097
1098         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1099         ITestInvocationListener mockListener
1100                 = EasyMock.createStrictMock(ITestInvocationListener.class);
1101         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
1102
1103         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1104         tests.add(testId);
1105
1106         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
1107         OptionSetter setter = new OptionSetter(deqpTest);
1108         setter.setOptionValue("deqp-screen-rotation", rotation);
1109
1110         deqpTest.setDevice(mockDevice);
1111
1112         int version = 3 << 16;
1113         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1114                 .andReturn(Integer.toString(version)).atLeastOnce();
1115
1116         if (!rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED)) {
1117             EasyMock.expect(mockDevice.executeShellCommand("pm list features"))
1118                     .andReturn(featureString);
1119         }
1120
1121         final boolean isPortraitOrientation =
1122                 rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_PORTRAIT) ||
1123                 rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT);
1124         final boolean isLandscapeOrientation =
1125                 rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_LANDSCAPE) ||
1126                 rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE);
1127         final boolean executable =
1128                 rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED) ||
1129                 (isPortraitOrientation &&
1130                 featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) ||
1131                 (isLandscapeOrientation &&
1132                 featureString.contains(DeqpTestRunner.FEATURE_LANDSCAPE));
1133
1134         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1135             andReturn("").once();
1136
1137         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1138                 EasyMock.eq(true),
1139                 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1140                 .once();
1141
1142         if (executable) {
1143             expectRenderConfigQuery(mockDevice, String.format(
1144                     "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=%s "
1145                     + "--deqp-surface-type=window --deqp-gl-major-version=3 "
1146                     + "--deqp-gl-minor-version=0", rotation));
1147
1148             String commandLine = String.format(
1149                     "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
1150                     + "--deqp-screen-rotation=%s "
1151                     + "--deqp-surface-type=window "
1152                     + "--deqp-log-images=disable "
1153                     + "--deqp-watchdog=enable",
1154                     CASE_LIST_FILE_NAME, rotation);
1155
1156             runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
1157                     output);
1158         }
1159
1160         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
1161                 .andReturn("").once();
1162
1163         mockListener.testRunStarted(getTestId(deqpTest), 1);
1164         EasyMock.expectLastCall().once();
1165
1166         mockListener.testStarted(EasyMock.eq(testId));
1167         EasyMock.expectLastCall().once();
1168
1169         mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
1170         EasyMock.expectLastCall().once();
1171
1172         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1173         EasyMock.expectLastCall().once();
1174
1175         EasyMock.replay(mockDevice, mockIDevice);
1176         EasyMock.replay(mockListener);
1177         deqpTest.run(mockListener);
1178         EasyMock.verify(mockListener);
1179         EasyMock.verify(mockDevice, mockIDevice);
1180     }
1181
1182     /**
1183      * Test OpeGL ES3 tests on device with OpenGL ES2.
1184      */
1185     public void testRun_require30DeviceVersion20() throws Exception {
1186         testGlesVersion(3, 0, 2, 0);
1187     }
1188
1189     /**
1190      * Test OpeGL ES3.1 tests on device with OpenGL ES2.
1191      */
1192     public void testRun_require31DeviceVersion20() throws Exception {
1193         testGlesVersion(3, 1, 2, 0);
1194     }
1195
1196     /**
1197      * Test OpeGL ES3 tests on device with OpenGL ES3.
1198      */
1199     public void testRun_require30DeviceVersion30() throws Exception {
1200         testGlesVersion(3, 0, 3, 0);
1201     }
1202
1203     /**
1204      * Test OpeGL ES3.1 tests on device with OpenGL ES3.
1205      */
1206     public void testRun_require31DeviceVersion30() throws Exception {
1207         testGlesVersion(3, 1, 3, 0);
1208     }
1209
1210     /**
1211      * Test OpeGL ES3 tests on device with OpenGL ES3.1.
1212      */
1213     public void testRun_require30DeviceVersion31() throws Exception {
1214         testGlesVersion(3, 0, 3, 1);
1215     }
1216
1217     /**
1218      * Test OpeGL ES3.1 tests on device with OpenGL ES3.1.
1219      */
1220     public void testRun_require31DeviceVersion31() throws Exception {
1221         testGlesVersion(3, 1, 3, 1);
1222     }
1223
1224     /**
1225      * Test dEQP Pass result code.
1226      */
1227     public void testRun_resultPass() throws Exception {
1228         testResultCode("Pass", true);
1229     }
1230
1231     /**
1232      * Test dEQP Fail result code.
1233      */
1234     public void testRun_resultFail() throws Exception {
1235         testResultCode("Fail", false);
1236     }
1237
1238     /**
1239      * Test dEQP NotSupported result code.
1240      */
1241     public void testRun_resultNotSupported() throws Exception {
1242         testResultCode("NotSupported", true);
1243     }
1244
1245     /**
1246      * Test dEQP QualityWarning result code.
1247      */
1248     public void testRun_resultQualityWarning() throws Exception {
1249         testResultCode("QualityWarning", true);
1250     }
1251
1252     /**
1253      * Test dEQP CompatibilityWarning result code.
1254      */
1255     public void testRun_resultCompatibilityWarning() throws Exception {
1256         testResultCode("CompatibilityWarning", true);
1257     }
1258
1259     /**
1260      * Test dEQP ResourceError result code.
1261      */
1262     public void testRun_resultResourceError() throws Exception {
1263         testResultCode("ResourceError", false);
1264     }
1265
1266     /**
1267      * Test dEQP InternalError result code.
1268      */
1269     public void testRun_resultInternalError() throws Exception {
1270         testResultCode("InternalError", false);
1271     }
1272
1273     /**
1274      * Test dEQP Crash result code.
1275      */
1276     public void testRun_resultCrash() throws Exception {
1277         testResultCode("Crash", false);
1278     }
1279
1280     /**
1281      * Test dEQP Timeout result code.
1282      */
1283     public void testRun_resultTimeout() throws Exception {
1284         testResultCode("Timeout", false);
1285     }
1286     /**
1287      * Test dEQP Orientation
1288      */
1289     public void testRun_orientationLandscape() throws Exception {
1290         testOrientation("90", ALL_FEATURES);
1291     }
1292
1293     /**
1294      * Test dEQP Orientation
1295      */
1296     public void testRun_orientationPortrait() throws Exception {
1297         testOrientation("0", ALL_FEATURES);
1298     }
1299
1300     /**
1301      * Test dEQP Orientation
1302      */
1303     public void testRun_orientationReverseLandscape() throws Exception {
1304         testOrientation("270", ALL_FEATURES);
1305     }
1306
1307     /**
1308      * Test dEQP Orientation
1309      */
1310     public void testRun_orientationReversePortrait() throws Exception {
1311         testOrientation("180", ALL_FEATURES);
1312     }
1313
1314     /**
1315      * Test dEQP Orientation
1316      */
1317     public void testRun_orientationUnspecified() throws Exception {
1318         testOrientation("unspecified", ALL_FEATURES);
1319     }
1320
1321     /**
1322      * Test dEQP Orientation with limited features
1323      */
1324     public void testRun_orientationUnspecifiedLimitedFeatures() throws Exception {
1325         testOrientation("unspecified", ONLY_LANDSCAPE_FEATURES);
1326     }
1327
1328     /**
1329      * Test dEQP Orientation with limited features
1330      */
1331     public void testRun_orientationLandscapeLimitedFeatures() throws Exception {
1332         testOrientation("90", ONLY_LANDSCAPE_FEATURES);
1333     }
1334
1335     /**
1336      * Test dEQP Orientation with limited features
1337      */
1338     public void testRun_orientationPortraitLimitedFeatures() throws Exception {
1339         testOrientation("0", ONLY_LANDSCAPE_FEATURES);
1340     }
1341
1342     /**
1343      * Test dEQP unsupported pixel format
1344      */
1345     public void testRun_unsupportedPixelFormat() throws Exception {
1346         final String pixelFormat = "rgba5658d16m4";
1347         final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.pixelformat", "test");
1348         final String testPath = "dEQP-GLES3.pixelformat.test";
1349         final String testTrie = "{dEQP-GLES3{pixelformat{test}}}";
1350         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
1351                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1352                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
1353                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1354                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
1355                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1356                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
1357                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1358                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
1359                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1360                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
1361                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1362                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
1363                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1364                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
1365                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
1366                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1367                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
1368                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
1369                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
1370                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1371                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
1372                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1373                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
1374                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1375                 + "INSTRUMENTATION_CODE: 0\r\n";
1376
1377         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1378         ITestInvocationListener mockListener
1379                 = EasyMock.createStrictMock(ITestInvocationListener.class);
1380
1381         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1382         tests.add(testId);
1383
1384         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
1385         OptionSetter setter = new OptionSetter(deqpTest);
1386         setter.setOptionValue("deqp-gl-config-name", pixelFormat);
1387
1388         deqpTest.setDevice(mockDevice);
1389
1390         int version = 3 << 16;
1391         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1392                 .andReturn(Integer.toString(version)).atLeastOnce();
1393
1394         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1395             andReturn("").once();
1396
1397         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1398                 EasyMock.eq(true),
1399                 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1400                 .once();
1401
1402         expectRenderConfigQueryAndReturn(mockDevice, String.format(
1403                 "--deqp-gl-config-name=%s --deqp-screen-rotation=unspecified "
1404                 + "--deqp-surface-type=window "
1405                 + "--deqp-gl-major-version=3 "
1406                 + "--deqp-gl-minor-version=0", pixelFormat), "No");
1407
1408         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG)))
1409                 .andReturn("").once();
1410
1411         mockListener.testRunStarted(getTestId(deqpTest), 1);
1412         EasyMock.expectLastCall().once();
1413
1414         mockListener.testStarted(EasyMock.eq(testId));
1415         EasyMock.expectLastCall().once();
1416
1417         mockListener.testEnded(EasyMock.eq(testId), EasyMock.<Map<String, String>>notNull());
1418         EasyMock.expectLastCall().once();
1419
1420         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1421         EasyMock.expectLastCall().once();
1422
1423         EasyMock.replay(mockDevice);
1424         EasyMock.replay(mockListener);
1425         deqpTest.run(mockListener);
1426         EasyMock.verify(mockListener);
1427         EasyMock.verify(mockDevice);
1428     }
1429
1430     public static interface RecoverableTestDevice extends ITestDevice {
1431         public void recoverDevice() throws DeviceNotAvailableException;
1432     }
1433
1434     private static enum RecoveryEvent {
1435         PROGRESS,
1436         FAIL_CONNECTION_REFUSED,
1437         FAIL_LINK_KILLED,
1438     };
1439
1440     private void runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events)
1441             throws DeviceNotAvailableException {
1442         for (RecoveryEvent event : events) {
1443             switch (event) {
1444                 case PROGRESS:
1445                     recovery.onExecutionProgressed();
1446                     break;
1447                 case FAIL_CONNECTION_REFUSED:
1448                     recovery.recoverConnectionRefused();
1449                     break;
1450                 case FAIL_LINK_KILLED:
1451                     recovery.recoverComLinkKilled();
1452                     break;
1453             }
1454         }
1455     }
1456
1457     private void setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider) {
1458         mockSleepProvider.sleep(EasyMock.gt(0));
1459         EasyMock.expectLastCall().once();
1460     }
1461
1462     private void setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice,
1463             DeqpTestRunner.ISleepProvider mockSleepProvider) throws DeviceNotAvailableException {
1464         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1465                 andReturn("root 1234 com.drawelement.deqp").once();
1466
1467         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
1468                 andReturn("").once();
1469
1470         // Recovery checks if kill failed
1471         mockSleepProvider.sleep(EasyMock.gt(0));
1472         EasyMock.expectLastCall().once();
1473         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1474                 andReturn("").once();
1475     }
1476
1477     private void setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice)
1478             throws DeviceNotAvailableException {
1479         mockDevice.recoverDevice();
1480         EasyMock.expectLastCall().once();
1481     }
1482
1483     private void setRecoveryExpectationReboot(RecoverableTestDevice mockDevice)
1484             throws DeviceNotAvailableException {
1485         mockDevice.reboot();
1486         EasyMock.expectLastCall().once();
1487     }
1488
1489     private int setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice,
1490             DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)
1491             throws DeviceNotAvailableException {
1492         switch (numConsecutiveErrors) {
1493             case 0:
1494             case 1:
1495                 setRecoveryExpectationRecovery(mockDevice);
1496                 return 2;
1497             case 2:
1498                 setRecoveryExpectationReboot(mockDevice);
1499                 return 3;
1500             default:
1501                 return 4;
1502         }
1503     }
1504
1505     private int setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice,
1506             DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)
1507             throws DeviceNotAvailableException {
1508         switch (numConsecutiveErrors) {
1509             case 0:
1510                 setRecoveryExpectationWait(mockSleepProvider);
1511                 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider);
1512                 return 1;
1513             case 1:
1514                 setRecoveryExpectationRecovery(mockDevice);
1515                 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider);
1516                 return 2;
1517             case 2:
1518                 setRecoveryExpectationReboot(mockDevice);
1519                 return 3;
1520             default:
1521                 return 4;
1522         }
1523     }
1524
1525     private void setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice,
1526             DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events)
1527             throws DeviceNotAvailableException {
1528         int numConsecutiveErrors = 0;
1529         for (RecoveryEvent event : events) {
1530             switch (event) {
1531                 case PROGRESS:
1532                     numConsecutiveErrors = 0;
1533                     break;
1534                 case FAIL_CONNECTION_REFUSED:
1535                     numConsecutiveErrors = setRecoveryExpectationOfAConnFailure(mockDevice,
1536                             mockSleepProvider, numConsecutiveErrors);
1537                     break;
1538                 case FAIL_LINK_KILLED:
1539                     numConsecutiveErrors = setRecoveryExpectationOfAComKilled(mockDevice,
1540                             mockSleepProvider, numConsecutiveErrors);
1541                     break;
1542             }
1543         }
1544     }
1545
1546     /**
1547      * Test dEQP runner recovery state machine.
1548      */
1549     private void testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern)
1550             throws Exception {
1551         DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery();
1552         IMocksControl orderedControl = EasyMock.createStrictControl();
1553         RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class);
1554         DeqpTestRunner.ISleepProvider mockSleepProvider =
1555                 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class);
1556
1557         setRecoveryExpectationsOfAPattern(mockDevice, mockSleepProvider, pattern);
1558
1559         orderedControl.replay();
1560
1561         recovery.setDevice(mockDevice);
1562         recovery.setSleepProvider(mockSleepProvider);
1563         try {
1564             runRecoveryWithPattern(recovery, pattern);
1565             if (!expectSuccess) {
1566                 fail("Expected DeviceNotAvailableException");
1567             }
1568         } catch (DeviceNotAvailableException ex) {
1569             if (expectSuccess) {
1570                 fail("Did not expect DeviceNotAvailableException");
1571             }
1572         }
1573
1574         orderedControl.verify();
1575     }
1576
1577     // basic patterns
1578
1579     public void testRecovery_NoEvents() throws Exception {
1580         testRecoveryWithPattern(true);
1581     }
1582
1583     public void testRecovery_AllOk() throws Exception {
1584         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, RecoveryEvent.PROGRESS);
1585     }
1586
1587     // conn fail patterns
1588
1589     public void testRecovery_OneConnectionFailureBegin() throws Exception {
1590         testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1591                 RecoveryEvent.PROGRESS);
1592     }
1593
1594     public void testRecovery_TwoConnectionFailuresBegin() throws Exception {
1595         testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1596                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS);
1597     }
1598
1599     public void testRecovery_ThreeConnectionFailuresBegin() throws Exception {
1600         testRecoveryWithPattern(false, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1601                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED);
1602     }
1603
1604     public void testRecovery_OneConnectionFailureMid() throws Exception {
1605         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1606                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS);
1607     }
1608
1609     public void testRecovery_TwoConnectionFailuresMid() throws Exception {
1610         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1611                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1612                 RecoveryEvent.PROGRESS);
1613     }
1614
1615     public void testRecovery_ThreeConnectionFailuresMid() throws Exception {
1616         testRecoveryWithPattern(false, RecoveryEvent.PROGRESS,
1617                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1618                 RecoveryEvent.FAIL_CONNECTION_REFUSED);
1619     }
1620
1621     // link fail patterns
1622
1623     public void testRecovery_OneLinkFailureBegin() throws Exception {
1624         testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
1625                 RecoveryEvent.PROGRESS);
1626     }
1627
1628     public void testRecovery_TwoLinkFailuresBegin() throws Exception {
1629         testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
1630                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
1631     }
1632
1633     public void testRecovery_ThreeLinkFailuresBegin() throws Exception {
1634         testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
1635                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1636                 RecoveryEvent.PROGRESS);
1637     }
1638
1639     public void testRecovery_FourLinkFailuresBegin() throws Exception {
1640         testRecoveryWithPattern(false, RecoveryEvent.FAIL_LINK_KILLED,
1641                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1642                 RecoveryEvent.FAIL_LINK_KILLED);
1643     }
1644
1645     public void testRecovery_OneLinkFailureMid() throws Exception {
1646         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1647                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
1648     }
1649
1650     public void testRecovery_TwoLinkFailuresMid() throws Exception {
1651         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1652                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1653                 RecoveryEvent.PROGRESS);
1654     }
1655
1656     public void testRecovery_ThreeLinkFailuresMid() throws Exception {
1657         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
1658                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1659                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
1660     }
1661
1662     public void testRecovery_FourLinkFailuresMid() throws Exception {
1663         testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
1664                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
1665                 RecoveryEvent.FAIL_LINK_KILLED);
1666     }
1667
1668     // mixed patterns
1669
1670     public void testRecovery_MixedFailuresProgressBetween() throws Exception {
1671         testRecoveryWithPattern(true,
1672                 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
1673                 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1674                 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
1675                 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED,
1676                 RecoveryEvent.PROGRESS);
1677     }
1678
1679     public void testRecovery_MixedFailuresNoProgressBetween() throws Exception {
1680         testRecoveryWithPattern(true,
1681                 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
1682                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_LINK_KILLED,
1683                 RecoveryEvent.PROGRESS);
1684     }
1685
1686     /**
1687      * Test recovery if process cannot be killed
1688      */
1689     public void testRecovery_unkillableProcess () throws Exception {
1690         DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery();
1691         IMocksControl orderedControl = EasyMock.createStrictControl();
1692         RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class);
1693         DeqpTestRunner.ISleepProvider mockSleepProvider =
1694                 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class);
1695
1696         // recovery attempts to kill the process after a timeout
1697         mockSleepProvider.sleep(EasyMock.gt(0));
1698         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1699                 andReturn("root 1234 com.drawelement.deqp").once();
1700         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
1701                 andReturn("").once();
1702
1703         // Recovery checks if kill failed
1704         mockSleepProvider.sleep(EasyMock.gt(0));
1705         EasyMock.expectLastCall().once();
1706         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1707                 andReturn("root 1234 com.drawelement.deqp").once();
1708
1709         // Recovery resets the connection
1710         mockDevice.recoverDevice();
1711         EasyMock.expectLastCall().once();
1712
1713         // and attempts to kill the process again
1714         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1715                 andReturn("root 1234 com.drawelement.deqp").once();
1716         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
1717                 andReturn("").once();
1718
1719         // Recovery checks if kill failed
1720         mockSleepProvider.sleep(EasyMock.gt(0));
1721         EasyMock.expectLastCall().once();
1722         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
1723                 andReturn("root 1234 com.drawelement.deqp").once();
1724
1725         // recovery reboots the device
1726         mockDevice.reboot();
1727         EasyMock.expectLastCall().once();
1728
1729         orderedControl.replay();
1730         recovery.setDevice(mockDevice);
1731         recovery.setSleepProvider(mockSleepProvider);
1732         recovery.recoverComLinkKilled();
1733         orderedControl.verify();
1734     }
1735
1736     /**
1737      * Test external interruption before batch run.
1738      */
1739     public void testInterrupt_killBeforeBatch() throws Exception {
1740         final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.interrupt", "test");
1741
1742         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1743         tests.add(testId);
1744
1745         ITestInvocationListener mockListener
1746                 = EasyMock.createStrictMock(ITestInvocationListener.class);
1747         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1748         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
1749         IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class);
1750
1751         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
1752
1753         deqpTest.setDevice(mockDevice);
1754         deqpTest.setRunUtil(mockRunUtil);
1755
1756         int version = 3 << 16;
1757         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1758                 .andReturn(Integer.toString(version)).atLeastOnce();
1759
1760         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1761             andReturn("").once();
1762
1763         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1764                 EasyMock.eq(true),
1765                 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1766                 .once();
1767
1768         expectRenderConfigQuery(mockDevice,
1769                 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified "
1770                 + "--deqp-surface-type=window --deqp-gl-major-version=3 "
1771                 + "--deqp-gl-minor-version=0");
1772
1773         mockRunUtil.sleep(0);
1774         EasyMock.expectLastCall().andThrow(new RunInterruptedException());
1775
1776         mockListener.testRunStarted(getTestId(deqpTest), 1);
1777         EasyMock.expectLastCall().once();
1778
1779         EasyMock.replay(mockDevice, mockIDevice);
1780         EasyMock.replay(mockListener);
1781         EasyMock.replay(mockRunUtil);
1782         try {
1783             deqpTest.run(mockListener);
1784             fail("expected RunInterruptedException");
1785         } catch (RunInterruptedException ex) {
1786             // expected
1787         }
1788         EasyMock.verify(mockRunUtil);
1789         EasyMock.verify(mockListener);
1790         EasyMock.verify(mockDevice, mockIDevice);
1791     }
1792
1793     private void runShardedTest(TestIdentifier[] testIds,
1794             ArrayList<ArrayList<TestIdentifier>> testsForShard) throws Exception {
1795         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1796         for (TestIdentifier id : testIds) tests.add(id);
1797
1798         DeqpTestRunner runner = buildGlesTestRunner(3, 0, tests);
1799         ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split();
1800
1801         for (int shardIndex = 0; shardIndex < shards.size(); shardIndex++) {
1802             DeqpTestRunner shard = (DeqpTestRunner)shards.get(shardIndex);
1803             shard.setBuildHelper(getMockBuildHelper());
1804
1805             ArrayList<TestIdentifier> shardTests = testsForShard.get(shardIndex);
1806
1807             ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1808             ITestInvocationListener mockListener
1809                     = EasyMock.createStrictMock(ITestInvocationListener.class);
1810             IDevice mockIDevice = EasyMock.createMock(IDevice.class);
1811             int version = 3 << 16;
1812             EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1813                     .andReturn(Integer.toString(version)).atLeastOnce();
1814
1815             EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
1816                     .once();
1817             EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1818                     EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
1819                     .andReturn(null).once();
1820
1821             mockListener.testRunStarted(getTestId(shard), shardTests.size());
1822             EasyMock.expectLastCall().once();
1823
1824             expectRenderConfigQuery(mockDevice, 3, 0);
1825
1826             String testOut = buildTestProcessOutput(shardTests);
1827             // NOTE: This assumes that there won't be multiple batches per shard!
1828             runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut);
1829
1830             for (int i = 0; i < shardTests.size(); i++) {
1831                 mockListener.testStarted(EasyMock.eq(shardTests.get(i)));
1832                 EasyMock.expectLastCall().once();
1833
1834                 mockListener.testEnded(EasyMock.eq(shardTests.get(i)),
1835                                        EasyMock.<Map<String, String>>notNull());
1836
1837                 EasyMock.expectLastCall().once();
1838             }
1839
1840             mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
1841             EasyMock.expectLastCall().once();
1842
1843             EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
1844                     .once();
1845
1846             EasyMock.replay(mockDevice, mockIDevice);
1847             EasyMock.replay(mockListener);
1848
1849             shard.setDevice(mockDevice);
1850             shard.run(mockListener);
1851
1852             EasyMock.verify(mockListener);
1853             EasyMock.verify(mockDevice, mockIDevice);
1854         }
1855     }
1856
1857     public void testSharding_smallTrivial() throws Exception {
1858         final TestIdentifier[] testIds = {
1859                 new TestIdentifier("dEQP-GLES3.info", "vendor"),
1860                 new TestIdentifier("dEQP-GLES3.info", "renderer"),
1861                 new TestIdentifier("dEQP-GLES3.info", "version"),
1862                 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
1863                 new TestIdentifier("dEQP-GLES3.info", "extensions"),
1864                 new TestIdentifier("dEQP-GLES3.info", "render_target")
1865         };
1866         ArrayList<ArrayList<TestIdentifier>> shardedTests = new ArrayList<>();
1867         ArrayList<TestIdentifier> shardOne = new ArrayList<>();
1868         for (int i = 0; i < testIds.length; i++) {
1869             shardOne.add(testIds[i]);
1870         }
1871         shardedTests.add(shardOne);
1872         runShardedTest(testIds, shardedTests);
1873     }
1874
1875     public void testSharding_twoShards() throws Exception {
1876         final int TEST_COUNT = 1237;
1877         final int SHARD_SIZE = 1000;
1878
1879         ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT);
1880         for (int i = 0; i < TEST_COUNT; i++) {
1881             testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i)));
1882         }
1883
1884         ArrayList<ArrayList<TestIdentifier>> shardedTests = new ArrayList<>();
1885         ArrayList<TestIdentifier> shard = new ArrayList<>();
1886         for (int i = 0; i < testIds.size(); i++) {
1887             if (i == SHARD_SIZE) {
1888                 shardedTests.add(shard);
1889                 shard = new ArrayList<>();
1890             }
1891             shard.add(testIds.get(i));
1892         }
1893         shardedTests.add(shard);
1894         runShardedTest(testIds.toArray(new TestIdentifier[testIds.size()]), shardedTests);
1895     }
1896
1897     public void testSharding_empty() throws Exception {
1898         DeqpTestRunner runner = buildGlesTestRunner(3, 0, new ArrayList<TestIdentifier>());
1899         ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split();
1900         // \todo [2015-11-23 kalle] What should the result be? The runner or nothing?
1901     }
1902
1903     /**
1904      * Test external interruption in testFailed().
1905      */
1906     public void testInterrupt_killReportTestFailed() throws Exception {
1907         final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.interrupt", "test");
1908         final String testPath = "dEQP-GLES3.interrupt.test";
1909         final String testTrie = "{dEQP-GLES3{interrupt{test}}}";
1910         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
1911                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1912                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
1913                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1914                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
1915                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1916                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
1917                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1918                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
1919                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
1920                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
1921                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1922                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
1923                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1924                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
1925                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
1926                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1927                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Fail\r\n"
1928                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Fail\r\n"
1929                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
1930                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1931                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
1932                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1933                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
1934                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
1935                 + "INSTRUMENTATION_CODE: 0\r\n";
1936
1937         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
1938         tests.add(testId);
1939
1940         ITestInvocationListener mockListener
1941                 = EasyMock.createStrictMock(ITestInvocationListener.class);
1942         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
1943         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
1944         IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class);
1945
1946         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
1947
1948         deqpTest.setDevice(mockDevice);
1949         deqpTest.setRunUtil(mockRunUtil);
1950
1951         int version = 3 << 16;
1952         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
1953                 .andReturn(Integer.toString(version)).atLeastOnce();
1954
1955         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).
1956             andReturn("").once();
1957
1958         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
1959                 EasyMock.eq(true),
1960                 EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))).andReturn(null)
1961                 .once();
1962
1963         expectRenderConfigQuery(mockDevice,
1964                 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified "
1965                 + "--deqp-surface-type=window --deqp-gl-major-version=3 "
1966                 + "--deqp-gl-minor-version=0");
1967
1968         mockRunUtil.sleep(0);
1969         EasyMock.expectLastCall().once();
1970
1971         String commandLine = String.format(
1972                 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
1973                 + "--deqp-screen-rotation=unspecified "
1974                 + "--deqp-surface-type=window "
1975                 + "--deqp-log-images=disable "
1976                 + "--deqp-watchdog=enable",
1977                 CASE_LIST_FILE_NAME);
1978
1979         runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
1980                 output);
1981
1982         mockListener.testRunStarted(getTestId(deqpTest), 1);
1983         EasyMock.expectLastCall().once();
1984
1985         mockListener.testStarted(EasyMock.eq(testId));
1986         EasyMock.expectLastCall().once();
1987
1988         mockListener.testFailed(EasyMock.eq(testId), EasyMock.<String>notNull());
1989         EasyMock.expectLastCall().andThrow(new RunInterruptedException());
1990
1991         EasyMock.replay(mockDevice, mockIDevice);
1992         EasyMock.replay(mockListener);
1993         EasyMock.replay(mockRunUtil);
1994         try {
1995             deqpTest.run(mockListener);
1996             fail("expected RunInterruptedException");
1997         } catch (RunInterruptedException ex) {
1998             // expected
1999         }
2000         EasyMock.verify(mockRunUtil);
2001         EasyMock.verify(mockListener);
2002         EasyMock.verify(mockDevice, mockIDevice);
2003     }
2004
2005     public void testRuntimeHint_optionSet() throws Exception {
2006         /* MultiLineReceiver expects "\r\n" line ending. */
2007         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
2008                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
2009                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
2010                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2011                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
2012                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
2013                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
2014                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2015                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
2016                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
2017                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
2018                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2019                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
2020                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2021                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2022                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n"
2023                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2024                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2025                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2026                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2027                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2028                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2029                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2030                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2031                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n"
2032                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2033                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2034                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2035                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2036                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2037                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2038                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2039                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2040                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n"
2041                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2042                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2043                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2044                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2045                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2046                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2047                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2048                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2049                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n"
2050                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2051                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2052                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2053                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2054                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2055                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2056                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2057                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2058                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n"
2059                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2060                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2061                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2062                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2063                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2064                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2065                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2066                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
2067                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n"
2068                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2069                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
2070                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
2071                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
2072                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2073                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
2074                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2075                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
2076                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
2077                 + "INSTRUMENTATION_CODE: 0\r\n";
2078
2079         final TestIdentifier[] testIds = {
2080                 new TestIdentifier("dEQP-GLES3.info", "vendor"),
2081                 new TestIdentifier("dEQP-GLES3.info", "renderer"),
2082                 new TestIdentifier("dEQP-GLES3.info", "version"),
2083                 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
2084                 new TestIdentifier("dEQP-GLES3.info", "extensions"),
2085                 new TestIdentifier("dEQP-GLES3.info", "render_target")
2086         };
2087
2088         final String[] testPaths = {
2089                 "dEQP-GLES3.info.vendor",
2090                 "dEQP-GLES3.info.renderer",
2091                 "dEQP-GLES3.info.version",
2092                 "dEQP-GLES3.info.shading_language_version",
2093                 "dEQP-GLES3.info.extensions",
2094                 "dEQP-GLES3.info.render_target"
2095         };
2096
2097         final String testTrie
2098                 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}";
2099
2100         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
2101         ITestInvocationListener mockListener
2102                 = EasyMock.createStrictMock(ITestInvocationListener.class);
2103         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
2104
2105         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
2106
2107         for (TestIdentifier id : testIds) {
2108             tests.add(id);
2109         }
2110
2111         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
2112         OptionSetter setter = new OptionSetter(deqpTest);
2113         final long runtimeMs = 123456;
2114         setter.setOptionValue("runtime-hint", String.valueOf(runtimeMs));
2115         assertEquals("Wrong expected runtime - option not passed cleanly", runtimeMs, deqpTest.getRuntimeHint());
2116
2117         // Try running the tests as well. The unit tests do not set the hint be default,
2118         // so that case is covered.
2119
2120         int version = 3 << 16;
2121         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
2122                 .andReturn(Integer.toString(version)).atLeastOnce();
2123
2124         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
2125                 .once();
2126         EasyMock.expect(mockDevice.installPackage(EasyMock.<File>anyObject(),
2127                 EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName()))))
2128                 .andReturn(null).once();
2129
2130         expectRenderConfigQuery(mockDevice, 3, 0);
2131
2132         String commandLine = String.format(
2133                 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
2134                 + "--deqp-screen-rotation=unspecified "
2135                 + "--deqp-surface-type=window "
2136                 + "--deqp-log-images=disable "
2137                 + "--deqp-watchdog=enable",
2138                 CASE_LIST_FILE_NAME);
2139
2140         runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
2141
2142         mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
2143         EasyMock.expectLastCall().once();
2144
2145         for (int i = 0; i < testPaths.length; i++) {
2146             mockListener.testStarted(EasyMock.eq(testIds[i]));
2147             EasyMock.expectLastCall().once();
2148
2149             mockListener.testEnded(EasyMock.eq(testIds[i]),
2150                     EasyMock.<Map<String, String>>notNull());
2151
2152             EasyMock.expectLastCall().once();
2153         }
2154
2155         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
2156         EasyMock.expectLastCall().once();
2157
2158         EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("")
2159                 .once();
2160
2161         EasyMock.replay(mockDevice, mockIDevice);
2162         EasyMock.replay(mockListener);
2163
2164         deqpTest.setDevice(mockDevice);
2165         deqpTest.run(mockListener);
2166
2167         EasyMock.verify(mockListener);
2168         EasyMock.verify(mockDevice, mockIDevice);
2169     }
2170
2171     public void testRuntimeHint_optionSetSharded() throws Exception {
2172         final int TEST_COUNT = 1237;
2173         final int SHARD_SIZE = 1000;
2174
2175         ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT);
2176         for (int i = 0; i < TEST_COUNT; i++) {
2177             testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i)));
2178         }
2179
2180         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds);
2181         OptionSetter setter = new OptionSetter(deqpTest);
2182         final long fullRuntimeMs = testIds.size()*100;
2183         setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs));
2184
2185         ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)deqpTest.split();
2186         assertEquals("First shard's time not proportional to test count",
2187                  (fullRuntimeMs*SHARD_SIZE)/TEST_COUNT,
2188                  ((IRuntimeHintProvider)shards.get(0)).getRuntimeHint());
2189         assertEquals("Second shard's time not proportional to test count",
2190                  (fullRuntimeMs*(TEST_COUNT-SHARD_SIZE))/TEST_COUNT,
2191                  ((IRuntimeHintProvider)shards.get(1)).getRuntimeHint());
2192     }
2193
2194     public void testRuntimeHint_optionNotSet() throws Exception {
2195         final TestIdentifier[] testIds = {
2196                 new TestIdentifier("dEQP-GLES3.info", "vendor"),
2197                 new TestIdentifier("dEQP-GLES3.info", "renderer"),
2198                 new TestIdentifier("dEQP-GLES3.info", "version"),
2199                 new TestIdentifier("dEQP-GLES3.info", "shading_language_version"),
2200                 new TestIdentifier("dEQP-GLES3.info", "extensions"),
2201                 new TestIdentifier("dEQP-GLES3.info", "render_target")
2202         };
2203         Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
2204
2205         for (TestIdentifier id : testIds) {
2206             tests.add(id);
2207         }
2208
2209         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
2210
2211         long runtime = deqpTest.getRuntimeHint();
2212         assertTrue("Runtime for tests must be positive", runtime > 0);
2213         assertTrue("Runtime for tests must be reasonable", runtime < (1000 * 10)); // Must be done in 10s
2214     }
2215
2216
2217     private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice,
2218             final String output) throws Exception {
2219         String cmd = String.format(
2220             "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
2221             + "--deqp-screen-rotation=unspecified "
2222             + "--deqp-surface-type=window "
2223             + "--deqp-log-images=disable "
2224             + "--deqp-watchdog=enable",
2225             CASE_LIST_FILE_NAME);
2226         runInstrumentationLineAndAnswer(mockDevice, mockIDevice, null, cmd, output);
2227     }
2228
2229     private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice,
2230             final String testTrie, final String cmd, final String output) throws Exception {
2231         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME)))
2232                 .andReturn("").once();
2233
2234         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME)))
2235                 .andReturn("").once();
2236
2237         if (testTrie == null) {
2238             mockDevice.pushString((String)EasyMock.anyObject(), EasyMock.eq(CASE_LIST_FILE_NAME));
2239         }
2240         else {
2241             mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME);
2242         }
2243         EasyMock.expectLastCall().andReturn(true).once();
2244
2245         String command = String.format(
2246                 "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\" "
2247                     + "-e deqpLogData \"%s\" %s",
2248                 AbiUtils.createAbiFlag(ABI.getName()), LOG_FILE_NAME, cmd, false,
2249                 INSTRUMENTATION_NAME);
2250
2251         EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice);
2252         mockIDevice.executeShellCommand(EasyMock.eq(command),
2253                 EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(),
2254                 EasyMock.isA(TimeUnit.class));
2255
2256         EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
2257             @Override
2258             public Object answer() {
2259                 IShellOutputReceiver receiver
2260                         = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
2261
2262                 receiver.addOutput(output.getBytes(), 0, output.length());
2263                 receiver.flush();
2264
2265                 return null;
2266             }
2267         });
2268     }
2269 }