android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java

   1 /*
   2  * Copyright (C) 2014 The Android Open Source Project
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package com.drawelements.deqp.runner;
  17
  18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
  19 import com.android.compatibility.common.util.AbiUtils;
  20 import com.android.ddmlib.AdbCommandRejectedException;
  21 import com.android.ddmlib.IShellOutputReceiver;
  22 import com.android.ddmlib.MultiLineReceiver;
  23 import com.android.ddmlib.ShellCommandUnresponsiveException;
  24 import com.android.ddmlib.TimeoutException;
  25 import com.android.ddmlib.testrunner.TestIdentifier;
  26 import com.android.tradefed.build.IBuildInfo;
  27 import com.android.tradefed.build.IFolderBuildInfo;
  28 import com.android.tradefed.config.Option;
  29 import com.android.tradefed.config.OptionClass;
  30 import com.android.tradefed.device.DeviceNotAvailableException;
  31 import com.android.tradefed.device.ITestDevice;
  32 import com.android.tradefed.log.LogUtil.CLog;
  33 import com.android.tradefed.result.ByteArrayInputStreamSource;
  34 import com.android.tradefed.result.ITestInvocationListener;
  35 import com.android.tradefed.result.LogDataType;
  36 import com.android.tradefed.testtype.IAbi;
  37 import com.android.tradefed.testtype.IBuildReceiver;
  38 import com.android.tradefed.testtype.IAbiReceiver;
  39 import com.android.tradefed.testtype.IDeviceTest;
  40 import com.android.tradefed.testtype.IRemoteTest;
  41 import com.android.tradefed.testtype.IRuntimeHintProvider;
  42 import com.android.tradefed.testtype.IShardableTest;
  43 import com.android.tradefed.testtype.ITestCollector;
  44 import com.android.tradefed.testtype.ITestFilterReceiver;
  45 import com.android.tradefed.util.IRunUtil;
  46 import com.android.tradefed.util.RunInterruptedException;
  47 import com.android.tradefed.util.RunUtil;
  48
  49 import java.io.BufferedReader;
  50 import java.io.File;
  51 import java.io.FileNotFoundException;
  52 import java.io.FileReader;
  53 import java.io.IOException;
  54 import java.io.Reader;
  55 import java.lang.reflect.Method;
  56 import java.lang.reflect.InvocationTargetException;
  57 import java.util.ArrayList;
  58 import java.util.Collection;
  59 import java.util.Collections;
  60 import java.util.HashMap;
  61 import java.util.HashSet;
  62 import java.util.Iterator;
  63 import java.util.LinkedHashMap;
  64 import java.util.LinkedHashSet;
  65 import java.util.LinkedList;
  66 import java.util.List;
  67 import java.util.Map;
  68 import java.util.regex.Pattern;
  69 import java.util.Set;
  70 import java.util.concurrent.TimeUnit;
  71
  72 /**
  73  * Test runner for dEQP tests
  74  *
  75  * Supports running drawElements Quality Program tests found under external/deqp.
  76  */
  77 @OptionClass(alias="deqp-test-runner")
  78 public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
  79         ITestFilterReceiver, IAbiReceiver, IShardableTest, ITestCollector,
  80         IRuntimeHintProvider {
  81     private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk";
  82     private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp";
  83     private static final String INCOMPLETE_LOG_MESSAGE = "Crash: Incomplete test log";
  84     private static final String SKIPPED_INSTANCE_LOG_MESSAGE = "Configuration skipped";
  85     private static final String NOT_EXECUTABLE_LOG_MESSAGE = "Abort: Test cannot be executed";
  86     private static final String CASE_LIST_FILE_NAME = "/sdcard/dEQP-TestCaseList.txt";
  87     private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa";
  88     public static final String FEATURE_LANDSCAPE = "android.hardware.screen.landscape";
  89     public static final String FEATURE_PORTRAIT = "android.hardware.screen.portrait";
  90     public static final String FEATURE_VULKAN_LEVEL = "android.hardware.vulkan.level";
  91
  92     private static final int TESTCASE_BATCH_LIMIT = 1000;
  93     private static final int UNRESPONSIVE_CMD_TIMEOUT_MS = 10 * 60 * 1000; // 10min
  94
  95     // !NOTE: There's a static method copyOptions() for copying options during split.
  96     // If you add state update copyOptions() as appropriate!
  97
  98     @Option(name="deqp-package",
  99             description="Name of the deqp module used. Determines GLES version.",
 100             importance=Option.Importance.ALWAYS)
 101     private String mDeqpPackage;
 102     @Option(name="deqp-gl-config-name",
 103             description="GL render target config. See deqp documentation for syntax. ",
 104             importance=Option.Importance.NEVER)
 105     private String mConfigName = "";
 106     @Option(name="deqp-caselist-file",
 107             description="File listing the names of the cases to be run.",
 108             importance=Option.Importance.ALWAYS)
 109     private String mCaselistFile;
 110     @Option(name="deqp-screen-rotation",
 111             description="Screen orientation. Defaults to 'unspecified'",
 112             importance=Option.Importance.NEVER)
 113     private String mScreenRotation = "unspecified";
 114     @Option(name="deqp-surface-type",
 115             description="Surface type ('window', 'pbuffer', 'fbo'). Defaults to 'window'",
 116             importance=Option.Importance.NEVER)
 117     private String mSurfaceType = "window";
 118     @Option(name="deqp-config-required",
 119             description="Is current config required if API is supported? Defaults to false.",
 120             importance=Option.Importance.NEVER)
 121     private boolean mConfigRequired = false;
 122     @Option(name = "include-filter",
 123             description="Test include filter. '*' is zero or more letters. '.' has no special meaning.")
 124     private List<String> mIncludeFilters = new ArrayList<>();
 125     @Option(name = "exclude-filter",
 126             description="Test exclude filter. '*' is zero or more letters. '.' has no special meaning.")
 127     private List<String> mExcludeFilters = new ArrayList<>();
 128     @Option(name = "collect-tests-only",
 129             description = "Only invoke the instrumentation to collect list of applicable test "
 130                     + "cases. All test run callbacks will be triggered, but test execution will "
 131                     + "not be actually carried out.")
 132     private boolean mCollectTestsOnly = false;
 133     @Option(name = "runtime-hint",
 134             isTimeVal = true,
 135             description="The estimated config runtime. Defaults to 200ms x num tests.")
 136     private long mRuntimeHint = -1;
 137
 138     private Collection<TestIdentifier> mRemainingTests = null;
 139     private Map<TestIdentifier, Set<BatchRunConfiguration>> mTestInstances = null;
 140     private final TestInstanceResultListener mInstanceListerner = new TestInstanceResultListener();
 141     private final Map<TestIdentifier, Integer> mTestInstabilityRatings = new HashMap<>();
 142     private IAbi mAbi;
 143     private CompatibilityBuildHelper mBuildHelper;
 144     private boolean mLogData = false;
 145     private ITestDevice mDevice;
 146     private Set<String> mDeviceFeatures;
 147     private Map<String, Boolean> mConfigQuerySupportCache = new HashMap<>();
 148     private IRunUtil mRunUtil = RunUtil.getDefault();
 149     // When set will override the mCaselistFile for testing purposes.
 150     private Reader mCaselistReader = null;
 151
 152     private IRecovery mDeviceRecovery = new Recovery(); {
 153         mDeviceRecovery.setSleepProvider(new SleepProvider());
 154     }
 155
 156     public DeqpTestRunner() {
 157     }
 158
 159     private DeqpTestRunner(DeqpTestRunner optionTemplate,
 160                 Map<TestIdentifier, Set<BatchRunConfiguration>> tests) {
 161         copyOptions(this, optionTemplate);
 162         mTestInstances = tests;
 163     }
 164
 165     /**
 166      * @param abi the ABI to run the test on
 167      */
 168     @Override
 169     public void setAbi(IAbi abi) {
 170         mAbi = abi;
 171     }
 172
 173     /**
 174      * {@inheritDoc}
 175      */
 176     @Override
 177     public void setBuild(IBuildInfo buildInfo) {
 178         setBuildHelper(new CompatibilityBuildHelper(buildInfo));
 179     }
 180
 181     /**
 182      * Exposed for better mockability during testing. In real use, always flows from
 183      * setBuild() called by the framework
 184      */
 185     public void setBuildHelper(CompatibilityBuildHelper helper) {
 186         mBuildHelper = helper;
 187     }
 188
 189     /**
 190      * Enable or disable raw dEQP test log collection.
 191      */
 192     public void setCollectLogs(boolean logData) {
 193         mLogData = logData;
 194     }
 195
 196     /**
 197      * Get the deqp-package option contents.
 198      */
 199     public String getPackageName() {
 200         return mDeqpPackage;
 201     }
 202
 203     /**
 204      * {@inheritDoc}
 205      */
 206     @Override
 207     public void setDevice(ITestDevice device) {
 208         mDevice = device;
 209     }
 210
 211     /**
 212      * {@inheritDoc}
 213      */
 214     @Override
 215     public ITestDevice getDevice() {
 216         return mDevice;
 217     }
 218
 219     /**
 220      * Set recovery handler.
 221      *
 222      * Exposed for unit testing.
 223      */
 224     public void setRecovery(IRecovery deviceRecovery) {
 225         mDeviceRecovery = deviceRecovery;
 226     }
 227
 228     /**
 229      * Set IRunUtil.
 230      *
 231      * Exposed for unit testing.
 232      */
 233     public void setRunUtil(IRunUtil runUtil) {
 234         mRunUtil = runUtil;
 235     }
 236
 237     /**
 238      * Exposed for unit testing
 239      */
 240     public void setCaselistReader(Reader caselistReader) {
 241         mCaselistReader = caselistReader;
 242     }
 243
 244     private static final class CapabilityQueryFailureException extends Exception {
 245     }
 246
 247     /**
 248      * Test configuration of dEPQ test instance execution.
 249      * Exposed for unit testing
 250      */
 251     public static final class BatchRunConfiguration {
 252         public static final String ROTATION_UNSPECIFIED = "unspecified";
 253         public static final String ROTATION_PORTRAIT = "0";
 254         public static final String ROTATION_LANDSCAPE = "90";
 255         public static final String ROTATION_REVERSE_PORTRAIT = "180";
 256         public static final String ROTATION_REVERSE_LANDSCAPE = "270";
 257
 258         private final String mGlConfig;
 259         private final String mRotation;
 260         private final String mSurfaceType;
 261         private final boolean mRequired;
 262
 263         public BatchRunConfiguration(String glConfig, String rotation, String surfaceType, boolean required) {
 264             mGlConfig = glConfig;
 265             mRotation = rotation;
 266             mSurfaceType = surfaceType;
 267             mRequired = required;
 268         }
 269
 270         /**
 271          * Get string that uniquely identifies this config
 272          */
 273         public String getId() {
 274             return String.format("{glformat=%s,rotation=%s,surfacetype=%s,required=%b}",
 275                     mGlConfig, mRotation, mSurfaceType, mRequired);
 276         }
 277
 278         /**
 279          * Get the GL config used in this configuration.
 280          */
 281         public String getGlConfig() {
 282             return mGlConfig;
 283         }
 284
 285         /**
 286          * Get the screen rotation used in this configuration.
 287          */
 288         public String getRotation() {
 289             return mRotation;
 290         }
 291
 292         /**
 293          * Get the surface type used in this configuration.
 294          */
 295         public String getSurfaceType() {
 296             return mSurfaceType;
 297         }
 298
 299         /**
 300          * Is this configuration mandatory to support, if target API is supported?
 301          */
 302         public boolean isRequired() {
 303             return mRequired;
 304         }
 305
 306         @Override
 307         public boolean equals(Object other) {
 308             if (other == null) {
 309                 return false;
 310             } else if (!(other instanceof BatchRunConfiguration)) {
 311                 return false;
 312             } else {
 313                 return getId().equals(((BatchRunConfiguration)other).getId());
 314             }
 315         }
 316
 317         @Override
 318         public int hashCode() {
 319             return getId().hashCode();
 320         }
 321     }
 322
 323     /**
 324      * dEQP test instance listerer and invocation result forwarded
 325      */
 326     private class TestInstanceResultListener {
 327         private ITestInvocationListener mSink;
 328         private BatchRunConfiguration mRunConfig;
 329
 330         private TestIdentifier mCurrentTestId;
 331         private boolean mGotTestResult;
 332         private String mCurrentTestLog;
 333
 334         private class PendingResult {
 335             boolean allInstancesPassed;
 336             Map<BatchRunConfiguration, String> testLogs;
 337             Map<BatchRunConfiguration, String> errorMessages;
 338             Set<BatchRunConfiguration> remainingConfigs;
 339         }
 340
 341         private final Map<TestIdentifier, PendingResult> mPendingResults = new HashMap<>();
 342
 343         public void setSink(ITestInvocationListener sink) {
 344             mSink = sink;
 345         }
 346
 347         public void setCurrentConfig(BatchRunConfiguration runConfig) {
 348             mRunConfig = runConfig;
 349         }
 350
 351         /**
 352          * Get currently processed test id, or null if not currently processing a test case
 353          */
 354         public TestIdentifier getCurrentTestId() {
 355             return mCurrentTestId;
 356         }
 357
 358         /**
 359          * Forward result to sink
 360          */
 361         private void forwardFinalizedPendingResult(TestIdentifier testId) {
 362             if (mRemainingTests.contains(testId)) {
 363                 final PendingResult result = mPendingResults.get(testId);
 364
 365                 mPendingResults.remove(testId);
 366                 mRemainingTests.remove(testId);
 367
 368                 // Forward results to the sink
 369                 mSink.testStarted(testId);
 370
 371                 // Test Log
 372                 if (mLogData) {
 373                     for (Map.Entry<BatchRunConfiguration, String> entry :
 374                             result.testLogs.entrySet()) {
 375                         final ByteArrayInputStreamSource source
 376                                 = new ByteArrayInputStreamSource(entry.getValue().getBytes());
 377
 378                         mSink.testLog(testId.getClassName() + "." + testId.getTestName() + "@"
 379                                 + entry.getKey().getId(), LogDataType.XML, source);
 380
 381                         source.cancel();
 382                     }
 383                 }
 384
 385                 // Error message
 386                 if (!result.allInstancesPassed) {
 387                     final StringBuilder errorLog = new StringBuilder();
 388
 389                     for (Map.Entry<BatchRunConfiguration, String> entry :
 390                             result.errorMessages.entrySet()) {
 391                         if (errorLog.length() > 0) {
 392                             errorLog.append('\n');
 393                         }
 394                         errorLog.append(String.format("=== with config %s ===\n",
 395                                 entry.getKey().getId()));
 396                         errorLog.append(entry.getValue());
 397                     }
 398
 399                     mSink.testFailed(testId, errorLog.toString());
 400                 }
 401
 402                 final Map<String, String> emptyMap = Collections.emptyMap();
 403                 mSink.testEnded(testId, emptyMap);
 404             }
 405         }
 406
 407         /**
 408          * Declare existence of a test and instances
 409          */
 410         public void setTestInstances(TestIdentifier testId, Set<BatchRunConfiguration> configs) {
 411             // Test instances cannot change at runtime, ignore if we have already set this
 412             if (!mPendingResults.containsKey(testId)) {
 413                 final PendingResult pendingResult = new PendingResult();
 414                 pendingResult.allInstancesPassed = true;
 415                 pendingResult.testLogs = new LinkedHashMap<>();
 416                 pendingResult.errorMessages = new LinkedHashMap<>();
 417                 pendingResult.remainingConfigs = new HashSet<>(configs); // avoid mutating argument
 418                 mPendingResults.put(testId, pendingResult);
 419             }
 420         }
 421
 422         /**
 423          * Query if test instance has not yet been executed
 424          */
 425         public boolean isPendingTestInstance(TestIdentifier testId,
 426                 BatchRunConfiguration config) {
 427             final PendingResult result = mPendingResults.get(testId);
 428             if (result == null) {
 429                 // test is not in the current working batch of the runner, i.e. it cannot be
 430                 // "partially" completed.
 431                 if (!mRemainingTests.contains(testId)) {
 432                     // The test has been fully executed. Not pending.
 433                     return false;
 434                 } else {
 435                     // Test has not yet been executed. Check if such instance exists
 436                     return mTestInstances.get(testId).contains(config);
 437                 }
 438             } else {
 439                 // could be partially completed, check this particular config
 440                 return result.remainingConfigs.contains(config);
 441             }
 442         }
 443
 444         /**
 445          * Fake execution of an instance with current config
 446          */
 447         public void skipTest(TestIdentifier testId) {
 448             final PendingResult result = mPendingResults.get(testId);
 449
 450             result.errorMessages.put(mRunConfig, SKIPPED_INSTANCE_LOG_MESSAGE);
 451             result.remainingConfigs.remove(mRunConfig);
 452
 453             // Pending result finished, report result
 454             if (result.remainingConfigs.isEmpty()) {
 455                 forwardFinalizedPendingResult(testId);
 456             }
 457         }
 458
 459         /**
 460          * Fake failure of an instance with current config
 461          */
 462         public void abortTest(TestIdentifier testId, String errorMessage) {
 463             final PendingResult result = mPendingResults.get(testId);
 464
 465             // Mark as executed
 466             result.allInstancesPassed = false;
 467             result.errorMessages.put(mRunConfig, errorMessage);
 468             result.remainingConfigs.remove(mRunConfig);
 469
 470             // Pending result finished, report result
 471             if (result.remainingConfigs.isEmpty()) {
 472                 forwardFinalizedPendingResult(testId);
 473             }
 474
 475             if (testId.equals(mCurrentTestId)) {
 476                 mCurrentTestId = null;
 477             }
 478         }
 479
 480         /**
 481          * Handles beginning of dEQP session.
 482          */
 483         private void handleBeginSession(Map<String, String> values) {
 484             // ignore
 485         }
 486
 487         /**
 488          * Handles end of dEQP session.
 489          */
 490         private void handleEndSession(Map<String, String> values) {
 491             // ignore
 492         }
 493
 494         /**
 495          * Handles beginning of dEQP testcase.
 496          */
 497         private void handleBeginTestCase(Map<String, String> values) {
 498             mCurrentTestId = pathToIdentifier(values.get("dEQP-BeginTestCase-TestCasePath"));
 499             mCurrentTestLog = "";
 500             mGotTestResult = false;
 501
 502             // mark instance as started
 503             if (mPendingResults.get(mCurrentTestId) != null) {
 504                 mPendingResults.get(mCurrentTestId).remainingConfigs.remove(mRunConfig);
 505             } else {
 506                 CLog.w("Got unexpected start of %s", mCurrentTestId);
 507             }
 508         }
 509
 510         /**
 511          * Handles end of dEQP testcase.
 512          */
 513         private void handleEndTestCase(Map<String, String> values) {
 514             final PendingResult result = mPendingResults.get(mCurrentTestId);
 515
 516             if (result != null) {
 517                 if (!mGotTestResult) {
 518                     result.allInstancesPassed = false;
 519                     result.errorMessages.put(mRunConfig, INCOMPLETE_LOG_MESSAGE);
 520                 }
 521
 522                 if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
 523                     result.testLogs.put(mRunConfig, mCurrentTestLog);
 524                 }
 525
 526                 // Pending result finished, report result
 527                 if (result.remainingConfigs.isEmpty()) {
 528                     forwardFinalizedPendingResult(mCurrentTestId);
 529                 }
 530             } else {
 531                 CLog.w("Got unexpected end of %s", mCurrentTestId);
 532             }
 533             mCurrentTestId = null;
 534         }
 535
 536         /**
 537          * Handles dEQP testcase result.
 538          */
 539         private void handleTestCaseResult(Map<String, String> values) {
 540             String code = values.get("dEQP-TestCaseResult-Code");
 541             String details = values.get("dEQP-TestCaseResult-Details");
 542
 543             if (mPendingResults.get(mCurrentTestId) == null) {
 544                 CLog.w("Got unexpected result for %s", mCurrentTestId);
 545                 mGotTestResult = true;
 546                 return;
 547             }
 548
 549             if (code.compareTo("Pass") == 0) {
 550                 mGotTestResult = true;
 551             } else if (code.compareTo("NotSupported") == 0) {
 552                 mGotTestResult = true;
 553             } else if (code.compareTo("QualityWarning") == 0) {
 554                 mGotTestResult = true;
 555             } else if (code.compareTo("CompatibilityWarning") == 0) {
 556                 mGotTestResult = true;
 557             } else if (code.compareTo("Fail") == 0 || code.compareTo("ResourceError") == 0
 558                     || code.compareTo("InternalError") == 0 || code.compareTo("Crash") == 0
 559                     || code.compareTo("Timeout") == 0) {
 560                 mPendingResults.get(mCurrentTestId).allInstancesPassed = false;
 561                 mPendingResults.get(mCurrentTestId)
 562                         .errorMessages.put(mRunConfig, code + ": " + details);
 563                 mGotTestResult = true;
 564             } else {
 565                 String codeError = "Unknown result code: " + code;
 566                 mPendingResults.get(mCurrentTestId).allInstancesPassed = false;
 567                 mPendingResults.get(mCurrentTestId)
 568                         .errorMessages.put(mRunConfig, codeError + ": " + details);
 569                 mGotTestResult = true;
 570             }
 571         }
 572
 573         /**
 574          * Handles terminated dEQP testcase.
 575          */
 576         private void handleTestCaseTerminate(Map<String, String> values) {
 577             final PendingResult result = mPendingResults.get(mCurrentTestId);
 578
 579             if (result != null) {
 580                 String reason = values.get("dEQP-TerminateTestCase-Reason");
 581                 mPendingResults.get(mCurrentTestId).allInstancesPassed = false;
 582                 mPendingResults.get(mCurrentTestId)
 583                         .errorMessages.put(mRunConfig, "Terminated: " + reason);
 584
 585                 // Pending result finished, report result
 586                 if (result.remainingConfigs.isEmpty()) {
 587                     forwardFinalizedPendingResult(mCurrentTestId);
 588                 }
 589             } else {
 590                 CLog.w("Got unexpected termination of %s", mCurrentTestId);
 591             }
 592
 593             mCurrentTestId = null;
 594             mGotTestResult = true;
 595         }
 596
 597         /**
 598          * Handles dEQP testlog data.
 599          */
 600         private void handleTestLogData(Map<String, String> values) {
 601             mCurrentTestLog = mCurrentTestLog + values.get("dEQP-TestLogData-Log");
 602         }
 603
 604         /**
 605          * Handles new instrumentation status message.
 606          */
 607         public void handleStatus(Map<String, String> values) {
 608             String eventType = values.get("dEQP-EventType");
 609
 610             if (eventType == null) {
 611                 return;
 612             }
 613
 614             if (eventType.compareTo("BeginSession") == 0) {
 615                 handleBeginSession(values);
 616             } else if (eventType.compareTo("EndSession") == 0) {
 617                 handleEndSession(values);
 618             } else if (eventType.compareTo("BeginTestCase") == 0) {
 619                 handleBeginTestCase(values);
 620             } else if (eventType.compareTo("EndTestCase") == 0) {
 621                 handleEndTestCase(values);
 622             } else if (eventType.compareTo("TestCaseResult") == 0) {
 623                 handleTestCaseResult(values);
 624             } else if (eventType.compareTo("TerminateTestCase") == 0) {
 625                 handleTestCaseTerminate(values);
 626             } else if (eventType.compareTo("TestLogData") == 0) {
 627                 handleTestLogData(values);
 628             }
 629         }
 630
 631         /**
 632          * Signal listener that batch ended and forget incomplete results.
 633          */
 634         public void endBatch() {
 635             // end open test if when stream ends
 636             if (mCurrentTestId != null) {
 637                 // Current instance was removed from remainingConfigs when case
 638                 // started. Mark current instance as pending.
 639                 if (mPendingResults.get(mCurrentTestId) != null) {
 640                     mPendingResults.get(mCurrentTestId).remainingConfigs.add(mRunConfig);
 641                 } else {
 642                     CLog.w("Got unexpected internal state of %s", mCurrentTestId);
 643                 }
 644             }
 645             mCurrentTestId = null;
 646         }
 647     }
 648
 649     /**
 650      * dEQP instrumentation parser
 651      */
 652     private static class InstrumentationParser extends MultiLineReceiver {
 653         private TestInstanceResultListener mListener;
 654
 655         private Map<String, String> mValues;
 656         private String mCurrentName;
 657         private String mCurrentValue;
 658         private int mResultCode;
 659         private boolean mGotExitValue = false;
 660
 661
 662         public InstrumentationParser(TestInstanceResultListener listener) {
 663             mListener = listener;
 664         }
 665
 666         /**
 667          * {@inheritDoc}
 668          */
 669         @Override
 670         public void processNewLines(String[] lines) {
 671             for (String line : lines) {
 672                 if (mValues == null) mValues = new HashMap<String, String>();
 673
 674                 if (line.startsWith("INSTRUMENTATION_STATUS_CODE: ")) {
 675                     if (mCurrentName != null) {
 676                         mValues.put(mCurrentName, mCurrentValue);
 677
 678                         mCurrentName = null;
 679                         mCurrentValue = null;
 680                     }
 681
 682                     mListener.handleStatus(mValues);
 683                     mValues = null;
 684                 } else if (line.startsWith("INSTRUMENTATION_STATUS: dEQP-")) {
 685                     if (mCurrentName != null) {
 686                         mValues.put(mCurrentName, mCurrentValue);
 687
 688                         mCurrentValue = null;
 689                         mCurrentName = null;
 690                     }
 691
 692                     String prefix = "INSTRUMENTATION_STATUS: ";
 693                     int nameBegin = prefix.length();
 694                     int nameEnd = line.indexOf('=');
 695                     int valueBegin = nameEnd + 1;
 696
 697                     mCurrentName = line.substring(nameBegin, nameEnd);
 698                     mCurrentValue = line.substring(valueBegin);
 699                 } else if (line.startsWith("INSTRUMENTATION_CODE: ")) {
 700                     try {
 701                         mResultCode = Integer.parseInt(line.substring(22));
 702                         mGotExitValue = true;
 703                     } catch (NumberFormatException ex) {
 704                         CLog.w("Instrumentation code format unexpected");
 705                     }
 706                 } else if (mCurrentValue != null) {
 707                     mCurrentValue = mCurrentValue + line;
 708                 }
 709             }
 710         }
 711
 712         /**
 713          * {@inheritDoc}
 714          */
 715         @Override
 716         public void done() {
 717             if (mCurrentName != null) {
 718                 mValues.put(mCurrentName, mCurrentValue);
 719
 720                 mCurrentName = null;
 721                 mCurrentValue = null;
 722             }
 723
 724             if (mValues != null) {
 725                 mListener.handleStatus(mValues);
 726                 mValues = null;
 727             }
 728         }
 729
 730         /**
 731          * {@inheritDoc}
 732          */
 733         @Override
 734         public boolean isCancelled() {
 735             return false;
 736         }
 737
 738         /**
 739          * Returns whether target instrumentation exited normally.
 740          */
 741         public boolean wasSuccessful() {
 742             return mGotExitValue;
 743         }
 744
 745         /**
 746          * Returns Instrumentation return code
 747          */
 748         public int getResultCode() {
 749             return mResultCode;
 750         }
 751     }
 752
 753     /**
 754      * dEQP platfom query instrumentation parser
 755      */
 756     private static class PlatformQueryInstrumentationParser extends MultiLineReceiver {
 757         private Map<String,String> mResultMap = new LinkedHashMap<>();
 758         private int mResultCode;
 759         private boolean mGotExitValue = false;
 760
 761         /**
 762          * {@inheritDoc}
 763          */
 764         @Override
 765         public void processNewLines(String[] lines) {
 766             for (String line : lines) {
 767                 if (line.startsWith("INSTRUMENTATION_RESULT: ")) {
 768                     final String parts[] = line.substring(24).split("=",2);
 769                     if (parts.length == 2) {
 770                         mResultMap.put(parts[0], parts[1]);
 771                     } else {
 772                         CLog.w("Instrumentation status format unexpected");
 773                     }
 774                 } else if (line.startsWith("INSTRUMENTATION_CODE: ")) {
 775                     try {
 776                         mResultCode = Integer.parseInt(line.substring(22));
 777                         mGotExitValue = true;
 778                     } catch (NumberFormatException ex) {
 779                         CLog.w("Instrumentation code format unexpected");
 780                     }
 781                 }
 782             }
 783         }
 784
 785         /**
 786          * {@inheritDoc}
 787          */
 788         @Override
 789         public boolean isCancelled() {
 790             return false;
 791         }
 792
 793         /**
 794          * Returns whether target instrumentation exited normally.
 795          */
 796         public boolean wasSuccessful() {
 797             return mGotExitValue;
 798         }
 799
 800         /**
 801          * Returns Instrumentation return code
 802          */
 803         public int getResultCode() {
 804             return mResultCode;
 805         }
 806
 807         public Map<String,String> getResultMap() {
 808             return mResultMap;
 809         }
 810     }
 811
 812     /**
 813      * Interface for sleeping.
 814      *
 815      * Exposed for unit testing
 816      */
 817     public static interface ISleepProvider {
 818         public void sleep(int milliseconds);
 819     }
 820
 821     private static class SleepProvider implements ISleepProvider {
 822         public void sleep(int milliseconds) {
 823             try {
 824                 Thread.sleep(milliseconds);
 825             } catch (InterruptedException ex) {
 826             }
 827         }
 828     }
 829
 830     /**
 831      * Interface for failure recovery.
 832      *
 833      * Exposed for unit testing
 834      */
 835     public static interface IRecovery {
 836         /**
 837          * Sets the sleep provider IRecovery works on
 838          */
 839         public void setSleepProvider(ISleepProvider sleepProvider);
 840
 841         /**
 842          * Sets the device IRecovery works on
 843          */
 844         public void setDevice(ITestDevice device);
 845
 846         /**
 847          * Informs Recovery that test execution has progressed since the last recovery
 848          */
 849         public void onExecutionProgressed();
 850
 851         /**
 852          * Tries to recover device after failed refused connection.
 853          *
 854          * @throws DeviceNotAvailableException if recovery did not succeed
 855          */
 856         public void recoverConnectionRefused() throws DeviceNotAvailableException;
 857
 858         /**
 859          * Tries to recover device after abnormal execution termination or link failure.
 860          *
 861          * @param progressedSinceLastCall true if test execution has progressed since last call
 862          * @throws DeviceNotAvailableException if recovery did not succeed
 863          */
 864         public void recoverComLinkKilled() throws DeviceNotAvailableException;
 865     };
 866
 867     /**
 868      * State machine for execution failure recovery.
 869      *
 870      * Exposed for unit testing
 871      */
 872     public static class Recovery implements IRecovery {
 873         private int RETRY_COOLDOWN_MS = 6000; // 6 seconds
 874         private int PROCESS_KILL_WAIT_MS = 1000; // 1 second
 875
 876         private static enum MachineState {
 877             WAIT, // recover by waiting
 878             RECOVER, // recover by calling recover()
 879             REBOOT, // recover by rebooting
 880             FAIL, // cannot recover
 881         };
 882
 883         private MachineState mState = MachineState.WAIT;
 884         private ITestDevice mDevice;
 885         private ISleepProvider mSleepProvider;
 886
 887         private static class ProcessKillFailureException extends Exception {
 888         }
 889
 890         /**
 891          * {@inheritDoc}
 892          */
 893         public void setSleepProvider(ISleepProvider sleepProvider) {
 894             mSleepProvider = sleepProvider;
 895         }
 896
 897         /**
 898          * {@inheritDoc}
 899          */
 900         @Override
 901         public void setDevice(ITestDevice device) {
 902             mDevice = device;
 903         }
 904
 905         /**
 906          * {@inheritDoc}
 907          */
 908         @Override
 909         public void onExecutionProgressed() {
 910             mState = MachineState.WAIT;
 911         }
 912
 913         /**
 914          * {@inheritDoc}
 915          */
 916         @Override
 917         public void recoverConnectionRefused() throws DeviceNotAvailableException {
 918             switch (mState) {
 919                 case WAIT: // not a valid stratedy for connection refusal, fallthrough
 920                 case RECOVER:
 921                     // First failure, just try to recover
 922                     CLog.w("ADB connection failed, trying to recover");
 923                     mState = MachineState.REBOOT; // the next step is to reboot
 924
 925                     try {
 926                         recoverDevice();
 927                     } catch (DeviceNotAvailableException ex) {
 928                         // chain forward
 929                         recoverConnectionRefused();
 930                     }
 931                     break;
 932
 933                 case REBOOT:
 934                     // Second failure in a row, try to reboot
 935                     CLog.w("ADB connection failed after recovery, rebooting device");
 936                     mState = MachineState.FAIL; // the next step is to fail
 937
 938                     try {
 939                         rebootDevice();
 940                     } catch (DeviceNotAvailableException ex) {
 941                         // chain forward
 942                         recoverConnectionRefused();
 943                     }
 944                     break;
 945
 946                 case FAIL:
 947                     // Third failure in a row, just fail
 948                     CLog.w("Cannot recover ADB connection");
 949                     throw new DeviceNotAvailableException("failed to connect after reboot");
 950             }
 951         }
 952
 953         /**
 954          * {@inheritDoc}
 955          */
 956         @Override
 957         public void recoverComLinkKilled() throws DeviceNotAvailableException {
 958             switch (mState) {
 959                 case WAIT:
 960                     // First failure, just try to wait and try again
 961                     CLog.w("ADB link failed, retrying after a cooldown period");
 962                     mState = MachineState.RECOVER; // the next step is to recover the device
 963
 964                     waitCooldown();
 965
 966                     // even if the link to deqp on-device process was killed, the process might
 967                     // still be alive. Locate and terminate such unwanted processes.
 968                     try {
 969                         killDeqpProcess();
 970                     } catch (DeviceNotAvailableException ex) {
 971                         // chain forward
 972                         recoverComLinkKilled();
 973                     } catch (ProcessKillFailureException ex) {
 974                         // chain forward
 975                         recoverComLinkKilled();
 976                     }
 977                     break;
 978
 979                 case RECOVER:
 980                     // Second failure, just try to recover
 981                     CLog.w("ADB link failed, trying to recover");
 982                     mState = MachineState.REBOOT; // the next step is to reboot
 983
 984                     try {
 985                         recoverDevice();
 986                         killDeqpProcess();
 987                     } catch (DeviceNotAvailableException ex) {
 988                         // chain forward
 989                         recoverComLinkKilled();
 990                     } catch (ProcessKillFailureException ex) {
 991                         // chain forward
 992                         recoverComLinkKilled();
 993                     }
 994                     break;
 995
 996                 case REBOOT:
 997                     // Third failure in a row, try to reboot
 998                     CLog.w("ADB link failed after recovery, rebooting device");
 999                     mState = MachineState.FAIL; // the next step is to fail
1000
1001                     try {
1002                         rebootDevice();
1003                     } catch (DeviceNotAvailableException ex) {
1004                         // chain forward
1005                         recoverComLinkKilled();
1006                     }
1007                     break;
1008
1009                 case FAIL:
1010                     // Fourth failure in a row, just fail
1011                     CLog.w("Cannot recover ADB connection");
1012                     throw new DeviceNotAvailableException("link killed after reboot");
1013             }
1014         }
1015
1016         private void waitCooldown() {
1017             mSleepProvider.sleep(RETRY_COOLDOWN_MS);
1018         }
1019
1020         private Iterable<Integer> getDeqpProcessPids() throws DeviceNotAvailableException {
1021             final List<Integer> pids = new ArrayList<Integer>(2);
1022             final String processes = mDevice.executeShellCommand("ps | grep com.drawelements");
1023             final String[] lines = processes.split("(\\r|\\n)+");
1024             for (String line : lines) {
1025                 final String[] fields = line.split("\\s+");
1026                 if (fields.length < 2) {
1027                     continue;
1028                 }
1029
1030                 try {
1031                     final int processId = Integer.parseInt(fields[1], 10);
1032                     pids.add(processId);
1033                 } catch (NumberFormatException ex) {
1034                     continue;
1035                 }
1036             }
1037             return pids;
1038         }
1039
1040         private void killDeqpProcess() throws DeviceNotAvailableException,
1041                 ProcessKillFailureException {
1042             for (Integer processId : getDeqpProcessPids()) {
1043                 mDevice.executeShellCommand(String.format("kill -9 %d", processId));
1044             }
1045
1046             mSleepProvider.sleep(PROCESS_KILL_WAIT_MS);
1047
1048             // check that processes actually died
1049             if (getDeqpProcessPids().iterator().hasNext()) {
1050                 // a process is still alive, killing failed
1051                 throw new ProcessKillFailureException();
1052             }
1053         }
1054
1055         public void recoverDevice() throws DeviceNotAvailableException {
1056             // Work around the API. We need to call recoverDevice() on the test device and
1057             // we know that mDevice is a TestDevice. However even though the recoverDevice()
1058             // method is public suggesting it should be publicly accessible, the class itself
1059             // and its super-interface (IManagedTestDevice) are package-private.
1060             final Method recoverDeviceMethod;
1061             try {
1062                 recoverDeviceMethod = mDevice.getClass().getMethod("recoverDevice");
1063                 recoverDeviceMethod.setAccessible(true);
1064             } catch (NoSuchMethodException ex) {
1065                 throw new AssertionError("Test device must have recoverDevice()");
1066             }
1067
1068             try {
1069                 recoverDeviceMethod.invoke(mDevice);
1070             } catch (InvocationTargetException ex) {
1071                 if (ex.getCause() instanceof DeviceNotAvailableException) {
1072                     throw (DeviceNotAvailableException)ex.getCause();
1073                 } else if (ex.getCause() instanceof RuntimeException) {
1074                     throw (RuntimeException)ex.getCause();
1075                 } else {
1076                     throw new AssertionError("unexpected throw", ex);
1077                 }
1078             } catch (IllegalAccessException ex) {
1079                 throw new AssertionError("unexpected throw", ex);
1080             }
1081         }
1082
1083         private void rebootDevice() throws DeviceNotAvailableException {
1084             mDevice.reboot();
1085         }
1086     }
1087
1088     private static Map<TestIdentifier, Set<BatchRunConfiguration>> generateTestInstances(
1089             Reader testlist, String configName, String screenRotation, String surfaceType, boolean required) throws FileNotFoundException {
1090         // Note: This is specifically a LinkedHashMap to guarantee that tests are iterated
1091         // in the insertion order.
1092         final Map<TestIdentifier, Set<BatchRunConfiguration>> instances = new LinkedHashMap<>();
1093         try {
1094             BufferedReader testlistReader = new BufferedReader(testlist);
1095             String testName;
1096             while ((testName = testlistReader.readLine()) != null) {
1097                 // Test name -> testId -> only one config -> done.
1098                 final Set<BatchRunConfiguration> testInstanceSet = new LinkedHashSet<>();
1099                 BatchRunConfiguration config = new BatchRunConfiguration(configName, screenRotation, surfaceType, required);
1100                 testInstanceSet.add(config);
1101                 TestIdentifier test = pathToIdentifier(testName);
1102                 instances.put(test, testInstanceSet);
1103             }
1104             testlistReader.close();
1105         }
1106         catch (IOException e)
1107         {
1108             throw new RuntimeException("Failure while reading the test case list for deqp: " + e.getMessage());
1109         }
1110
1111         return instances;
1112     }
1113
1114     private Set<BatchRunConfiguration> getTestRunConfigs (TestIdentifier testId) {
1115         return mTestInstances.get(testId);
1116     }
1117
1118     /**
1119      * Converts dEQP testcase path to TestIdentifier.
1120      */
1121     private static TestIdentifier pathToIdentifier(String testPath) {
1122         int indexOfLastDot = testPath.lastIndexOf('.');
1123         String className = testPath.substring(0, indexOfLastDot);
1124         String testName = testPath.substring(indexOfLastDot+1);
1125
1126         return new TestIdentifier(className, testName);
1127     }
1128
1129     // \todo [2015-10-16 kalle] How unique should this be?
1130     private String getId() {
1131         return AbiUtils.createId(mAbi.getName(), mDeqpPackage);
1132     }
1133
1134     /**
1135      * Generates tescase trie from dEQP testcase paths. Used to define which testcases to execute.
1136      */
1137     private static String generateTestCaseTrieFromPaths(Collection<String> tests) {
1138         String result = "{";
1139         boolean first = true;
1140
1141         // Add testcases to results
1142         for (Iterator<String> iter = tests.iterator(); iter.hasNext();) {
1143             String test = iter.next();
1144             String[] components = test.split("\\.");
1145
1146             if (components.length == 1) {
1147                 if (!first) {
1148                     result = result + ",";
1149                 }
1150                 first = false;
1151
1152                 result += components[0];
1153                 iter.remove();
1154             }
1155         }
1156
1157         if (!tests.isEmpty()) {
1158             HashMap<String, ArrayList<String> > testGroups = new HashMap<>();
1159
1160             // Collect all sub testgroups
1161             for (String test : tests) {
1162                 String[] components = test.split("\\.");
1163                 ArrayList<String> testGroup = testGroups.get(components[0]);
1164
1165                 if (testGroup == null) {
1166                     testGroup = new ArrayList<String>();
1167                     testGroups.put(components[0], testGroup);
1168                 }
1169
1170                 testGroup.add(test.substring(components[0].length()+1));
1171             }
1172
1173             for (String testGroup : testGroups.keySet()) {
1174                 if (!first) {
1175                     result = result + ",";
1176                 }
1177
1178                 first = false;
1179                 result = result + testGroup
1180                         + generateTestCaseTrieFromPaths(testGroups.get(testGroup));
1181             }
1182         }
1183
1184         return result + "}";
1185     }
1186
1187     /**
1188      * Generates testcase trie from TestIdentifiers.
1189      */
1190     private static String generateTestCaseTrie(Collection<TestIdentifier> tests) {
1191         ArrayList<String> testPaths = new ArrayList<String>();
1192
1193         for (TestIdentifier test : tests) {
1194             testPaths.add(test.getClassName() + "." + test.getTestName());
1195         }
1196
1197         return generateTestCaseTrieFromPaths(testPaths);
1198     }
1199
1200     private static class TestBatch {
1201         public BatchRunConfiguration config;
1202         public List<TestIdentifier> tests;
1203     }
1204
1205     /**
1206      * Creates a TestBatch from the given tests or null if not tests remaining.
1207      *
1208      *  @param pool List of tests to select from
1209      *  @param requiredConfig Select only instances with pending requiredConfig, or null to select
1210      *         any run configuration.
1211      */
1212     private TestBatch selectRunBatch(Collection<TestIdentifier> pool,
1213             BatchRunConfiguration requiredConfig) {
1214         // select one test (leading test) that is going to be executed and then pack along as many
1215         // other compatible instances as possible.
1216
1217         TestIdentifier leadingTest = null;
1218         for (TestIdentifier test : pool) {
1219             if (!mRemainingTests.contains(test)) {
1220                 continue;
1221             }
1222             if (requiredConfig != null &&
1223                     !mInstanceListerner.isPendingTestInstance(test, requiredConfig)) {
1224                 continue;
1225             }
1226             leadingTest = test;
1227             break;
1228         }
1229
1230         // no remaining tests?
1231         if (leadingTest == null) {
1232             return null;
1233         }
1234
1235         BatchRunConfiguration leadingTestConfig = null;
1236         if (requiredConfig != null) {
1237             leadingTestConfig = requiredConfig;
1238         } else {
1239             for (BatchRunConfiguration runConfig : getTestRunConfigs(leadingTest)) {
1240                 if (mInstanceListerner.isPendingTestInstance(leadingTest, runConfig)) {
1241                     leadingTestConfig = runConfig;
1242                     break;
1243                 }
1244             }
1245         }
1246
1247         // test pending <=> test has a pending config
1248         if (leadingTestConfig == null) {
1249             throw new AssertionError("search postcondition failed");
1250         }
1251
1252         final int leadingInstability = getTestInstabilityRating(leadingTest);
1253
1254         final TestBatch runBatch = new TestBatch();
1255         runBatch.config = leadingTestConfig;
1256         runBatch.tests = new ArrayList<>();
1257         runBatch.tests.add(leadingTest);
1258
1259         for (TestIdentifier test : pool) {
1260             if (test == leadingTest) {
1261                 // do not re-select the leading tests
1262                 continue;
1263             }
1264             if (!mInstanceListerner.isPendingTestInstance(test, leadingTestConfig)) {
1265                 // select only compatible
1266                 continue;
1267             }
1268             if (getTestInstabilityRating(test) != leadingInstability) {
1269                 // pack along only cases in the same stability category. Packing more dangerous
1270                 // tests along jeopardizes the stability of this run. Packing more stable tests
1271                 // along jeopardizes their stability rating.
1272                 continue;
1273             }
1274             if (runBatch.tests.size() >= getBatchSizeLimitForInstability(leadingInstability)) {
1275                 // batch size is limited.
1276                 break;
1277             }
1278             runBatch.tests.add(test);
1279         }
1280
1281         return runBatch;
1282     }
1283
1284     private int getBatchNumPendingCases(TestBatch batch) {
1285         int numPending = 0;
1286         for (TestIdentifier test : batch.tests) {
1287             if (mInstanceListerner.isPendingTestInstance(test, batch.config)) {
1288                 ++numPending;
1289             }
1290         }
1291         return numPending;
1292     }
1293
1294     private int getBatchSizeLimitForInstability(int batchInstabilityRating) {
1295         // reduce group size exponentially down to one
1296         return Math.max(1, TESTCASE_BATCH_LIMIT / (1 << batchInstabilityRating));
1297     }
1298
1299     private int getTestInstabilityRating(TestIdentifier testId) {
1300         if (mTestInstabilityRatings.containsKey(testId)) {
1301             return mTestInstabilityRatings.get(testId);
1302         } else {
1303             return 0;
1304         }
1305     }
1306
1307     private void recordTestInstability(TestIdentifier testId) {
1308         mTestInstabilityRatings.put(testId, getTestInstabilityRating(testId) + 1);
1309     }
1310
1311     private void clearTestInstability(TestIdentifier testId) {
1312         mTestInstabilityRatings.put(testId, 0);
1313     }
1314
1315     /**
1316      * Executes all tests on the device.
1317      */
1318     private void runTests() throws DeviceNotAvailableException, CapabilityQueryFailureException {
1319         for (;;) {
1320             TestBatch batch = selectRunBatch(mRemainingTests, null);
1321
1322             if (batch == null) {
1323                 break;
1324             }
1325
1326             runTestRunBatch(batch);
1327         }
1328     }
1329
1330     /**
1331      * Runs a TestBatch by either faking it or executing it on a device.
1332      */
1333     private void runTestRunBatch(TestBatch batch) throws DeviceNotAvailableException,
1334             CapabilityQueryFailureException {
1335         // prepare instance listener
1336         mInstanceListerner.setCurrentConfig(batch.config);
1337         for (TestIdentifier test : batch.tests) {
1338             mInstanceListerner.setTestInstances(test, getTestRunConfigs(test));
1339         }
1340
1341         // execute only if config is executable, else fake results
1342         if (isSupportedRunConfiguration(batch.config)) {
1343             executeTestRunBatch(batch);
1344         } else {
1345             if (batch.config.isRequired()) {
1346                 fakeFailTestRunBatch(batch);
1347             } else {
1348                 fakePassTestRunBatch(batch);
1349             }
1350         }
1351     }
1352
1353     private boolean isSupportedRunConfiguration(BatchRunConfiguration runConfig)
1354             throws DeviceNotAvailableException, CapabilityQueryFailureException {
1355         // orientation support
1356         if (!BatchRunConfiguration.ROTATION_UNSPECIFIED.equals(runConfig.getRotation())) {
1357             final Set<String> features = getDeviceFeatures(mDevice);
1358
1359             if (isPortraitClassRotation(runConfig.getRotation()) &&
1360                     !features.contains(FEATURE_PORTRAIT)) {
1361                 return false;
1362             }
1363             if (isLandscapeClassRotation(runConfig.getRotation()) &&
1364                     !features.contains(FEATURE_LANDSCAPE)) {
1365                 return false;
1366             }
1367         }
1368
1369         if (isOpenGlEsPackage()) {
1370             // renderability support for OpenGL ES tests
1371             return isSupportedGlesRenderConfig(runConfig);
1372         } else {
1373             return true;
1374         }
1375     }
1376
1377     private static final class AdbComLinkOpenError extends Exception {
1378         public AdbComLinkOpenError(String description, Throwable inner) {
1379             super(description, inner);
1380         }
1381     }
1382
1383     private static final class AdbComLinkKilledError extends Exception {
1384         public AdbComLinkKilledError(String description, Throwable inner) {
1385             super(description, inner);
1386         }
1387     }
1388
1389     /**
1390      * Executes a given command in adb shell
1391      *
1392      * @throws AdbComLinkOpenError if connection cannot be established.
1393      * @throws AdbComLinkKilledError if established connection is killed prematurely.
1394      */
1395     private void executeShellCommandAndReadOutput(final String command,
1396             final IShellOutputReceiver receiver)
1397             throws AdbComLinkOpenError, AdbComLinkKilledError {
1398         try {
1399             mDevice.getIDevice().executeShellCommand(command, receiver,
1400                     UNRESPONSIVE_CMD_TIMEOUT_MS, TimeUnit.MILLISECONDS);
1401         } catch (TimeoutException ex) {
1402             // Opening connection timed out
1403             throw new AdbComLinkOpenError("opening connection timed out", ex);
1404         } catch (AdbCommandRejectedException ex) {
1405             // Command rejected
1406             throw new AdbComLinkOpenError("command rejected", ex);
1407         } catch (IOException ex) {
1408             // shell command channel killed
1409             throw new AdbComLinkKilledError("command link killed", ex);
1410         } catch (ShellCommandUnresponsiveException ex) {
1411             // shell command halted
1412             throw new AdbComLinkKilledError("command link hung", ex);
1413         }
1414     }
1415
1416     /**
1417      * Executes given test batch on a device
1418      */
1419     private void executeTestRunBatch(TestBatch batch) throws DeviceNotAvailableException {
1420         // attempt full run once
1421         executeTestRunBatchRun(batch);
1422
1423         // split remaining tests to two sub batches and execute both. This will terminate
1424         // since executeTestRunBatchRun will always progress for a batch of size 1.
1425         final ArrayList<TestIdentifier> pendingTests = new ArrayList<>();
1426
1427         for (TestIdentifier test : batch.tests) {
1428             if (mInstanceListerner.isPendingTestInstance(test, batch.config)) {
1429                 pendingTests.add(test);
1430             }
1431         }
1432
1433         final int divisorNdx = pendingTests.size() / 2;
1434         final List<TestIdentifier> headList = pendingTests.subList(0, divisorNdx);
1435         final List<TestIdentifier> tailList = pendingTests.subList(divisorNdx, pendingTests.size());
1436
1437         // head
1438         for (;;) {
1439             TestBatch subBatch = selectRunBatch(headList, batch.config);
1440
1441             if (subBatch == null) {
1442                 break;
1443             }
1444
1445             executeTestRunBatch(subBatch);
1446         }
1447
1448         // tail
1449         for (;;) {
1450             TestBatch subBatch = selectRunBatch(tailList, batch.config);
1451
1452             if (subBatch == null) {
1453                 break;
1454             }
1455
1456             executeTestRunBatch(subBatch);
1457         }
1458
1459         if (getBatchNumPendingCases(batch) != 0) {
1460             throw new AssertionError("executeTestRunBatch postcondition failed");
1461         }
1462     }
1463
1464     /**
1465      * Runs one execution pass over the given batch.
1466      *
1467      * Tries to run the batch. Always makes progress (executes instances or modifies stability
1468      * scores).
1469      */
1470     private void executeTestRunBatchRun(TestBatch batch) throws DeviceNotAvailableException {
1471         if (getBatchNumPendingCases(batch) != batch.tests.size()) {
1472             throw new AssertionError("executeTestRunBatchRun precondition failed");
1473         }
1474
1475         checkInterrupted(); // throws if interrupted
1476
1477         final String testCases = generateTestCaseTrie(batch.tests);
1478
1479         mDevice.executeShellCommand("rm " + CASE_LIST_FILE_NAME);
1480         mDevice.executeShellCommand("rm " + LOG_FILE_NAME);
1481         mDevice.pushString(testCases + "\n", CASE_LIST_FILE_NAME);
1482
1483         final String instrumentationName =
1484                 "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
1485
1486         final StringBuilder deqpCmdLine = new StringBuilder();
1487         deqpCmdLine.append("--deqp-caselist-file=");
1488         deqpCmdLine.append(CASE_LIST_FILE_NAME);
1489         deqpCmdLine.append(" ");
1490         deqpCmdLine.append(getRunConfigDisplayCmdLine(batch.config));
1491
1492         // If we are not logging data, do not bother outputting the images from the test exe.
1493         if (!mLogData) {
1494             deqpCmdLine.append(" --deqp-log-images=disable");
1495         }
1496
1497         deqpCmdLine.append(" --deqp-watchdog=enable");
1498
1499         final String command = String.format(
1500                 "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\""
1501                     + " -e deqpLogData \"%s\" %s",
1502                 AbiUtils.createAbiFlag(mAbi.getName()), LOG_FILE_NAME, deqpCmdLine.toString(),
1503                 mLogData, instrumentationName);
1504
1505         final int numRemainingInstancesBefore = getNumRemainingInstances();
1506         final InstrumentationParser parser = new InstrumentationParser(mInstanceListerner);
1507         Throwable interruptingError = null;
1508
1509         try {
1510             executeShellCommandAndReadOutput(command, parser);
1511         } catch (Throwable ex) {
1512             interruptingError = ex;
1513         } finally {
1514             parser.flush();
1515         }
1516
1517         final boolean progressedSinceLastCall = mInstanceListerner.getCurrentTestId() != null ||
1518                 getNumRemainingInstances() < numRemainingInstancesBefore;
1519
1520         if (progressedSinceLastCall) {
1521             mDeviceRecovery.onExecutionProgressed();
1522         }
1523
1524         // interrupted, try to recover
1525         if (interruptingError != null) {
1526             if (interruptingError instanceof AdbComLinkOpenError) {
1527                 mDeviceRecovery.recoverConnectionRefused();
1528             } else if (interruptingError instanceof AdbComLinkKilledError) {
1529                 mDeviceRecovery.recoverComLinkKilled();
1530             } else if (interruptingError instanceof RunInterruptedException) {
1531                 // external run interruption request. Terminate immediately.
1532                 throw (RunInterruptedException)interruptingError;
1533             } else {
1534                 CLog.e(interruptingError);
1535                 throw new RuntimeException(interruptingError);
1536             }
1537
1538             // recoverXXX did not throw => recovery succeeded
1539         } else if (!parser.wasSuccessful()) {
1540             mDeviceRecovery.recoverComLinkKilled();
1541             // recoverXXX did not throw => recovery succeeded
1542         }
1543
1544         // Progress guarantees.
1545         if (batch.tests.size() == 1) {
1546             final TestIdentifier onlyTest = batch.tests.iterator().next();
1547             final boolean wasTestExecuted =
1548                     !mInstanceListerner.isPendingTestInstance(onlyTest, batch.config) &&
1549                     mInstanceListerner.getCurrentTestId() == null;
1550             final boolean wasLinkFailure = !parser.wasSuccessful() || interruptingError != null;
1551
1552             // Link failures can be caused by external events, require at least two observations
1553             // until bailing.
1554             if (!wasTestExecuted && (!wasLinkFailure || getTestInstabilityRating(onlyTest) > 0)) {
1555                 recordTestInstability(onlyTest);
1556                 // If we cannot finish the test, mark the case as a crash.
1557                 //
1558                 // If we couldn't even start the test, fail the test instance as non-executable.
1559                 // This is required so that a consistently crashing or non-existent tests will
1560                 // not cause futile (non-terminating) re-execution attempts.
1561                 if (mInstanceListerner.getCurrentTestId() != null) {
1562                     mInstanceListerner.abortTest(onlyTest, INCOMPLETE_LOG_MESSAGE);
1563                 } else {
1564                     mInstanceListerner.abortTest(onlyTest, NOT_EXECUTABLE_LOG_MESSAGE);
1565                 }
1566             } else if (wasTestExecuted) {
1567                 clearTestInstability(onlyTest);
1568             }
1569         }
1570         else
1571         {
1572             // Analyze results to update test stability ratings. If there is no interrupting test
1573             // logged, increase instability rating of all remaining tests. If there is a
1574             // interrupting test logged, increase only its instability rating.
1575             //
1576             // A successful run of tests clears instability rating.
1577             if (mInstanceListerner.getCurrentTestId() == null) {
1578                 for (TestIdentifier test : batch.tests) {
1579                     if (mInstanceListerner.isPendingTestInstance(test, batch.config)) {
1580                         recordTestInstability(test);
1581                     } else {
1582                         clearTestInstability(test);
1583                     }
1584                 }
1585             } else {
1586                 recordTestInstability(mInstanceListerner.getCurrentTestId());
1587                 for (TestIdentifier test : batch.tests) {
1588                     // \note: isPendingTestInstance is false for getCurrentTestId. Current ID is
1589                     // considered 'running' and will be restored to 'pending' in endBatch().
1590                     if (!test.equals(mInstanceListerner.getCurrentTestId()) &&
1591                             !mInstanceListerner.isPendingTestInstance(test, batch.config)) {
1592                         clearTestInstability(test);
1593                     }
1594                 }
1595             }
1596         }
1597
1598         mInstanceListerner.endBatch();
1599     }
1600
1601     private static String getRunConfigDisplayCmdLine(BatchRunConfiguration runConfig) {
1602         final StringBuilder deqpCmdLine = new StringBuilder();
1603         if (!runConfig.getGlConfig().isEmpty()) {
1604             deqpCmdLine.append("--deqp-gl-config-name=");
1605             deqpCmdLine.append(runConfig.getGlConfig());
1606         }
1607         if (!runConfig.getRotation().isEmpty()) {
1608             if (deqpCmdLine.length() != 0) {
1609                 deqpCmdLine.append(" ");
1610             }
1611             deqpCmdLine.append("--deqp-screen-rotation=");
1612             deqpCmdLine.append(runConfig.getRotation());
1613         }
1614         if (!runConfig.getSurfaceType().isEmpty()) {
1615             if (deqpCmdLine.length() != 0) {
1616                 deqpCmdLine.append(" ");
1617             }
1618             deqpCmdLine.append("--deqp-surface-type=");
1619             deqpCmdLine.append(runConfig.getSurfaceType());
1620         }
1621         return deqpCmdLine.toString();
1622     }
1623
1624     private int getNumRemainingInstances() {
1625         int retVal = 0;
1626         for (TestIdentifier testId : mRemainingTests) {
1627             // If case is in current working set, sum only not yet executed instances.
1628             // If case is not in current working set, sum all instances (since they are not yet
1629             // executed).
1630             if (mInstanceListerner.mPendingResults.containsKey(testId)) {
1631                 retVal += mInstanceListerner.mPendingResults.get(testId).remainingConfigs.size();
1632             } else {
1633                 retVal += mTestInstances.get(testId).size();
1634             }
1635         }
1636         return retVal;
1637     }
1638
1639     /**
1640      * Checks if this execution has been marked as interrupted and throws if it has.
1641      */
1642     private void checkInterrupted() throws RunInterruptedException {
1643         // Work around the API. RunUtil::checkInterrupted is private but we can call it indirectly
1644         // by sleeping a value <= 0.
1645         mRunUtil.sleep(0);
1646     }
1647
1648     /**
1649      * Pass given batch tests without running it
1650      */
1651     private void fakePassTestRunBatch(TestBatch batch) {
1652         for (TestIdentifier test : batch.tests) {
1653             CLog.d("Marking '%s' invocation in config '%s' as passed without running", test.toString(),
1654                     batch.config.getId());
1655             mInstanceListerner.skipTest(test);
1656         }
1657     }
1658
1659     /**
1660      * Fail given batch tests without running it
1661      */
1662     private void fakeFailTestRunBatch(TestBatch batch) {
1663         for (TestIdentifier test : batch.tests) {
1664             CLog.d("Marking '%s' invocation in config '%s' as failed without running", test.toString(),
1665                     batch.config.getId());
1666             mInstanceListerner.abortTest(test, "Required config not supported");
1667         }
1668     }
1669
1670     /**
1671      * Pass all remaining tests without running them
1672      */
1673     private void fakePassTests(ITestInvocationListener listener) {
1674         Map <String, String> emptyMap = Collections.emptyMap();
1675         for (TestIdentifier test : mRemainingTests) {
1676             CLog.d("Skipping test '%s', Opengl ES version not supported", test.toString());
1677             listener.testStarted(test);
1678             listener.testEnded(test, emptyMap);
1679         }
1680         mRemainingTests.clear();
1681     }
1682
1683     /**
1684      * Check if device supports Vulkan.
1685      */
1686     private boolean isSupportedVulkan ()
1687             throws DeviceNotAvailableException, CapabilityQueryFailureException {
1688         final Set<String> features = getDeviceFeatures(mDevice);
1689
1690         for (String feature : features) {
1691             if (feature.startsWith(FEATURE_VULKAN_LEVEL)) {
1692                 return true;
1693             }
1694         }
1695
1696         return false;
1697     }
1698
1699     /**
1700      * Check if device supports OpenGL ES version.
1701      */
1702     private static boolean isSupportedGles(ITestDevice device, int requiredMajorVersion,
1703             int requiredMinorVersion) throws DeviceNotAvailableException {
1704         String roOpenglesVersion = device.getProperty("ro.opengles.version");
1705
1706         if (roOpenglesVersion == null)
1707             return false;
1708
1709         int intValue = Integer.parseInt(roOpenglesVersion);
1710
1711         int majorVersion = ((intValue & 0xffff0000) >> 16);
1712         int minorVersion = (intValue & 0xffff);
1713
1714         return (majorVersion > requiredMajorVersion)
1715                 || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion);
1716     }
1717
1718     /**
1719      * Query if rendertarget is supported
1720      */
1721     private boolean isSupportedGlesRenderConfig(BatchRunConfiguration runConfig)
1722             throws DeviceNotAvailableException, CapabilityQueryFailureException {
1723         // query if configuration is supported
1724         final StringBuilder configCommandLine =
1725                 new StringBuilder(getRunConfigDisplayCmdLine(runConfig));
1726         if (configCommandLine.length() != 0) {
1727             configCommandLine.append(" ");
1728         }
1729         configCommandLine.append("--deqp-gl-major-version=");
1730         configCommandLine.append(getGlesMajorVersion());
1731         configCommandLine.append(" --deqp-gl-minor-version=");
1732         configCommandLine.append(getGlesMinorVersion());
1733
1734         final String commandLine = configCommandLine.toString();
1735
1736         // check for cached result first
1737         if (mConfigQuerySupportCache.containsKey(commandLine)) {
1738             return mConfigQuerySupportCache.get(commandLine);
1739         }
1740
1741         final boolean supported = queryIsSupportedConfigCommandLine(commandLine);
1742         mConfigQuerySupportCache.put(commandLine, supported);
1743         return supported;
1744     }
1745
1746     private boolean queryIsSupportedConfigCommandLine(String deqpCommandLine)
1747             throws DeviceNotAvailableException, CapabilityQueryFailureException {
1748         final String instrumentationName =
1749                 "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation";
1750         final String command = String.format(
1751                 "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine \"%s\""
1752                     + " %s",
1753                 AbiUtils.createAbiFlag(mAbi.getName()), deqpCommandLine, instrumentationName);
1754
1755         final PlatformQueryInstrumentationParser parser = new PlatformQueryInstrumentationParser();
1756         mDevice.executeShellCommand(command, parser);
1757         parser.flush();
1758
1759         if (parser.wasSuccessful() && parser.getResultCode() == 0 &&
1760                 parser.getResultMap().containsKey("Supported")) {
1761             if ("Yes".equals(parser.getResultMap().get("Supported"))) {
1762                 return true;
1763             } else if ("No".equals(parser.getResultMap().get("Supported"))) {
1764                 return false;
1765             } else {
1766                 CLog.e("Capability query did not return a result");
1767                 throw new CapabilityQueryFailureException();
1768             }
1769         } else if (parser.wasSuccessful()) {
1770             CLog.e("Failed to run capability query. Code: %d, Result: %s",
1771                     parser.getResultCode(), parser.getResultMap().toString());
1772             throw new CapabilityQueryFailureException();
1773         } else {
1774             CLog.e("Failed to run capability query");
1775             throw new CapabilityQueryFailureException();
1776         }
1777     }
1778
1779     /**
1780      * Return feature set supported by the device
1781      */
1782     private Set<String> getDeviceFeatures(ITestDevice device)
1783             throws DeviceNotAvailableException, CapabilityQueryFailureException {
1784         if (mDeviceFeatures == null) {
1785             mDeviceFeatures = queryDeviceFeatures(device);
1786         }
1787         return mDeviceFeatures;
1788     }
1789
1790     /**
1791      * Query feature set supported by the device
1792      */
1793     private static Set<String> queryDeviceFeatures(ITestDevice device)
1794             throws DeviceNotAvailableException, CapabilityQueryFailureException {
1795         // NOTE: Almost identical code in BaseDevicePolicyTest#hasDeviceFeatures
1796         // TODO: Move this logic to ITestDevice.
1797         String command = "pm list features";
1798         String commandOutput = device.executeShellCommand(command);
1799
1800         // Extract the id of the new user.
1801         HashSet<String> availableFeatures = new HashSet<>();
1802         for (String feature: commandOutput.split("\\s+")) {
1803             // Each line in the output of the command has the format "feature:{FEATURE_VALUE}".
1804             String[] tokens = feature.split(":");
1805             if (tokens.length < 2 || !"feature".equals(tokens[0])) {
1806                 CLog.e("Failed parse features. Unexpect format on line \"%s\"", tokens[0]);
1807                 throw new CapabilityQueryFailureException();
1808             }
1809             availableFeatures.add(tokens[1]);
1810         }
1811         return availableFeatures;
1812     }
1813
1814     private boolean isPortraitClassRotation(String rotation) {
1815         return BatchRunConfiguration.ROTATION_PORTRAIT.equals(rotation) ||
1816                 BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT.equals(rotation);
1817     }
1818
1819     private boolean isLandscapeClassRotation(String rotation) {
1820         return BatchRunConfiguration.ROTATION_LANDSCAPE.equals(rotation) ||
1821                 BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE.equals(rotation);
1822     }
1823
1824     /**
1825      * Install dEQP OnDevice Package
1826      */
1827     private void installTestApk() throws DeviceNotAvailableException {
1828         try {
1829             File apkFile = new File(mBuildHelper.getTestsDir(), DEQP_ONDEVICE_APK);
1830             String[] options = {AbiUtils.createAbiFlag(mAbi.getName())};
1831             String errorCode = getDevice().installPackage(apkFile, true, options);
1832             if (errorCode != null) {
1833                 CLog.e("Failed to install %s. Reason: %s", DEQP_ONDEVICE_APK, errorCode);
1834             }
1835         } catch (FileNotFoundException e) {
1836             CLog.e("Could not find test apk %s", DEQP_ONDEVICE_APK);
1837         }
1838     }
1839
1840     /**
1841      * Uninstall dEQP OnDevice Package
1842      */
1843     private void uninstallTestApk() throws DeviceNotAvailableException {
1844         getDevice().uninstallPackage(DEQP_ONDEVICE_PKG);
1845     }
1846
1847     /**
1848      * Parse gl nature from package name
1849      */
1850     private boolean isOpenGlEsPackage() {
1851         if ("dEQP-GLES2".equals(mDeqpPackage) || "dEQP-GLES3".equals(mDeqpPackage) ||
1852                 "dEQP-GLES31".equals(mDeqpPackage)) {
1853             return true;
1854         } else if ("dEQP-EGL".equals(mDeqpPackage) ||
1855                 "dEQP-VK".equals(mDeqpPackage)) {
1856             return false;
1857         } else {
1858             throw new IllegalStateException("dEQP runner was created with illegal name");
1859         }
1860     }
1861
1862     /**
1863      * Parse vulkan nature from package name
1864      */
1865     private boolean isVulkanPackage() {
1866         if ("dEQP-GLES2".equals(mDeqpPackage) || "dEQP-GLES3".equals(mDeqpPackage) ||
1867                 "dEQP-GLES31".equals(mDeqpPackage) || "dEQP-EGL".equals(mDeqpPackage)) {
1868             return false;
1869         } else if ("dEQP-VK".equals(mDeqpPackage)) {
1870             return true;
1871         } else {
1872             throw new IllegalStateException("dEQP runner was created with illegal name");
1873         }
1874     }
1875
1876     /**
1877      * Check GL support (based on package name)
1878      */
1879     private boolean isSupportedGles() throws DeviceNotAvailableException {
1880         return isSupportedGles(mDevice, getGlesMajorVersion(), getGlesMinorVersion());
1881     }
1882
1883     /**
1884      * Get GL major version (based on package name)
1885      */
1886     private int getGlesMajorVersion() throws DeviceNotAvailableException {
1887         if ("dEQP-GLES2".equals(mDeqpPackage)) {
1888             return 2;
1889         } else if ("dEQP-GLES3".equals(mDeqpPackage)) {
1890             return 3;
1891         } else if ("dEQP-GLES31".equals(mDeqpPackage)) {
1892             return 3;
1893         } else {
1894             throw new IllegalStateException("getGlesMajorVersion called for non gles pkg");
1895         }
1896     }
1897
1898     /**
1899      * Get GL minor version (based on package name)
1900      */
1901     private int getGlesMinorVersion() throws DeviceNotAvailableException {
1902         if ("dEQP-GLES2".equals(mDeqpPackage)) {
1903             return 0;
1904         } else if ("dEQP-GLES3".equals(mDeqpPackage)) {
1905             return 0;
1906         } else if ("dEQP-GLES31".equals(mDeqpPackage)) {
1907             return 1;
1908         } else {
1909             throw new IllegalStateException("getGlesMinorVersion called for non gles pkg");
1910         }
1911     }
1912
1913     private static List<Pattern> getPatternFilters(List<String> filters) {
1914         List<Pattern> patterns = new ArrayList<Pattern>();
1915         for (String filter : filters) {
1916             if (filter.contains("*")) {
1917                 patterns.add(Pattern.compile(filter.replace(".","\\.").replace("*",".*")));
1918             }
1919         }
1920         return patterns;
1921     }
1922
1923     private static Set<String> getNonPatternFilters(List<String> filters) {
1924         Set<String> nonPatternFilters = new HashSet<String>();
1925         for (String filter : filters) {
1926             if (!filter.contains("*")) {
1927                 nonPatternFilters.add(filter);
1928             }
1929         }
1930         return nonPatternFilters;
1931     }
1932
1933     private static boolean matchesAny(TestIdentifier test, List<Pattern> patterns) {
1934         for (Pattern pattern : patterns) {
1935             if (pattern.matcher(test.toString()).matches()) {
1936                 return true;
1937             }
1938         }
1939         return false;
1940     }
1941
1942     /**
1943      * Filter tests with the option of filtering by pattern.
1944      *
1945      * '*' is 0 or more characters.
1946      * '.' is interpreted verbatim.
1947      */
1948     private static void filterTests(Map<TestIdentifier, Set<BatchRunConfiguration>> tests,
1949                                     List<String> includeFilters,
1950                                     List<String> excludeFilters) {
1951         // We could filter faster by building the test case tree.
1952         // Let's see if this is fast enough.
1953         Set<String> includeStrings = getNonPatternFilters(includeFilters);
1954         Set<String> excludeStrings = getNonPatternFilters(excludeFilters);
1955         List<Pattern> includePatterns = getPatternFilters(includeFilters);
1956         List<Pattern> excludePatterns = getPatternFilters(excludeFilters);
1957
1958         List<TestIdentifier> testList = new ArrayList(tests.keySet());
1959         for (TestIdentifier test : testList) {
1960             if (excludeStrings.contains(test.toString())) {
1961                 tests.remove(test); // remove test if explicitly excluded
1962                 continue;
1963             }
1964             boolean includesExist = !includeStrings.isEmpty() || !includePatterns.isEmpty();
1965             boolean testIsIncluded = includeStrings.contains(test.toString())
1966                     || matchesAny(test, includePatterns);
1967             if ((includesExist && !testIsIncluded) || matchesAny(test, excludePatterns)) {
1968                 // if this test isn't included and other tests are,
1969                 // or if test matches exclude pattern, exclude test
1970                 tests.remove(test);
1971             }
1972         }
1973     }
1974
1975     /**
1976      * Loads tests into mTestInstances based on the options. Assumes
1977      * that no tests have been loaded for this instance before.
1978      */
1979     private void loadTests() {
1980         if (mTestInstances != null) throw new AssertionError("Re-load of tests not supported");
1981
1982         try {
1983             Reader reader = mCaselistReader;
1984             if (reader == null) {
1985                 File testlist = new File(mBuildHelper.getTestsDir(), mCaselistFile);
1986                 if (!testlist.isFile()) {
1987                     throw new FileNotFoundException();
1988                 }
1989                 reader = new FileReader(testlist);
1990             }
1991             mTestInstances = generateTestInstances(reader, mConfigName, mScreenRotation, mSurfaceType, mConfigRequired);
1992             mCaselistReader = null;
1993             reader.close();
1994         }
1995         catch (FileNotFoundException e) {
1996             throw new RuntimeException("Cannot read deqp test list file: "  + mCaselistFile);
1997         }
1998         catch (IOException e) {
1999             CLog.w("Failed to close test list reader.");
2000         }
2001         CLog.d("Filters");
2002         for (String filter : mIncludeFilters) {
2003             CLog.d("Include: %s", filter);
2004         }
2005         for (String filter : mExcludeFilters) {
2006             CLog.d("Exclude: %s", filter);
2007         }
2008
2009         long originalTestCount = mTestInstances.size();
2010         CLog.i("Num tests before filtering: %d", originalTestCount);
2011         if ((!mIncludeFilters.isEmpty() || !mExcludeFilters.isEmpty()) && originalTestCount > 0) {
2012             filterTests(mTestInstances, mIncludeFilters, mExcludeFilters);
2013
2014             // Update runtime estimation hint.
2015             if (mRuntimeHint != -1) {
2016                 mRuntimeHint = (mRuntimeHint * mTestInstances.size()) / originalTestCount;
2017             }
2018         }
2019         CLog.i("Num tests after filtering: %d", mTestInstances.size());
2020     }
2021
2022     /**
2023      * {@inheritDoc}
2024      */
2025     @Override
2026     public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
2027         final Map<String, String> emptyMap = Collections.emptyMap();
2028         // If sharded, split() will load the tests.
2029         if (mTestInstances == null) {
2030             loadTests();
2031         }
2032
2033         mRemainingTests = new LinkedList<>(mTestInstances.keySet());
2034         long startTime = System.currentTimeMillis();
2035         listener.testRunStarted(getId(), mRemainingTests.size());
2036
2037         try {
2038             final boolean isSupportedApi = (isOpenGlEsPackage() && isSupportedGles())
2039                                             || (isVulkanPackage() && isSupportedVulkan())
2040                                             || (!isOpenGlEsPackage() && !isVulkanPackage());
2041
2042             if (!isSupportedApi || mCollectTestsOnly) {
2043                 // Pass all tests if OpenGL ES version is not supported or we are collecting
2044                 // the names of the tests only
2045                 fakePassTests(listener);
2046             } else if (!mRemainingTests.isEmpty()) {
2047                 // Make sure there is no pre-existing package form earlier interrupted test run.
2048                 uninstallTestApk();
2049                 installTestApk();
2050
2051                 mInstanceListerner.setSink(listener);
2052                 mDeviceRecovery.setDevice(mDevice);
2053                 runTests();
2054
2055                 uninstallTestApk();
2056             }
2057         } catch (CapabilityQueryFailureException ex) {
2058             // Platform is not behaving correctly, for example crashing when trying to create
2059             // a window. Instead of silenty failing, signal failure by leaving the rest of the
2060             // test cases in "NotExecuted" state
2061             CLog.e("Capability query failed - leaving tests unexecuted.");
2062             uninstallTestApk();
2063         }
2064
2065         listener.testRunEnded(System.currentTimeMillis() - startTime, emptyMap);
2066     }
2067
2068    /**
2069      * {@inheritDoc}
2070      */
2071     @Override
2072     public void addIncludeFilter(String filter) {
2073         mIncludeFilters.add(filter);
2074     }
2075
2076     /**
2077      * {@inheritDoc}
2078      */
2079     @Override
2080     public void addAllIncludeFilters(Set<String> filters) {
2081         mIncludeFilters.addAll(filters);
2082     }
2083
2084     /**
2085      * {@inheritDoc}
2086      */
2087     @Override
2088     public void addExcludeFilter(String filter) {
2089         mExcludeFilters.add(filter);
2090     }
2091
2092     /**
2093      * {@inheritDoc}
2094      */
2095     @Override
2096     public void addAllExcludeFilters(Set<String> filters) {
2097         mExcludeFilters.addAll(filters);
2098     }
2099
2100     /**
2101      * {@inheritDoc}
2102      */
2103     @Override
2104     public void setCollectTestsOnly(boolean collectTests) {
2105         mCollectTestsOnly = collectTests;
2106     }
2107
2108     private static void copyOptions(DeqpTestRunner destination, DeqpTestRunner source) {
2109         destination.mDeqpPackage = source.mDeqpPackage;
2110         destination.mConfigName = source.mConfigName;
2111         destination.mCaselistFile = source.mCaselistFile;
2112         destination.mScreenRotation = source.mScreenRotation;
2113         destination.mSurfaceType = source.mSurfaceType;
2114         destination.mConfigRequired = source.mConfigRequired;
2115         destination.mIncludeFilters = new ArrayList<>(source.mIncludeFilters);
2116         destination.mExcludeFilters = new ArrayList<>(source.mExcludeFilters);
2117         destination.mAbi = source.mAbi;
2118         destination.mLogData = source.mLogData;
2119         destination.mCollectTestsOnly = source.mCollectTestsOnly;
2120     }
2121
2122     /**
2123      * {@inheritDoc}
2124      */
2125     @Override
2126     public Collection<IRemoteTest> split() {
2127         if (mTestInstances != null) {
2128             throw new AssertionError("Re-splitting or splitting running instance?");
2129         }
2130         // \todo [2015-11-23 kalle] If we split to batches at shard level, we could
2131         // basically get rid of batching. Except that sharding is optional?
2132
2133         // Assume that tests have not been yet loaded.
2134         loadTests();
2135
2136         Collection<IRemoteTest> runners = new ArrayList<>();
2137         // NOTE: Use linked hash map to keep the insertion order in iteration
2138         Map<TestIdentifier, Set<BatchRunConfiguration>> currentSet = new LinkedHashMap<>();
2139         Map<TestIdentifier, Set<BatchRunConfiguration>> iterationSet = this.mTestInstances;
2140
2141         // Go through tests, split
2142         for (TestIdentifier test: iterationSet.keySet()) {
2143             currentSet.put(test, iterationSet.get(test));
2144             if (currentSet.size() >= TESTCASE_BATCH_LIMIT) {
2145                 runners.add(new DeqpTestRunner(this, currentSet));
2146                 // NOTE: Use linked hash map to keep the insertion order in iteration
2147                 currentSet = new LinkedHashMap<>();
2148              }
2149         }
2150         runners.add(new DeqpTestRunner(this, currentSet));
2151
2152         // Compute new runtime hints
2153         long originalSize = iterationSet.size();
2154         if (originalSize > 0) {
2155             long fullRuntimeMs = getRuntimeHint();
2156             for (IRemoteTest remote: runners) {
2157                 DeqpTestRunner runner = (DeqpTestRunner)remote;
2158                 long shardRuntime = (fullRuntimeMs * runner.mTestInstances.size()) / originalSize;
2159                 runner.mRuntimeHint = shardRuntime;
2160             }
2161         }
2162
2163         CLog.i("Split deqp tests into %d shards", runners.size());
2164         return runners;
2165     }
2166
2167     /**
2168      * {@inheritDoc}
2169      */
2170     @Override
2171     public long getRuntimeHint() {
2172         if (mRuntimeHint != -1) {
2173             return mRuntimeHint;
2174         }
2175         if (mTestInstances == null) {
2176             loadTests();
2177         }
2178         // Tests normally take something like ~100ms. Some take a
2179         // second. Let's guess 200ms per test.
2180         return 200 * mTestInstances.size();
2181     }
2182 }