android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java

   1 /*
   2  * Copyright (C) 2014 The Android Open Source Project
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package com.drawelements.deqp.runner;
  17
  18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
  19 import com.android.ddmlib.AdbCommandRejectedException;
  20 import com.android.ddmlib.IShellOutputReceiver;
  21 import com.android.ddmlib.MultiLineReceiver;
  22 import com.android.ddmlib.ShellCommandUnresponsiveException;
  23 import com.android.ddmlib.TimeoutException;
  24 import com.android.ddmlib.testrunner.TestIdentifier;
  25 import com.android.tradefed.build.IBuildInfo;
  26 import com.android.tradefed.config.Option;
  27 import com.android.tradefed.config.OptionClass;
  28 import com.android.tradefed.device.DeviceNotAvailableException;
  29 import com.android.tradefed.device.ITestDevice;
  30 import com.android.tradefed.log.LogUtil.CLog;
  31 import com.android.tradefed.result.ByteArrayInputStreamSource;
  32 import com.android.tradefed.result.ITestInvocationListener;
  33 import com.android.tradefed.result.LogDataType;
  34 import com.android.tradefed.testtype.IAbi;
  35 import com.android.tradefed.testtype.IAbiReceiver;
  36 import com.android.tradefed.testtype.IBuildReceiver;
  37 import com.android.tradefed.testtype.IDeviceTest;
  38 import com.android.tradefed.testtype.IRemoteTest;
  39 import com.android.tradefed.testtype.IRuntimeHintProvider;
  40 import com.android.tradefed.testtype.IShardableTest;
  41 import com.android.tradefed.testtype.IStrictShardableTest;
  42 import com.android.tradefed.testtype.ITestCollector;
  43 import com.android.tradefed.testtype.ITestFilterReceiver;
  44 import com.android.tradefed.util.AbiUtils;
  45 import com.android.tradefed.util.IRunUtil;
  46 import com.android.tradefed.util.RunInterruptedException;
  47 import com.android.tradefed.util.RunUtil;
  48
  49 import java.io.BufferedReader;
  50 import java.io.File;
  51 import java.io.FileNotFoundException;
  52 import java.io.FileReader;
  53 import java.io.IOException;
  54 import java.io.Reader;
  55 import java.lang.reflect.InvocationTargetException;
  56 import java.lang.reflect.Method;
  57 import java.util.ArrayList;
  58 import java.util.Collection;
  59 import java.util.Collections;
  60 import java.util.HashMap;
  61 import java.util.HashSet;
  62 import java.util.Iterator;
  63 import java.util.LinkedHashMap;
  64 import java.util.LinkedHashSet;
  65 import java.util.LinkedList;
  66 import java.util.List;
  67 import java.util.Map;
  68 import java.util.Set;
  69 import java.util.concurrent.TimeUnit;
  70 import java.util.regex.Pattern;
  71
  72 /**
  73  * Test runner for dEQP tests
  74  *
  75  * Supports running drawElements Quality Program tests found under external/deqp.
  76  */
  77 @OptionClass(alias="deqp-test-runner")
  78 public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
  79         ITestFilterReceiver, IAbiReceiver, IShardableTest, ITestCollector,
  80         IRuntimeHintProvider, IStrictShardableTest {
  81     private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk";
  82     private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp";
  83     private static final String INCOMPLETE_LOG_MESSAGE = "Crash: Incomplete test log";
  84     private static final String SKIPPED_INSTANCE_LOG_MESSAGE = "Configuration skipped";
  85     private static final String NOT_EXECUTABLE_LOG_MESSAGE = "Abort: Test cannot be executed";
  86     private static final String CASE_LIST_FILE_NAME = "/sdcard/dEQP-TestCaseList.txt";
  87     private static final String LOG_FILE_NAME = "/sdcard/TestLog.qpa";
  88     public static final String FEATURE_LANDSCAPE = "android.hardware.screen.landscape";
  89     public static final String FEATURE_PORTRAIT = "android.hardware.screen.portrait";
  90     public static final String FEATURE_VULKAN_LEVEL = "android.hardware.vulkan.level";
  91
  92     private static final int TESTCASE_BATCH_LIMIT = 1000;
  93     private static final int UNRESPONSIVE_CMD_TIMEOUT_MS = 10 * 60 * 1000; // 10min
  94
  95     // !NOTE: There's a static method copyOptions() for copying options during split.
  96     // If you add state update copyOptions() as appropriate!
  97
  98     @Option(name="deqp-package",
  99             description="Name of the deqp module used. Determines GLES version.",
 100             importance=Option.Importance.ALWAYS)
 101     private String mDeqpPackage;
 102     @Option(name="deqp-gl-config-name",
 103             description="GL render target config. See deqp documentation for syntax. ",
 104             importance=Option.Importance.NEVER)
 105     private String mConfigName = "";
 106     @Option(name="deqp-caselist-file",
 107             description="File listing the names of the cases to be run.",
 108             importance=Option.Importance.ALWAYS)
 109     private String mCaselistFile;
 110     @Option(name="deqp-screen-rotation",
 111             description="Screen orientation. Defaults to 'unspecified'",
 112             importance=Option.Importance.NEVER)
 113     private String mScreenRotation = "unspecified";
 114     @Option(name="deqp-surface-type",
 115             description="Surface type ('window', 'pbuffer', 'fbo'). Defaults to 'window'",
 116             importance=Option.Importance.NEVER)
 117     private String mSurfaceType = "window";
 118     @Option(name="deqp-config-required",
 119             description="Is current config required if API is supported? Defaults to false.",
 120             importance=Option.Importance.NEVER)
 121     private boolean mConfigRequired = false;
 122     @Option(name = "include-filter",
 123             description="Test include filter. '*' is zero or more letters. '.' has no special meaning.")
 124     private List<String> mIncludeFilters = new ArrayList<>();
 125     @Option(name = "include-filter-file",
 126             description="Load list of includes from the files given.")
 127     private List<String> mIncludeFilterFiles = new ArrayList<>();
 128     @Option(name = "exclude-filter",
 129             description="Test exclude filter. '*' is zero or more letters. '.' has no special meaning.")
 130     private List<String> mExcludeFilters = new ArrayList<>();
 131     @Option(name = "exclude-filter-file",
 132             description="Load list of excludes from the files given.")
 133     private List<String> mExcludeFilterFiles = new ArrayList<>();
 134     @Option(name = "collect-tests-only",
 135             description = "Only invoke the instrumentation to collect list of applicable test "
 136                     + "cases. All test run callbacks will be triggered, but test execution will "
 137                     + "not be actually carried out.")
 138     private boolean mCollectTestsOnly = false;
 139     @Option(name = "runtime-hint",
 140             isTimeVal = true,
 141             description="The estimated config runtime. Defaults to 200ms x num tests.")
 142     private long mRuntimeHint = -1;
 143
 144     private Collection<TestIdentifier> mRemainingTests = null;
 145     private Map<TestIdentifier, Set<BatchRunConfiguration>> mTestInstances = null;
 146     private final TestInstanceResultListener mInstanceListerner = new TestInstanceResultListener();
 147     private final Map<TestIdentifier, Integer> mTestInstabilityRatings = new HashMap<>();
 148     private IAbi mAbi;
 149     private CompatibilityBuildHelper mBuildHelper;
 150     private boolean mLogData = false;
 151     private ITestDevice mDevice;
 152     private Set<String> mDeviceFeatures;
 153     private Map<String, Boolean> mConfigQuerySupportCache = new HashMap<>();
 154     private IRunUtil mRunUtil = RunUtil.getDefault();
 155     // When set will override the mCaselistFile for testing purposes.
 156     private Reader mCaselistReader = null;
 157
 158     private IRecovery mDeviceRecovery = new Recovery(); {
 159         mDeviceRecovery.setSleepProvider(new SleepProvider());
 160     }
 161
 162     public DeqpTestRunner() {
 163     }
 164
 165     private DeqpTestRunner(DeqpTestRunner optionTemplate,
 166                 Map<TestIdentifier, Set<BatchRunConfiguration>> tests) {
 167         copyOptions(this, optionTemplate);
 168         mTestInstances = tests;
 169     }
 170
 171     /**
 172      * @param abi the ABI to run the test on
 173      */
 174     @Override
 175     public void setAbi(IAbi abi) {
 176         mAbi = abi;
 177     }
 178
 179     /**
 180      * {@inheritDoc}
 181      */
 182     @Override
 183     public void setBuild(IBuildInfo buildInfo) {
 184         setBuildHelper(new CompatibilityBuildHelper(buildInfo));
 185     }
 186
 187     /**
 188      * Exposed for better mockability during testing. In real use, always flows from
 189      * setBuild() called by the framework
 190      */
 191     public void setBuildHelper(CompatibilityBuildHelper helper) {
 192         mBuildHelper = helper;
 193     }
 194
 195     /**
 196      * Enable or disable raw dEQP test log collection.
 197      */
 198     public void setCollectLogs(boolean logData) {
 199         mLogData = logData;
 200     }
 201
 202     /**
 203      * Get the deqp-package option contents.
 204      */
 205     public String getPackageName() {
 206         return mDeqpPackage;
 207     }
 208
 209     /**
 210      * {@inheritDoc}
 211      */
 212     @Override
 213     public void setDevice(ITestDevice device) {
 214         mDevice = device;
 215     }
 216
 217     /**
 218      * {@inheritDoc}
 219      */
 220     @Override
 221     public ITestDevice getDevice() {
 222         return mDevice;
 223     }
 224
 225     /**
 226      * Set recovery handler.
 227      *
 228      * Exposed for unit testing.
 229      */
 230     public void setRecovery(IRecovery deviceRecovery) {
 231         mDeviceRecovery = deviceRecovery;
 232     }
 233
 234     /**
 235      * Set IRunUtil.
 236      *
 237      * Exposed for unit testing.
 238      */
 239     public void setRunUtil(IRunUtil runUtil) {
 240         mRunUtil = runUtil;
 241     }
 242
 243     /**
 244      * Exposed for unit testing
 245      */
 246     public void setCaselistReader(Reader caselistReader) {
 247         mCaselistReader = caselistReader;
 248     }
 249
 250     private static final class CapabilityQueryFailureException extends Exception {
 251     }
 252
 253     /**
 254      * dEQP test instance listerer and invocation result forwarded
 255      */
 256     private class TestInstanceResultListener {
 257         private ITestInvocationListener mSink;
 258         private BatchRunConfiguration mRunConfig;
 259
 260         private TestIdentifier mCurrentTestId;
 261         private boolean mGotTestResult;
 262         private String mCurrentTestLog;
 263
 264         private class PendingResult {
 265             boolean allInstancesPassed;
 266             Map<BatchRunConfiguration, String> testLogs;
 267             Map<BatchRunConfiguration, String> errorMessages;
 268             Set<BatchRunConfiguration> remainingConfigs;
 269         }
 270
 271         private final Map<TestIdentifier, PendingResult> mPendingResults = new HashMap<>();
 272
 273         public void setSink(ITestInvocationListener sink) {
 274             mSink = sink;
 275         }
 276
 277         public void setCurrentConfig(BatchRunConfiguration runConfig) {
 278             mRunConfig = runConfig;
 279         }
 280
 281         /**
 282          * Get currently processed test id, or null if not currently processing a test case
 283          */
 284         public TestIdentifier getCurrentTestId() {
 285             return mCurrentTestId;
 286         }
 287
 288         /**
 289          * Forward result to sink
 290          */
 291         private void forwardFinalizedPendingResult(TestIdentifier testId) {
 292             if (mRemainingTests.contains(testId)) {
 293                 final PendingResult result = mPendingResults.get(testId);
 294
 295                 mPendingResults.remove(testId);
 296                 mRemainingTests.remove(testId);
 297
 298                 // Forward results to the sink
 299                 mSink.testStarted(testId);
 300
 301                 // Test Log
 302                 if (mLogData) {
 303                     for (Map.Entry<BatchRunConfiguration, String> entry :
 304                             result.testLogs.entrySet()) {
 305                         final ByteArrayInputStreamSource source
 306                                 = new ByteArrayInputStreamSource(entry.getValue().getBytes());
 307
 308                         mSink.testLog(testId.getClassName() + "." + testId.getTestName() + "@"
 309                                 + entry.getKey().getId(), LogDataType.XML, source);
 310
 311                         source.cancel();
 312                     }
 313                 }
 314
 315                 // Error message
 316                 if (!result.allInstancesPassed) {
 317                     final StringBuilder errorLog = new StringBuilder();
 318
 319                     for (Map.Entry<BatchRunConfiguration, String> entry :
 320                             result.errorMessages.entrySet()) {
 321                         if (errorLog.length() > 0) {
 322                             errorLog.append('\n');
 323                         }
 324                         errorLog.append(String.format("=== with config %s ===\n",
 325                                 entry.getKey().getId()));
 326                         errorLog.append(entry.getValue());
 327                     }
 328
 329                     mSink.testFailed(testId, errorLog.toString());
 330                 }
 331
 332                 final Map<String, String> emptyMap = Collections.emptyMap();
 333                 mSink.testEnded(testId, emptyMap);
 334             }
 335         }
 336
 337         /**
 338          * Declare existence of a test and instances
 339          */
 340         public void setTestInstances(TestIdentifier testId, Set<BatchRunConfiguration> configs) {
 341             // Test instances cannot change at runtime, ignore if we have already set this
 342             if (!mPendingResults.containsKey(testId)) {
 343                 final PendingResult pendingResult = new PendingResult();
 344                 pendingResult.allInstancesPassed = true;
 345                 pendingResult.testLogs = new LinkedHashMap<>();
 346                 pendingResult.errorMessages = new LinkedHashMap<>();
 347                 pendingResult.remainingConfigs = new HashSet<>(configs); // avoid mutating argument
 348                 mPendingResults.put(testId, pendingResult);
 349             }
 350         }
 351
 352         /**
 353          * Query if test instance has not yet been executed
 354          */
 355         public boolean isPendingTestInstance(TestIdentifier testId,
 356                 BatchRunConfiguration config) {
 357             final PendingResult result = mPendingResults.get(testId);
 358             if (result == null) {
 359                 // test is not in the current working batch of the runner, i.e. it cannot be
 360                 // "partially" completed.
 361                 if (!mRemainingTests.contains(testId)) {
 362                     // The test has been fully executed. Not pending.
 363                     return false;
 364                 } else {
 365                     // Test has not yet been executed. Check if such instance exists
 366                     return mTestInstances.get(testId).contains(config);
 367                 }
 368             } else {
 369                 // could be partially completed, check this particular config
 370                 return result.remainingConfigs.contains(config);
 371             }
 372         }
 373
 374         /**
 375          * Fake execution of an instance with current config
 376          */
 377         public void skipTest(TestIdentifier testId) {
 378             final PendingResult result = mPendingResults.get(testId);
 379
 380             result.errorMessages.put(mRunConfig, SKIPPED_INSTANCE_LOG_MESSAGE);
 381             result.remainingConfigs.remove(mRunConfig);
 382
 383             // Pending result finished, report result
 384             if (result.remainingConfigs.isEmpty()) {
 385                 forwardFinalizedPendingResult(testId);
 386             }
 387         }
 388
 389         /**
 390          * Fake failure of an instance with current config
 391          */
 392         public void abortTest(TestIdentifier testId, String errorMessage) {
 393             final PendingResult result = mPendingResults.get(testId);
 394
 395             // Mark as executed
 396             result.allInstancesPassed = false;
 397             result.errorMessages.put(mRunConfig, errorMessage);
 398             result.remainingConfigs.remove(mRunConfig);
 399
 400             // Pending result finished, report result
 401             if (result.remainingConfigs.isEmpty()) {
 402                 forwardFinalizedPendingResult(testId);
 403             }
 404
 405             if (testId.equals(mCurrentTestId)) {
 406                 mCurrentTestId = null;
 407             }
 408         }
 409
 410         /**
 411          * Handles beginning of dEQP session.
 412          */
 413         private void handleBeginSession(Map<String, String> values) {
 414             // ignore
 415         }
 416
 417         /**
 418          * Handles end of dEQP session.
 419          */
 420         private void handleEndSession(Map<String, String> values) {
 421             // ignore
 422         }
 423
 424         /**
 425          * Handles beginning of dEQP testcase.
 426          */
 427         private void handleBeginTestCase(Map<String, String> values) {
 428             mCurrentTestId = pathToIdentifier(values.get("dEQP-BeginTestCase-TestCasePath"));
 429             mCurrentTestLog = "";
 430             mGotTestResult = false;
 431
 432             // mark instance as started
 433             if (mPendingResults.get(mCurrentTestId) != null) {
 434                 mPendingResults.get(mCurrentTestId).remainingConfigs.remove(mRunConfig);
 435             } else {
 436                 CLog.w("Got unexpected start of %s", mCurrentTestId);
 437             }
 438         }
 439
 440         /**
 441          * Handles end of dEQP testcase.
 442          */
 443         private void handleEndTestCase(Map<String, String> values) {
 444             final PendingResult result = mPendingResults.get(mCurrentTestId);
 445
 446             if (result != null) {
 447                 if (!mGotTestResult) {
 448                     result.allInstancesPassed = false;
 449                     result.errorMessages.put(mRunConfig, INCOMPLETE_LOG_MESSAGE);
 450                 }
 451
 452                 if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
 453                     result.testLogs.put(mRunConfig, mCurrentTestLog);
 454                 }
 455
 456                 // Pending result finished, report result
 457                 if (result.remainingConfigs.isEmpty()) {
 458                     forwardFinalizedPendingResult(mCurrentTestId);
 459                 }
 460             } else {
 461                 CLog.w("Got unexpected end of %s", mCurrentTestId);
 462             }
 463             mCurrentTestId = null;
 464         }
 465
 466         /**
 467          * Handles dEQP testcase result.
 468          */
 469         private void handleTestCaseResult(Map<String, String> values) {
 470             String code = values.get("dEQP-TestCaseResult-Code");
 471             String details = values.get("dEQP-TestCaseResult-Details");
 472
 473             if (mPendingResults.get(mCurrentTestId) == null) {
 474                 CLog.w("Got unexpected result for %s", mCurrentTestId);
 475                 mGotTestResult = true;
 476                 return;
 477             }
 478
 479             if (code.compareTo("Pass") == 0) {
 480                 mGotTestResult = true;
 481             } else if (code.compareTo("NotSupported") == 0) {
 482                 mGotTestResult = true;
 483             } else if (code.compareTo("QualityWarning") == 0) {
 484                 mGotTestResult = true;
 485             } else if (code.compareTo("CompatibilityWarning") == 0) {
 486                 mGotTestResult = true;
 487             } else if (code.compareTo("Fail") == 0 || code.compareTo("ResourceError") == 0
 488                     || code.compareTo("InternalError") == 0 || code.compareTo("Crash") == 0
 489                     || code.compareTo("Timeout") == 0) {
 490                 mPendingResults.get(mCurrentTestId).allInstancesPassed = false;
 491                 mPendingResults.get(mCurrentTestId)
 492                         .errorMessages.put(mRunConfig, code + ": " + details);
 493                 mGotTestResult = true;
 494             } else {
 495                 String codeError = "Unknown result code: " + code;
 496                 mPendingResults.get(mCurrentTestId).allInstancesPassed = false;
 497                 mPendingResults.get(mCurrentTestId)
 498                         .errorMessages.put(mRunConfig, codeError + ": " + details);
 499                 mGotTestResult = true;
 500             }
 501         }
 502
 503         /**
 504          * Handles terminated dEQP testcase.
 505          */
 506         private void handleTestCaseTerminate(Map<String, String> values) {
 507             final PendingResult result = mPendingResults.get(mCurrentTestId);
 508
 509             if (result != null) {
 510                 String reason = values.get("dEQP-TerminateTestCase-Reason");
 511                 mPendingResults.get(mCurrentTestId).allInstancesPassed = false;
 512                 mPendingResults.get(mCurrentTestId)
 513                         .errorMessages.put(mRunConfig, "Terminated: " + reason);
 514
 515                 // Pending result finished, report result
 516                 if (result.remainingConfigs.isEmpty()) {
 517                     forwardFinalizedPendingResult(mCurrentTestId);
 518                 }
 519             } else {
 520                 CLog.w("Got unexpected termination of %s", mCurrentTestId);
 521             }
 522
 523             mCurrentTestId = null;
 524             mGotTestResult = true;
 525         }
 526
 527         /**
 528          * Handles dEQP testlog data.
 529          */
 530         private void handleTestLogData(Map<String, String> values) {
 531             mCurrentTestLog = mCurrentTestLog + values.get("dEQP-TestLogData-Log");
 532         }
 533
 534         /**
 535          * Handles new instrumentation status message.
 536          */
 537         public void handleStatus(Map<String, String> values) {
 538             String eventType = values.get("dEQP-EventType");
 539
 540             if (eventType == null) {
 541                 return;
 542             }
 543
 544             if (eventType.compareTo("BeginSession") == 0) {
 545                 handleBeginSession(values);
 546             } else if (eventType.compareTo("EndSession") == 0) {
 547                 handleEndSession(values);
 548             } else if (eventType.compareTo("BeginTestCase") == 0) {
 549                 handleBeginTestCase(values);
 550             } else if (eventType.compareTo("EndTestCase") == 0) {
 551                 handleEndTestCase(values);
 552             } else if (eventType.compareTo("TestCaseResult") == 0) {
 553                 handleTestCaseResult(values);
 554             } else if (eventType.compareTo("TerminateTestCase") == 0) {
 555                 handleTestCaseTerminate(values);
 556             } else if (eventType.compareTo("TestLogData") == 0) {
 557                 handleTestLogData(values);
 558             }
 559         }
 560
 561         /**
 562          * Signal listener that batch ended and forget incomplete results.
 563          */
 564         public void endBatch() {
 565             // end open test if when stream ends
 566             if (mCurrentTestId != null) {
 567                 // Current instance was removed from remainingConfigs when case
 568                 // started. Mark current instance as pending.
 569                 if (mPendingResults.get(mCurrentTestId) != null) {
 570                     mPendingResults.get(mCurrentTestId).remainingConfigs.add(mRunConfig);
 571                 } else {
 572                     CLog.w("Got unexpected internal state of %s", mCurrentTestId);
 573                 }
 574             }
 575             mCurrentTestId = null;
 576         }
 577     }
 578
 579     /**
 580      * dEQP instrumentation parser
 581      */
 582     private static class InstrumentationParser extends MultiLineReceiver {
 583         private TestInstanceResultListener mListener;
 584
 585         private Map<String, String> mValues;
 586         private String mCurrentName;
 587         private String mCurrentValue;
 588         private int mResultCode;
 589         private boolean mGotExitValue = false;
 590
 591
 592         public InstrumentationParser(TestInstanceResultListener listener) {
 593             mListener = listener;
 594         }
 595
 596         /**
 597          * {@inheritDoc}
 598          */
 599         @Override
 600         public void processNewLines(String[] lines) {
 601             for (String line : lines) {
 602                 if (mValues == null) mValues = new HashMap<String, String>();
 603
 604                 if (line.startsWith("INSTRUMENTATION_STATUS_CODE: ")) {
 605                     if (mCurrentName != null) {
 606                         mValues.put(mCurrentName, mCurrentValue);
 607
 608                         mCurrentName = null;
 609                         mCurrentValue = null;
 610                     }
 611
 612                     mListener.handleStatus(mValues);
 613                     mValues = null;
 614                 } else if (line.startsWith("INSTRUMENTATION_STATUS: dEQP-")) {
 615                     if (mCurrentName != null) {
 616                         mValues.put(mCurrentName, mCurrentValue);
 617
 618                         mCurrentValue = null;
 619                         mCurrentName = null;
 620                     }
 621
 622                     String prefix = "INSTRUMENTATION_STATUS: ";
 623                     int nameBegin = prefix.length();
 624                     int nameEnd = line.indexOf('=');
 625                     int valueBegin = nameEnd + 1;
 626
 627                     mCurrentName = line.substring(nameBegin, nameEnd);
 628                     mCurrentValue = line.substring(valueBegin);
 629                 } else if (line.startsWith("INSTRUMENTATION_CODE: ")) {
 630                     try {
 631                         mResultCode = Integer.parseInt(line.substring(22));
 632                         mGotExitValue = true;
 633                     } catch (NumberFormatException ex) {
 634                         CLog.w("Instrumentation code format unexpected");
 635                     }
 636                 } else if (mCurrentValue != null) {
 637                     mCurrentValue = mCurrentValue + line;
 638                 }
 639             }
 640         }
 641
 642         /**
 643          * {@inheritDoc}
 644          */
 645         @Override
 646         public void done() {
 647             if (mCurrentName != null) {
 648                 mValues.put(mCurrentName, mCurrentValue);
 649
 650                 mCurrentName = null;
 651                 mCurrentValue = null;
 652             }
 653
 654             if (mValues != null) {
 655                 mListener.handleStatus(mValues);
 656                 mValues = null;
 657             }
 658         }
 659
 660         /**
 661          * {@inheritDoc}
 662          */
 663         @Override
 664         public boolean isCancelled() {
 665             return false;
 666         }
 667
 668         /**
 669          * Returns whether target instrumentation exited normally.
 670          */
 671         public boolean wasSuccessful() {
 672             return mGotExitValue;
 673         }
 674
 675         /**
 676          * Returns Instrumentation return code
 677          */
 678         public int getResultCode() {
 679             return mResultCode;
 680         }
 681     }
 682
 683     /**
 684      * dEQP platfom query instrumentation parser
 685      */
 686     private static class PlatformQueryInstrumentationParser extends MultiLineReceiver {
 687         private Map<String,String> mResultMap = new LinkedHashMap<>();
 688         private int mResultCode;
 689         private boolean mGotExitValue = false;
 690
 691         /**
 692          * {@inheritDoc}
 693          */
 694         @Override
 695         public void processNewLines(String[] lines) {
 696             for (String line : lines) {
 697                 if (line.startsWith("INSTRUMENTATION_RESULT: ")) {
 698                     final String parts[] = line.substring(24).split("=",2);
 699                     if (parts.length == 2) {
 700                         mResultMap.put(parts[0], parts[1]);
 701                     } else {
 702                         CLog.w("Instrumentation status format unexpected");
 703                     }
 704                 } else if (line.startsWith("INSTRUMENTATION_CODE: ")) {
 705                     try {
 706                         mResultCode = Integer.parseInt(line.substring(22));
 707                         mGotExitValue = true;
 708                     } catch (NumberFormatException ex) {
 709                         CLog.w("Instrumentation code format unexpected");
 710                     }
 711                 }
 712             }
 713         }
 714
 715         /**
 716          * {@inheritDoc}
 717          */
 718         @Override
 719         public boolean isCancelled() {
 720             return false;
 721         }
 722
 723         /**
 724          * Returns whether target instrumentation exited normally.
 725          */
 726         public boolean wasSuccessful() {
 727             return mGotExitValue;
 728         }
 729
 730         /**
 731          * Returns Instrumentation return code
 732          */
 733         public int getResultCode() {
 734             return mResultCode;
 735         }
 736
 737         public Map<String,String> getResultMap() {
 738             return mResultMap;
 739         }
 740     }
 741
 742     /**
 743      * Interface for sleeping.
 744      *
 745      * Exposed for unit testing
 746      */
 747     public static interface ISleepProvider {
 748         public void sleep(int milliseconds);
 749     }
 750
 751     private static class SleepProvider implements ISleepProvider {
 752         @Override
 753         public void sleep(int milliseconds) {
 754             RunUtil.getDefault().sleep(milliseconds);
 755         }
 756     }
 757
 758     /**
 759      * Interface for failure recovery.
 760      *
 761      * Exposed for unit testing
 762      */
 763     public static interface IRecovery {
 764         /**
 765          * Sets the sleep provider IRecovery works on
 766          */
 767         public void setSleepProvider(ISleepProvider sleepProvider);
 768
 769         /**
 770          * Sets the device IRecovery works on
 771          */
 772         public void setDevice(ITestDevice device);
 773
 774         /**
 775          * Informs Recovery that test execution has progressed since the last recovery
 776          */
 777         public void onExecutionProgressed();
 778
 779         /**
 780          * Tries to recover device after failed refused connection.
 781          *
 782          * @throws DeviceNotAvailableException if recovery did not succeed
 783          */
 784         public void recoverConnectionRefused() throws DeviceNotAvailableException;
 785
 786         /**
 787          * Tries to recover device after abnormal execution termination or link failure.
 788          *
 789          * @throws DeviceNotAvailableException if recovery did not succeed
 790          */
 791         public void recoverComLinkKilled() throws DeviceNotAvailableException;
 792     }
 793
 794     /**
 795      * State machine for execution failure recovery.
 796      *
 797      * Exposed for unit testing
 798      */
 799     public static class Recovery implements IRecovery {
 800         private int RETRY_COOLDOWN_MS = 6000; // 6 seconds
 801         private int PROCESS_KILL_WAIT_MS = 1000; // 1 second
 802
 803         private static enum MachineState {
 804             WAIT, // recover by waiting
 805             RECOVER, // recover by calling recover()
 806             REBOOT, // recover by rebooting
 807             FAIL, // cannot recover
 808         }
 809
 810         private MachineState mState = MachineState.WAIT;
 811         private ITestDevice mDevice;
 812         private ISleepProvider mSleepProvider;
 813
 814         private static class ProcessKillFailureException extends Exception {
 815         }
 816
 817         /**
 818          * {@inheritDoc}
 819          */
 820         @Override
 821         public void setSleepProvider(ISleepProvider sleepProvider) {
 822             mSleepProvider = sleepProvider;
 823         }
 824
 825         /**
 826          * {@inheritDoc}
 827          */
 828         @Override
 829         public void setDevice(ITestDevice device) {
 830             mDevice = device;
 831         }
 832
 833         /**
 834          * {@inheritDoc}
 835          */
 836         @Override
 837         public void onExecutionProgressed() {
 838             mState = MachineState.WAIT;
 839         }
 840
 841         /**
 842          * {@inheritDoc}
 843          */
 844         @Override
 845         public void recoverConnectionRefused() throws DeviceNotAvailableException {
 846             switch (mState) {
 847                 case WAIT: // not a valid stratedy for connection refusal, fallthrough
 848                 case RECOVER:
 849                     // First failure, just try to recover
 850                     CLog.w("ADB connection failed, trying to recover");
 851                     mState = MachineState.REBOOT; // the next step is to reboot
 852
 853                     try {
 854                         recoverDevice();
 855                     } catch (DeviceNotAvailableException ex) {
 856                         // chain forward
 857                         recoverConnectionRefused();
 858                     }
 859                     break;
 860
 861                 case REBOOT:
 862                     // Second failure in a row, try to reboot
 863                     CLog.w("ADB connection failed after recovery, rebooting device");
 864                     mState = MachineState.FAIL; // the next step is to fail
 865
 866                     try {
 867                         rebootDevice();
 868                     } catch (DeviceNotAvailableException ex) {
 869                         // chain forward
 870                         recoverConnectionRefused();
 871                     }
 872                     break;
 873
 874                 case FAIL:
 875                     // Third failure in a row, just fail
 876                     CLog.w("Cannot recover ADB connection");
 877                     throw new DeviceNotAvailableException("failed to connect after reboot",
 878                             mDevice.getSerialNumber());
 879             }
 880         }
 881
 882         /**
 883          * {@inheritDoc}
 884          */
 885         @Override
 886         public void recoverComLinkKilled() throws DeviceNotAvailableException {
 887             switch (mState) {
 888                 case WAIT:
 889                     // First failure, just try to wait and try again
 890                     CLog.w("ADB link failed, retrying after a cooldown period");
 891                     mState = MachineState.RECOVER; // the next step is to recover the device
 892
 893                     waitCooldown();
 894
 895                     // even if the link to deqp on-device process was killed, the process might
 896                     // still be alive. Locate and terminate such unwanted processes.
 897                     try {
 898                         killDeqpProcess();
 899                     } catch (DeviceNotAvailableException ex) {
 900                         // chain forward
 901                         recoverComLinkKilled();
 902                     } catch (ProcessKillFailureException ex) {
 903                         // chain forward
 904                         recoverComLinkKilled();
 905                     }
 906                     break;
 907
 908                 case RECOVER:
 909                     // Second failure, just try to recover
 910                     CLog.w("ADB link failed, trying to recover");
 911                     mState = MachineState.REBOOT; // the next step is to reboot
 912
 913                     try {
 914                         recoverDevice();
 915                         killDeqpProcess();
 916                     } catch (DeviceNotAvailableException ex) {
 917                         // chain forward
 918                         recoverComLinkKilled();
 919                     } catch (ProcessKillFailureException ex) {
 920                         // chain forward
 921                         recoverComLinkKilled();
 922                     }
 923                     break;
 924
 925                 case REBOOT:
 926                     // Third failure in a row, try to reboot
 927                     CLog.w("ADB link failed after recovery, rebooting device");
 928                     mState = MachineState.FAIL; // the next step is to fail
 929
 930                     try {
 931                         rebootDevice();
 932                     } catch (DeviceNotAvailableException ex) {
 933                         // chain forward
 934                         recoverComLinkKilled();
 935                     }
 936                     break;
 937
 938                 case FAIL:
 939                     // Fourth failure in a row, just fail
 940                     CLog.w("Cannot recover ADB connection");
 941                     throw new DeviceNotAvailableException("link killed after reboot",
 942                             mDevice.getSerialNumber());
 943             }
 944         }
 945
 946         private void waitCooldown() {
 947             mSleepProvider.sleep(RETRY_COOLDOWN_MS);
 948         }
 949
 950         private Iterable<Integer> getDeqpProcessPids() throws DeviceNotAvailableException {
 951             final List<Integer> pids = new ArrayList<Integer>(2);
 952             final String processes = mDevice.executeShellCommand("ps | grep com.drawelements");
 953             final String[] lines = processes.split("(\\r|\\n)+");
 954             for (String line : lines) {
 955                 final String[] fields = line.split("\\s+");
 956                 if (fields.length < 2) {
 957                     continue;
 958                 }
 959
 960                 try {
 961                     final int processId = Integer.parseInt(fields[1], 10);
 962                     pids.add(processId);
 963                 } catch (NumberFormatException ex) {
 964                     continue;
 965                 }
 966             }
 967             return pids;
 968         }
 969
 970         private void killDeqpProcess() throws DeviceNotAvailableException,
 971                 ProcessKillFailureException {
 972             for (Integer processId : getDeqpProcessPids()) {
 973                 mDevice.executeShellCommand(String.format("kill -9 %d", processId));
 974             }
 975
 976             mSleepProvider.sleep(PROCESS_KILL_WAIT_MS);
 977
 978             // check that processes actually died
 979             if (getDeqpProcessPids().iterator().hasNext()) {
 980                 // a process is still alive, killing failed
 981                 throw new ProcessKillFailureException();
 982             }
 983         }
 984
 985         public void recoverDevice() throws DeviceNotAvailableException {
 986             // Work around the API. We need to call recoverDevice() on the test device and
 987             // we know that mDevice is a TestDevice. However even though the recoverDevice()
 988             // method is public suggesting it should be publicly accessible, the class itself
 989             // and its super-interface (IManagedTestDevice) are package-private.
 990             final Method recoverDeviceMethod;
 991             try {
 992                 recoverDeviceMethod = mDevice.getClass().getMethod("recoverDevice");
 993                 recoverDeviceMethod.setAccessible(true);
 994             } catch (NoSuchMethodException ex) {
 995                 throw new AssertionError("Test device must have recoverDevice()");
 996             }
 997
 998             try {
 999                 recoverDeviceMethod.invoke(mDevice);
1000             } catch (InvocationTargetException ex) {
1001                 if (ex.getCause() instanceof DeviceNotAvailableException) {
1002                     throw (DeviceNotAvailableException)ex.getCause();
1003                 } else if (ex.getCause() instanceof RuntimeException) {
1004                     throw (RuntimeException)ex.getCause();
1005                 } else {
1006                     throw new AssertionError("unexpected throw", ex);
1007                 }
1008             } catch (IllegalAccessException ex) {
1009                 throw new AssertionError("unexpected throw", ex);
1010             }
1011         }
1012
1013         private void rebootDevice() throws DeviceNotAvailableException {
1014             mDevice.reboot();
1015         }
1016     }
1017
1018     private static Map<TestIdentifier, Set<BatchRunConfiguration>> generateTestInstances(
1019             Reader testlist, String configName, String screenRotation, String surfaceType,
1020             boolean required) {
1021         // Note: This is specifically a LinkedHashMap to guarantee that tests are iterated
1022         // in the insertion order.
1023         final Map<TestIdentifier, Set<BatchRunConfiguration>> instances = new LinkedHashMap<>();
1024         try {
1025             BufferedReader testlistReader = new BufferedReader(testlist);
1026             String testName;
1027             while ((testName = testlistReader.readLine()) != null) {
1028                 // Test name -> testId -> only one config -> done.
1029                 final Set<BatchRunConfiguration> testInstanceSet = new LinkedHashSet<>();
1030                 BatchRunConfiguration config = new BatchRunConfiguration(configName, screenRotation, surfaceType, required);
1031                 testInstanceSet.add(config);
1032                 TestIdentifier test = pathToIdentifier(testName);
1033                 instances.put(test, testInstanceSet);
1034             }
1035             testlistReader.close();
1036         }
1037         catch (IOException e)
1038         {
1039             throw new RuntimeException("Failure while reading the test case list for deqp: " + e.getMessage());
1040         }
1041
1042         return instances;
1043     }
1044
1045     private Set<BatchRunConfiguration> getTestRunConfigs(TestIdentifier testId) {
1046         return mTestInstances.get(testId);
1047     }
1048
1049     /**
1050      * Get the test instance of the runner. Exposed for testing.
1051      */
1052     Map<TestIdentifier, Set<BatchRunConfiguration>> getTestInstance() {
1053         return mTestInstances;
1054     }
1055
1056     /**
1057      * Converts dEQP testcase path to TestIdentifier.
1058      */
1059     private static TestIdentifier pathToIdentifier(String testPath) {
1060         int indexOfLastDot = testPath.lastIndexOf('.');
1061         String className = testPath.substring(0, indexOfLastDot);
1062         String testName = testPath.substring(indexOfLastDot+1);
1063
1064         return new TestIdentifier(className, testName);
1065     }
1066
1067     // \todo [2015-10-16 kalle] How unique should this be?
1068     private String getId() {
1069         return AbiUtils.createId(mAbi.getName(), mDeqpPackage);
1070     }
1071
1072     /**
1073      * Generates tescase trie from dEQP testcase paths. Used to define which testcases to execute.
1074      */
1075     private static String generateTestCaseTrieFromPaths(Collection<String> tests) {
1076         String result = "{";
1077         boolean first = true;
1078
1079         // Add testcases to results
1080         for (Iterator<String> iter = tests.iterator(); iter.hasNext();) {
1081             String test = iter.next();
1082             String[] components = test.split("\\.");
1083
1084             if (components.length == 1) {
1085                 if (!first) {
1086                     result = result + ",";
1087                 }
1088                 first = false;
1089
1090                 result += components[0];
1091                 iter.remove();
1092             }
1093         }
1094
1095         if (!tests.isEmpty()) {
1096             HashMap<String, ArrayList<String> > testGroups = new HashMap<>();
1097
1098             // Collect all sub testgroups
1099             for (String test : tests) {
1100                 String[] components = test.split("\\.");
1101                 ArrayList<String> testGroup = testGroups.get(components[0]);
1102
1103                 if (testGroup == null) {
1104                     testGroup = new ArrayList<String>();
1105                     testGroups.put(components[0], testGroup);
1106                 }
1107
1108                 testGroup.add(test.substring(components[0].length()+1));
1109             }
1110
1111             for (String testGroup : testGroups.keySet()) {
1112                 if (!first) {
1113                     result = result + ",";
1114                 }
1115
1116                 first = false;
1117                 result = result + testGroup
1118                         + generateTestCaseTrieFromPaths(testGroups.get(testGroup));
1119             }
1120         }
1121
1122         return result + "}";
1123     }
1124
1125     /**
1126      * Generates testcase trie from TestIdentifiers.
1127      */
1128     private static String generateTestCaseTrie(Collection<TestIdentifier> tests) {
1129         ArrayList<String> testPaths = new ArrayList<String>();
1130
1131         for (TestIdentifier test : tests) {
1132             testPaths.add(test.getClassName() + "." + test.getTestName());
1133         }
1134
1135         return generateTestCaseTrieFromPaths(testPaths);
1136     }
1137
1138     private static class TestBatch {
1139         public BatchRunConfiguration config;
1140         public List<TestIdentifier> tests;
1141     }
1142
1143     /**
1144      * Creates a TestBatch from the given tests or null if not tests remaining.
1145      *
1146      *  @param pool List of tests to select from
1147      *  @param requiredConfig Select only instances with pending requiredConfig, or null to select
1148      *         any run configuration.
1149      */
1150     private TestBatch selectRunBatch(Collection<TestIdentifier> pool,
1151             BatchRunConfiguration requiredConfig) {
1152         // select one test (leading test) that is going to be executed and then pack along as many
1153         // other compatible instances as possible.
1154
1155         TestIdentifier leadingTest = null;
1156         for (TestIdentifier test : pool) {
1157             if (!mRemainingTests.contains(test)) {
1158                 continue;
1159             }
1160             if (requiredConfig != null &&
1161                     !mInstanceListerner.isPendingTestInstance(test, requiredConfig)) {
1162                 continue;
1163             }
1164             leadingTest = test;
1165             break;
1166         }
1167
1168         // no remaining tests?
1169         if (leadingTest == null) {
1170             return null;
1171         }
1172
1173         BatchRunConfiguration leadingTestConfig = null;
1174         if (requiredConfig != null) {
1175             leadingTestConfig = requiredConfig;
1176         } else {
1177             for (BatchRunConfiguration runConfig : getTestRunConfigs(leadingTest)) {
1178                 if (mInstanceListerner.isPendingTestInstance(leadingTest, runConfig)) {
1179                     leadingTestConfig = runConfig;
1180                     break;
1181                 }
1182             }
1183         }
1184
1185         // test pending <=> test has a pending config
1186         if (leadingTestConfig == null) {
1187             throw new AssertionError("search postcondition failed");
1188         }
1189
1190         final int leadingInstability = getTestInstabilityRating(leadingTest);
1191
1192         final TestBatch runBatch = new TestBatch();
1193         runBatch.config = leadingTestConfig;
1194         runBatch.tests = new ArrayList<>();
1195         runBatch.tests.add(leadingTest);
1196
1197         for (TestIdentifier test : pool) {
1198             if (test == leadingTest) {
1199                 // do not re-select the leading tests
1200                 continue;
1201             }
1202             if (!mInstanceListerner.isPendingTestInstance(test, leadingTestConfig)) {
1203                 // select only compatible
1204                 continue;
1205             }
1206             if (getTestInstabilityRating(test) != leadingInstability) {
1207                 // pack along only cases in the same stability category. Packing more dangerous
1208                 // tests along jeopardizes the stability of this run. Packing more stable tests
1209                 // along jeopardizes their stability rating.
1210                 continue;
1211             }
1212             if (runBatch.tests.size() >= getBatchSizeLimitForInstability(leadingInstability)) {
1213                 // batch size is limited.
1214                 break;
1215             }
1216             runBatch.tests.add(test);
1217         }
1218
1219         return runBatch;
1220     }
1221
1222     private int getBatchNumPendingCases(TestBatch batch) {
1223         int numPending = 0;
1224         for (TestIdentifier test : batch.tests) {
1225             if (mInstanceListerner.isPendingTestInstance(test, batch.config)) {
1226                 ++numPending;
1227             }
1228         }
1229         return numPending;
1230     }
1231
1232     private int getBatchSizeLimitForInstability(int batchInstabilityRating) {
1233         // reduce group size exponentially down to one
1234         return Math.max(1, TESTCASE_BATCH_LIMIT / (1 << batchInstabilityRating));
1235     }
1236
1237     private int getTestInstabilityRating(TestIdentifier testId) {
1238         if (mTestInstabilityRatings.containsKey(testId)) {
1239             return mTestInstabilityRatings.get(testId);
1240         } else {
1241             return 0;
1242         }
1243     }
1244
1245     private void recordTestInstability(TestIdentifier testId) {
1246         mTestInstabilityRatings.put(testId, getTestInstabilityRating(testId) + 1);
1247     }
1248
1249     private void clearTestInstability(TestIdentifier testId) {
1250         mTestInstabilityRatings.put(testId, 0);
1251     }
1252
1253     /**
1254      * Executes all tests on the device.
1255      */
1256     private void runTests() throws DeviceNotAvailableException, CapabilityQueryFailureException {
1257         for (;;) {
1258             TestBatch batch = selectRunBatch(mRemainingTests, null);
1259
1260             if (batch == null) {
1261                 break;
1262             }
1263
1264             runTestRunBatch(batch);
1265         }
1266     }
1267
1268     /**
1269      * Runs a TestBatch by either faking it or executing it on a device.
1270      */
1271     private void runTestRunBatch(TestBatch batch) throws DeviceNotAvailableException,
1272             CapabilityQueryFailureException {
1273         // prepare instance listener
1274         mInstanceListerner.setCurrentConfig(batch.config);
1275         for (TestIdentifier test : batch.tests) {
1276             mInstanceListerner.setTestInstances(test, getTestRunConfigs(test));
1277         }
1278
1279         // execute only if config is executable, else fake results
1280         if (isSupportedRunConfiguration(batch.config)) {
1281             executeTestRunBatch(batch);
1282         } else {
1283             if (batch.config.isRequired()) {
1284                 fakeFailTestRunBatch(batch);
1285             } else {
1286                 fakePassTestRunBatch(batch);
1287             }
1288         }
1289     }
1290
1291     private boolean isSupportedRunConfiguration(BatchRunConfiguration runConfig)
1292             throws DeviceNotAvailableException, CapabilityQueryFailureException {
1293         // orientation support
1294         if (!BatchRunConfiguration.ROTATION_UNSPECIFIED.equals(runConfig.getRotation())) {
1295             final Set<String> features = getDeviceFeatures(mDevice);
1296
1297             if (isPortraitClassRotation(runConfig.getRotation()) &&
1298                     !features.contains(FEATURE_PORTRAIT)) {
1299                 return false;
1300             }
1301             if (isLandscapeClassRotation(runConfig.getRotation()) &&
1302                     !features.contains(FEATURE_LANDSCAPE)) {
1303                 return false;
1304             }
1305         }
1306
1307         if (isOpenGlEsPackage()) {
1308             // renderability support for OpenGL ES tests
1309             return isSupportedGlesRenderConfig(runConfig);
1310         } else {
1311             return true;
1312         }
1313     }
1314
1315     private static final class AdbComLinkOpenError extends Exception {
1316         public AdbComLinkOpenError(String description, Throwable inner) {
1317             super(description, inner);
1318         }
1319     }
1320
1321     private static final class AdbComLinkKilledError extends Exception {
1322         public AdbComLinkKilledError(String description, Throwable inner) {
1323             super(description, inner);
1324         }
1325     }
1326
1327     /**
1328      * Executes a given command in adb shell
1329      *
1330      * @throws AdbComLinkOpenError if connection cannot be established.
1331      * @throws AdbComLinkKilledError if established connection is killed prematurely.
1332      */
1333     private void executeShellCommandAndReadOutput(final String command,
1334             final IShellOutputReceiver receiver)
1335             throws AdbComLinkOpenError, AdbComLinkKilledError {
1336         try {
1337             mDevice.getIDevice().executeShellCommand(command, receiver,
1338                     UNRESPONSIVE_CMD_TIMEOUT_MS, TimeUnit.MILLISECONDS);
1339         } catch (TimeoutException ex) {
1340             // Opening connection timed out
1341             throw new AdbComLinkOpenError("opening connection timed out", ex);
1342         } catch (AdbCommandRejectedException ex) {
1343             // Command rejected
1344             throw new AdbComLinkOpenError("command rejected", ex);
1345         } catch (IOException ex) {
1346             // shell command channel killed
1347             throw new AdbComLinkKilledError("command link killed", ex);
1348         } catch (ShellCommandUnresponsiveException ex) {
1349             // shell command halted
1350             throw new AdbComLinkKilledError("command link hung", ex);
1351         }
1352     }
1353
1354     /**
1355      * Executes given test batch on a device
1356      */
1357     private void executeTestRunBatch(TestBatch batch) throws DeviceNotAvailableException {
1358         // attempt full run once
1359         executeTestRunBatchRun(batch);
1360
1361         // split remaining tests to two sub batches and execute both. This will terminate
1362         // since executeTestRunBatchRun will always progress for a batch of size 1.
1363         final ArrayList<TestIdentifier> pendingTests = new ArrayList<>();
1364
1365         for (TestIdentifier test : batch.tests) {
1366             if (mInstanceListerner.isPendingTestInstance(test, batch.config)) {
1367                 pendingTests.add(test);
1368             }
1369         }
1370
1371         final int divisorNdx = pendingTests.size() / 2;
1372         final List<TestIdentifier> headList = pendingTests.subList(0, divisorNdx);
1373         final List<TestIdentifier> tailList = pendingTests.subList(divisorNdx, pendingTests.size());
1374
1375         // head
1376         for (;;) {
1377             TestBatch subBatch = selectRunBatch(headList, batch.config);
1378
1379             if (subBatch == null) {
1380                 break;
1381             }
1382
1383             executeTestRunBatch(subBatch);
1384         }
1385
1386         // tail
1387         for (;;) {
1388             TestBatch subBatch = selectRunBatch(tailList, batch.config);
1389
1390             if (subBatch == null) {
1391                 break;
1392             }
1393
1394             executeTestRunBatch(subBatch);
1395         }
1396
1397         if (getBatchNumPendingCases(batch) != 0) {
1398             throw new AssertionError("executeTestRunBatch postcondition failed");
1399         }
1400     }
1401
1402     /**
1403      * Runs one execution pass over the given batch.
1404      *
1405      * Tries to run the batch. Always makes progress (executes instances or modifies stability
1406      * scores).
1407      */
1408     private void executeTestRunBatchRun(TestBatch batch) throws DeviceNotAvailableException {
1409         if (getBatchNumPendingCases(batch) != batch.tests.size()) {
1410             throw new AssertionError("executeTestRunBatchRun precondition failed");
1411         }
1412
1413         checkInterrupted(); // throws if interrupted
1414
1415         final String testCases = generateTestCaseTrie(batch.tests);
1416
1417         mDevice.executeShellCommand("rm " + CASE_LIST_FILE_NAME);
1418         mDevice.executeShellCommand("rm " + LOG_FILE_NAME);
1419         mDevice.pushString(testCases + "\n", CASE_LIST_FILE_NAME);
1420
1421         final String instrumentationName =
1422                 "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
1423
1424         final StringBuilder deqpCmdLine = new StringBuilder();
1425         deqpCmdLine.append("--deqp-caselist-file=");
1426         deqpCmdLine.append(CASE_LIST_FILE_NAME);
1427         deqpCmdLine.append(" ");
1428         deqpCmdLine.append(getRunConfigDisplayCmdLine(batch.config));
1429
1430         // If we are not logging data, do not bother outputting the images from the test exe.
1431         if (!mLogData) {
1432             deqpCmdLine.append(" --deqp-log-images=disable");
1433         }
1434
1435         deqpCmdLine.append(" --deqp-watchdog=enable");
1436
1437         final String command = String.format(
1438                 "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\""
1439                     + " -e deqpLogData \"%s\" %s",
1440                 AbiUtils.createAbiFlag(mAbi.getName()), LOG_FILE_NAME, deqpCmdLine.toString(),
1441                 mLogData, instrumentationName);
1442
1443         final int numRemainingInstancesBefore = getNumRemainingInstances();
1444         final InstrumentationParser parser = new InstrumentationParser(mInstanceListerner);
1445         Throwable interruptingError = null;
1446
1447         try {
1448             executeShellCommandAndReadOutput(command, parser);
1449         } catch (Throwable ex) {
1450             interruptingError = ex;
1451         } finally {
1452             parser.flush();
1453         }
1454
1455         final boolean progressedSinceLastCall = mInstanceListerner.getCurrentTestId() != null ||
1456                 getNumRemainingInstances() < numRemainingInstancesBefore;
1457
1458         if (progressedSinceLastCall) {
1459             mDeviceRecovery.onExecutionProgressed();
1460         }
1461
1462         // interrupted, try to recover
1463         if (interruptingError != null) {
1464             if (interruptingError instanceof AdbComLinkOpenError) {
1465                 mDeviceRecovery.recoverConnectionRefused();
1466             } else if (interruptingError instanceof AdbComLinkKilledError) {
1467                 mDeviceRecovery.recoverComLinkKilled();
1468             } else if (interruptingError instanceof RunInterruptedException) {
1469                 // external run interruption request. Terminate immediately.
1470                 throw (RunInterruptedException)interruptingError;
1471             } else {
1472                 CLog.e(interruptingError);
1473                 throw new RuntimeException(interruptingError);
1474             }
1475
1476             // recoverXXX did not throw => recovery succeeded
1477         } else if (!parser.wasSuccessful()) {
1478             mDeviceRecovery.recoverComLinkKilled();
1479             // recoverXXX did not throw => recovery succeeded
1480         }
1481
1482         // Progress guarantees.
1483         if (batch.tests.size() == 1) {
1484             final TestIdentifier onlyTest = batch.tests.iterator().next();
1485             final boolean wasTestExecuted =
1486                     !mInstanceListerner.isPendingTestInstance(onlyTest, batch.config) &&
1487                     mInstanceListerner.getCurrentTestId() == null;
1488             final boolean wasLinkFailure = !parser.wasSuccessful() || interruptingError != null;
1489
1490             // Link failures can be caused by external events, require at least two observations
1491             // until bailing.
1492             if (!wasTestExecuted && (!wasLinkFailure || getTestInstabilityRating(onlyTest) > 0)) {
1493                 recordTestInstability(onlyTest);
1494                 // If we cannot finish the test, mark the case as a crash.
1495                 //
1496                 // If we couldn't even start the test, fail the test instance as non-executable.
1497                 // This is required so that a consistently crashing or non-existent tests will
1498                 // not cause futile (non-terminating) re-execution attempts.
1499                 if (mInstanceListerner.getCurrentTestId() != null) {
1500                     mInstanceListerner.abortTest(onlyTest, INCOMPLETE_LOG_MESSAGE);
1501                 } else {
1502                     mInstanceListerner.abortTest(onlyTest, NOT_EXECUTABLE_LOG_MESSAGE);
1503                 }
1504             } else if (wasTestExecuted) {
1505                 clearTestInstability(onlyTest);
1506             }
1507         }
1508         else
1509         {
1510             // Analyze results to update test stability ratings. If there is no interrupting test
1511             // logged, increase instability rating of all remaining tests. If there is a
1512             // interrupting test logged, increase only its instability rating.
1513             //
1514             // A successful run of tests clears instability rating.
1515             if (mInstanceListerner.getCurrentTestId() == null) {
1516                 for (TestIdentifier test : batch.tests) {
1517                     if (mInstanceListerner.isPendingTestInstance(test, batch.config)) {
1518                         recordTestInstability(test);
1519                     } else {
1520                         clearTestInstability(test);
1521                     }
1522                 }
1523             } else {
1524                 recordTestInstability(mInstanceListerner.getCurrentTestId());
1525                 for (TestIdentifier test : batch.tests) {
1526                     // \note: isPendingTestInstance is false for getCurrentTestId. Current ID is
1527                     // considered 'running' and will be restored to 'pending' in endBatch().
1528                     if (!test.equals(mInstanceListerner.getCurrentTestId()) &&
1529                             !mInstanceListerner.isPendingTestInstance(test, batch.config)) {
1530                         clearTestInstability(test);
1531                     }
1532                 }
1533             }
1534         }
1535
1536         mInstanceListerner.endBatch();
1537     }
1538
1539     private static String getRunConfigDisplayCmdLine(BatchRunConfiguration runConfig) {
1540         final StringBuilder deqpCmdLine = new StringBuilder();
1541         if (!runConfig.getGlConfig().isEmpty()) {
1542             deqpCmdLine.append("--deqp-gl-config-name=");
1543             deqpCmdLine.append(runConfig.getGlConfig());
1544         }
1545         if (!runConfig.getRotation().isEmpty()) {
1546             if (deqpCmdLine.length() != 0) {
1547                 deqpCmdLine.append(" ");
1548             }
1549             deqpCmdLine.append("--deqp-screen-rotation=");
1550             deqpCmdLine.append(runConfig.getRotation());
1551         }
1552         if (!runConfig.getSurfaceType().isEmpty()) {
1553             if (deqpCmdLine.length() != 0) {
1554                 deqpCmdLine.append(" ");
1555             }
1556             deqpCmdLine.append("--deqp-surface-type=");
1557             deqpCmdLine.append(runConfig.getSurfaceType());
1558         }
1559         return deqpCmdLine.toString();
1560     }
1561
1562     private int getNumRemainingInstances() {
1563         int retVal = 0;
1564         for (TestIdentifier testId : mRemainingTests) {
1565             // If case is in current working set, sum only not yet executed instances.
1566             // If case is not in current working set, sum all instances (since they are not yet
1567             // executed).
1568             if (mInstanceListerner.mPendingResults.containsKey(testId)) {
1569                 retVal += mInstanceListerner.mPendingResults.get(testId).remainingConfigs.size();
1570             } else {
1571                 retVal += mTestInstances.get(testId).size();
1572             }
1573         }
1574         return retVal;
1575     }
1576
1577     /**
1578      * Checks if this execution has been marked as interrupted and throws if it has.
1579      */
1580     private void checkInterrupted() throws RunInterruptedException {
1581         // Work around the API. RunUtil::checkInterrupted is private but we can call it indirectly
1582         // by sleeping a value <= 0.
1583         mRunUtil.sleep(0);
1584     }
1585
1586     /**
1587      * Pass given batch tests without running it
1588      */
1589     private void fakePassTestRunBatch(TestBatch batch) {
1590         for (TestIdentifier test : batch.tests) {
1591             CLog.d("Marking '%s' invocation in config '%s' as passed without running", test.toString(),
1592                     batch.config.getId());
1593             mInstanceListerner.skipTest(test);
1594         }
1595     }
1596
1597     /**
1598      * Fail given batch tests without running it
1599      */
1600     private void fakeFailTestRunBatch(TestBatch batch) {
1601         for (TestIdentifier test : batch.tests) {
1602             CLog.d("Marking '%s' invocation in config '%s' as failed without running", test.toString(),
1603                     batch.config.getId());
1604             mInstanceListerner.abortTest(test, "Required config not supported");
1605         }
1606     }
1607
1608     /**
1609      * Pass all remaining tests without running them
1610      */
1611     private void fakePassTests(ITestInvocationListener listener) {
1612         Map <String, String> emptyMap = Collections.emptyMap();
1613         for (TestIdentifier test : mRemainingTests) {
1614             CLog.d("Skipping test '%s', Opengl ES version not supported", test.toString());
1615             listener.testStarted(test);
1616             listener.testEnded(test, emptyMap);
1617         }
1618         mRemainingTests.clear();
1619     }
1620
1621     /**
1622      * Check if device supports Vulkan.
1623      */
1624     private boolean isSupportedVulkan ()
1625             throws DeviceNotAvailableException, CapabilityQueryFailureException {
1626         final Set<String> features = getDeviceFeatures(mDevice);
1627
1628         for (String feature : features) {
1629             if (feature.startsWith(FEATURE_VULKAN_LEVEL)) {
1630                 return true;
1631             }
1632         }
1633
1634         return false;
1635     }
1636
1637     /**
1638      * Check if device supports OpenGL ES version.
1639      */
1640     private static boolean isSupportedGles(ITestDevice device, int requiredMajorVersion,
1641             int requiredMinorVersion) throws DeviceNotAvailableException {
1642         String roOpenglesVersion = device.getProperty("ro.opengles.version");
1643
1644         if (roOpenglesVersion == null)
1645             return false;
1646
1647         int intValue = Integer.parseInt(roOpenglesVersion);
1648
1649         int majorVersion = ((intValue & 0xffff0000) >> 16);
1650         int minorVersion = (intValue & 0xffff);
1651
1652         return (majorVersion > requiredMajorVersion)
1653                 || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion);
1654     }
1655
1656     /**
1657      * Query if rendertarget is supported
1658      */
1659     private boolean isSupportedGlesRenderConfig(BatchRunConfiguration runConfig)
1660             throws DeviceNotAvailableException, CapabilityQueryFailureException {
1661         // query if configuration is supported
1662         final StringBuilder configCommandLine =
1663                 new StringBuilder(getRunConfigDisplayCmdLine(runConfig));
1664         if (configCommandLine.length() != 0) {
1665             configCommandLine.append(" ");
1666         }
1667         configCommandLine.append("--deqp-gl-major-version=");
1668         configCommandLine.append(getGlesMajorVersion());
1669         configCommandLine.append(" --deqp-gl-minor-version=");
1670         configCommandLine.append(getGlesMinorVersion());
1671
1672         final String commandLine = configCommandLine.toString();
1673
1674         // check for cached result first
1675         if (mConfigQuerySupportCache.containsKey(commandLine)) {
1676             return mConfigQuerySupportCache.get(commandLine);
1677         }
1678
1679         final boolean supported = queryIsSupportedConfigCommandLine(commandLine);
1680         mConfigQuerySupportCache.put(commandLine, supported);
1681         return supported;
1682     }
1683
1684     private boolean queryIsSupportedConfigCommandLine(String deqpCommandLine)
1685             throws DeviceNotAvailableException, CapabilityQueryFailureException {
1686         final String instrumentationName =
1687                 "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation";
1688         final String command = String.format(
1689                 "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine \"%s\""
1690                     + " %s",
1691                 AbiUtils.createAbiFlag(mAbi.getName()), deqpCommandLine, instrumentationName);
1692
1693         final PlatformQueryInstrumentationParser parser = new PlatformQueryInstrumentationParser();
1694         mDevice.executeShellCommand(command, parser);
1695         parser.flush();
1696
1697         if (parser.wasSuccessful() && parser.getResultCode() == 0 &&
1698                 parser.getResultMap().containsKey("Supported")) {
1699             if ("Yes".equals(parser.getResultMap().get("Supported"))) {
1700                 return true;
1701             } else if ("No".equals(parser.getResultMap().get("Supported"))) {
1702                 return false;
1703             } else {
1704                 CLog.e("Capability query did not return a result");
1705                 throw new CapabilityQueryFailureException();
1706             }
1707         } else if (parser.wasSuccessful()) {
1708             CLog.e("Failed to run capability query. Code: %d, Result: %s",
1709                     parser.getResultCode(), parser.getResultMap().toString());
1710             throw new CapabilityQueryFailureException();
1711         } else {
1712             CLog.e("Failed to run capability query");
1713             throw new CapabilityQueryFailureException();
1714         }
1715     }
1716
1717     /**
1718      * Return feature set supported by the device
1719      */
1720     private Set<String> getDeviceFeatures(ITestDevice device)
1721             throws DeviceNotAvailableException, CapabilityQueryFailureException {
1722         if (mDeviceFeatures == null) {
1723             mDeviceFeatures = queryDeviceFeatures(device);
1724         }
1725         return mDeviceFeatures;
1726     }
1727
1728     /**
1729      * Query feature set supported by the device
1730      */
1731     private static Set<String> queryDeviceFeatures(ITestDevice device)
1732             throws DeviceNotAvailableException, CapabilityQueryFailureException {
1733         // NOTE: Almost identical code in BaseDevicePolicyTest#hasDeviceFeatures
1734         // TODO: Move this logic to ITestDevice.
1735         String command = "pm list features";
1736         String commandOutput = device.executeShellCommand(command);
1737
1738         // Extract the id of the new user.
1739         HashSet<String> availableFeatures = new HashSet<>();
1740         for (String feature: commandOutput.split("\\s+")) {
1741             // Each line in the output of the command has the format "feature:{FEATURE_VALUE}".
1742             String[] tokens = feature.split(":");
1743             if (tokens.length < 2 || !"feature".equals(tokens[0])) {
1744                 CLog.e("Failed parse features. Unexpect format on line \"%s\"", tokens[0]);
1745                 throw new CapabilityQueryFailureException();
1746             }
1747             availableFeatures.add(tokens[1]);
1748         }
1749         return availableFeatures;
1750     }
1751
1752     private boolean isPortraitClassRotation(String rotation) {
1753         return BatchRunConfiguration.ROTATION_PORTRAIT.equals(rotation) ||
1754                 BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT.equals(rotation);
1755     }
1756
1757     private boolean isLandscapeClassRotation(String rotation) {
1758         return BatchRunConfiguration.ROTATION_LANDSCAPE.equals(rotation) ||
1759                 BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE.equals(rotation);
1760     }
1761
1762     /**
1763      * Install dEQP OnDevice Package
1764      */
1765     private void installTestApk() throws DeviceNotAvailableException {
1766         try {
1767             File apkFile = new File(mBuildHelper.getTestsDir(), DEQP_ONDEVICE_APK);
1768             String[] options = {AbiUtils.createAbiFlag(mAbi.getName())};
1769             String errorCode = getDevice().installPackage(apkFile, true, options);
1770             if (errorCode != null) {
1771                 CLog.e("Failed to install %s. Reason: %s", DEQP_ONDEVICE_APK, errorCode);
1772             }
1773         } catch (FileNotFoundException e) {
1774             CLog.e("Could not find test apk %s", DEQP_ONDEVICE_APK);
1775         }
1776     }
1777
1778     /**
1779      * Uninstall dEQP OnDevice Package
1780      */
1781     private void uninstallTestApk() throws DeviceNotAvailableException {
1782         getDevice().uninstallPackage(DEQP_ONDEVICE_PKG);
1783     }
1784
1785     /**
1786      * Parse gl nature from package name
1787      */
1788     private boolean isOpenGlEsPackage() {
1789         if ("dEQP-GLES2".equals(mDeqpPackage) || "dEQP-GLES3".equals(mDeqpPackage) ||
1790                 "dEQP-GLES31".equals(mDeqpPackage)) {
1791             return true;
1792         } else if ("dEQP-EGL".equals(mDeqpPackage) ||
1793                 "dEQP-VK".equals(mDeqpPackage)) {
1794             return false;
1795         } else {
1796             throw new IllegalStateException("dEQP runner was created with illegal name");
1797         }
1798     }
1799
1800     /**
1801      * Parse vulkan nature from package name
1802      */
1803     private boolean isVulkanPackage() {
1804         if ("dEQP-GLES2".equals(mDeqpPackage) || "dEQP-GLES3".equals(mDeqpPackage) ||
1805                 "dEQP-GLES31".equals(mDeqpPackage) || "dEQP-EGL".equals(mDeqpPackage)) {
1806             return false;
1807         } else if ("dEQP-VK".equals(mDeqpPackage)) {
1808             return true;
1809         } else {
1810             throw new IllegalStateException("dEQP runner was created with illegal name");
1811         }
1812     }
1813
1814     /**
1815      * Check GL support (based on package name)
1816      */
1817     private boolean isSupportedGles() throws DeviceNotAvailableException {
1818         return isSupportedGles(mDevice, getGlesMajorVersion(), getGlesMinorVersion());
1819     }
1820
1821     /**
1822      * Get GL major version (based on package name)
1823      */
1824     private int getGlesMajorVersion() {
1825         if ("dEQP-GLES2".equals(mDeqpPackage)) {
1826             return 2;
1827         } else if ("dEQP-GLES3".equals(mDeqpPackage)) {
1828             return 3;
1829         } else if ("dEQP-GLES31".equals(mDeqpPackage)) {
1830             return 3;
1831         } else {
1832             throw new IllegalStateException("getGlesMajorVersion called for non gles pkg");
1833         }
1834     }
1835
1836     /**
1837      * Get GL minor version (based on package name)
1838      */
1839     private int getGlesMinorVersion() {
1840         if ("dEQP-GLES2".equals(mDeqpPackage)) {
1841             return 0;
1842         } else if ("dEQP-GLES3".equals(mDeqpPackage)) {
1843             return 0;
1844         } else if ("dEQP-GLES31".equals(mDeqpPackage)) {
1845             return 1;
1846         } else {
1847             throw new IllegalStateException("getGlesMinorVersion called for non gles pkg");
1848         }
1849     }
1850
1851     private static List<Pattern> getPatternFilters(List<String> filters) {
1852         List<Pattern> patterns = new ArrayList<Pattern>();
1853         for (String filter : filters) {
1854             if (filter.contains("*")) {
1855                 patterns.add(Pattern.compile(filter.replace(".","\\.").replace("*",".*")));
1856             }
1857         }
1858         return patterns;
1859     }
1860
1861     private static Set<String> getNonPatternFilters(List<String> filters) {
1862         Set<String> nonPatternFilters = new HashSet<String>();
1863         for (String filter : filters) {
1864             if (!filter.contains("*")) {
1865                 // Deqp usesly only dots for separating between parts of the names
1866                 // Convert last dot to hash if needed.
1867                 if (!filter.contains("#")) {
1868                     int lastSeparator = filter.lastIndexOf('.');
1869                     String filterWithHash = filter.substring(0, lastSeparator) + "#" +
1870                         filter.substring(lastSeparator + 1, filter.length());
1871                     nonPatternFilters.add(filterWithHash);
1872                 }
1873                 else {
1874                     nonPatternFilters.add(filter);
1875                 }
1876             }
1877         }
1878         return nonPatternFilters;
1879     }
1880
1881     private static boolean matchesAny(TestIdentifier test, List<Pattern> patterns) {
1882         for (Pattern pattern : patterns) {
1883             if (pattern.matcher(test.toString()).matches()) {
1884                 return true;
1885             }
1886         }
1887         return false;
1888     }
1889
1890     /**
1891      * Filter tests with the option of filtering by pattern.
1892      *
1893      * '*' is 0 or more characters.
1894      * '.' is interpreted verbatim.
1895      */
1896     private static void filterTests(Map<TestIdentifier, Set<BatchRunConfiguration>> tests,
1897                                     List<String> includeFilters,
1898                                     List<String> excludeFilters) {
1899         // We could filter faster by building the test case tree.
1900         // Let's see if this is fast enough.
1901         Set<String> includeStrings = getNonPatternFilters(includeFilters);
1902         Set<String> excludeStrings = getNonPatternFilters(excludeFilters);
1903         List<Pattern> includePatterns = getPatternFilters(includeFilters);
1904         List<Pattern> excludePatterns = getPatternFilters(excludeFilters);
1905
1906         List<TestIdentifier> testList = new ArrayList<>(tests.keySet());
1907         for (TestIdentifier test : testList) {
1908             if (excludeStrings.contains(test.toString())) {
1909                 tests.remove(test); // remove test if explicitly excluded
1910                 continue;
1911             }
1912             boolean includesExist = !includeStrings.isEmpty() || !includePatterns.isEmpty();
1913             boolean testIsIncluded = includeStrings.contains(test.toString())
1914                     || matchesAny(test, includePatterns);
1915             if ((includesExist && !testIsIncluded) || matchesAny(test, excludePatterns)) {
1916                 // if this test isn't included and other tests are,
1917                 // or if test matches exclude pattern, exclude test
1918                 tests.remove(test);
1919             }
1920         }
1921     }
1922
1923     /**
1924      * Read a list of filters from a file.
1925      *
1926      * Note: Filters can be numerous so we prefer, for performance
1927      * reasons, to add directly to the target list instead of using
1928      * intermediate return value.
1929      */
1930     static private void readFilterFile(List<String> filterList, File file) throws FileNotFoundException {
1931         if (!file.canRead()) {
1932             CLog.e("Failed to read filter file '%s'", file.getPath());
1933             throw new FileNotFoundException();
1934         }
1935         try (Reader plainReader = new FileReader(file);
1936              BufferedReader reader = new BufferedReader(plainReader)) {
1937             String filter = "";
1938             while ((filter = reader.readLine()) != null) {
1939                 // TOOD: Sanity check filter
1940                 filterList.add(filter);
1941             }
1942             // Rely on try block to autoclose
1943         }
1944         catch (IOException e)
1945         {
1946             throw new RuntimeException("Failed to read filter list file '" + file.getPath() + "': " +
1947                      e.getMessage());
1948         }
1949     }
1950
1951     /**
1952      * Prints filters into debug log stream, limiting to 20 entries.
1953      */
1954     static private void printFilters(List<String> filters) {
1955         int numPrinted = 0;
1956         for (String filter : filters) {
1957             CLog.d("    %s", filter);
1958             if (++numPrinted == 20) {
1959                 CLog.d("    ... AND %d others", filters.size() - numPrinted);
1960                 break;
1961             }
1962         }
1963     }
1964
1965     /**
1966      * Loads tests into mTestInstances based on the options. Assumes
1967      * that no tests have been loaded for this instance before.
1968      */
1969     private void loadTests() {
1970         if (mTestInstances != null) throw new AssertionError("Re-load of tests not supported");
1971
1972         try {
1973             Reader reader = mCaselistReader;
1974             if (reader == null) {
1975                 File testlist = new File(mBuildHelper.getTestsDir(), mCaselistFile);
1976                 if (!testlist.isFile()) {
1977                     throw new FileNotFoundException();
1978                 }
1979                 reader = new FileReader(testlist);
1980             }
1981             mTestInstances = generateTestInstances(reader, mConfigName, mScreenRotation, mSurfaceType, mConfigRequired);
1982             mCaselistReader = null;
1983             reader.close();
1984         }
1985         catch (FileNotFoundException e) {
1986             throw new RuntimeException("Cannot read deqp test list file: "  + mCaselistFile);
1987         }
1988         catch (IOException e) {
1989             CLog.w("Failed to close test list reader.");
1990         }
1991
1992         try
1993         {
1994             for (String filterFile : mIncludeFilterFiles) {
1995                 CLog.d("Read include filter file '%s'", filterFile);
1996                 File file = new File(mBuildHelper.getTestsDir(), filterFile);
1997                 readFilterFile(mIncludeFilters, file);
1998             }
1999             for (String filterFile : mExcludeFilterFiles) {
2000                 CLog.d("Read exclude filter file '%s'", filterFile);
2001                 File file = new File(mBuildHelper.getTestsDir(), filterFile);
2002                 readFilterFile(mExcludeFilters, file);
2003             }
2004         }
2005         catch (FileNotFoundException e) {
2006             throw new RuntimeException("Cannot read deqp filter list file:" + e.getMessage());
2007         }
2008
2009         CLog.d("Include filters:");
2010         printFilters(mIncludeFilters);
2011         CLog.d("Exclude filters:");
2012         printFilters(mExcludeFilters);
2013
2014         long originalTestCount = mTestInstances.size();
2015         CLog.i("Num tests before filtering: %d", originalTestCount);
2016         if ((!mIncludeFilters.isEmpty() || !mExcludeFilters.isEmpty()) && originalTestCount > 0) {
2017             filterTests(mTestInstances, mIncludeFilters, mExcludeFilters);
2018
2019             // Update runtime estimation hint.
2020             if (mRuntimeHint != -1) {
2021                 mRuntimeHint = (mRuntimeHint * mTestInstances.size()) / originalTestCount;
2022             }
2023         }
2024         CLog.i("Num tests after filtering: %d", mTestInstances.size());
2025     }
2026
2027     /**
2028      * {@inheritDoc}
2029      */
2030     @Override
2031     public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
2032         final Map<String, String> emptyMap = Collections.emptyMap();
2033         // If sharded, split() will load the tests.
2034         if (mTestInstances == null) {
2035             loadTests();
2036         }
2037
2038         mRemainingTests = new LinkedList<>(mTestInstances.keySet());
2039         long startTime = System.currentTimeMillis();
2040         listener.testRunStarted(getId(), mRemainingTests.size());
2041
2042         try {
2043             if (mRemainingTests.isEmpty()) {
2044                 CLog.d("No tests to run.");
2045                 return;
2046             }
2047             final boolean isSupportedApi = (isOpenGlEsPackage() && isSupportedGles())
2048                                             || (isVulkanPackage() && isSupportedVulkan())
2049                                             || (!isOpenGlEsPackage() && !isVulkanPackage());
2050
2051             if (!isSupportedApi || mCollectTestsOnly) {
2052                 // Pass all tests if OpenGL ES version is not supported or we are collecting
2053                 // the names of the tests only
2054                 fakePassTests(listener);
2055             } else if (!mRemainingTests.isEmpty()) {
2056                 // Make sure there is no pre-existing package form earlier interrupted test run.
2057                 uninstallTestApk();
2058                 installTestApk();
2059
2060                 mInstanceListerner.setSink(listener);
2061                 mDeviceRecovery.setDevice(mDevice);
2062                 runTests();
2063
2064                 uninstallTestApk();
2065             }
2066         } catch (CapabilityQueryFailureException ex) {
2067             // Platform is not behaving correctly, for example crashing when trying to create
2068             // a window. Instead of silenty failing, signal failure by leaving the rest of the
2069             // test cases in "NotExecuted" state
2070             CLog.e("Capability query failed - leaving tests unexecuted.");
2071             uninstallTestApk();
2072         } finally {
2073             listener.testRunEnded(System.currentTimeMillis() - startTime, emptyMap);
2074         }
2075     }
2076
2077    /**
2078      * {@inheritDoc}
2079      */
2080     @Override
2081     public void addIncludeFilter(String filter) {
2082         mIncludeFilters.add(filter);
2083     }
2084
2085     /**
2086      * {@inheritDoc}
2087      */
2088     @Override
2089     public void addAllIncludeFilters(Set<String> filters) {
2090         mIncludeFilters.addAll(filters);
2091     }
2092
2093     /**
2094      * {@inheritDoc}
2095      */
2096     @Override
2097     public void addExcludeFilter(String filter) {
2098         mExcludeFilters.add(filter);
2099     }
2100
2101     /**
2102      * {@inheritDoc}
2103      */
2104     @Override
2105     public void addAllExcludeFilters(Set<String> filters) {
2106         mExcludeFilters.addAll(filters);
2107     }
2108
2109     /**
2110      * {@inheritDoc}
2111      */
2112     @Override
2113     public void setCollectTestsOnly(boolean collectTests) {
2114         mCollectTestsOnly = collectTests;
2115     }
2116
2117     private static void copyOptions(DeqpTestRunner destination, DeqpTestRunner source) {
2118         destination.mDeqpPackage = source.mDeqpPackage;
2119         destination.mConfigName = source.mConfigName;
2120         destination.mCaselistFile = source.mCaselistFile;
2121         destination.mScreenRotation = source.mScreenRotation;
2122         destination.mSurfaceType = source.mSurfaceType;
2123         destination.mConfigRequired = source.mConfigRequired;
2124         destination.mIncludeFilters = new ArrayList<>(source.mIncludeFilters);
2125         destination.mIncludeFilterFiles = new ArrayList<>(source.mIncludeFilterFiles);
2126         destination.mExcludeFilters = new ArrayList<>(source.mExcludeFilters);
2127         destination.mExcludeFilterFiles = new ArrayList<>(source.mExcludeFilterFiles);
2128         destination.mAbi = source.mAbi;
2129         destination.mLogData = source.mLogData;
2130         destination.mCollectTestsOnly = source.mCollectTestsOnly;
2131     }
2132
2133     /**
2134      * Helper to update the RuntimeHint of the tests after being sharded.
2135      */
2136     private void updateRuntimeHint(long originalSize, Collection<IRemoteTest> runners) {
2137         if (originalSize > 0) {
2138             long fullRuntimeMs = getRuntimeHint();
2139             for (IRemoteTest remote: runners) {
2140                 DeqpTestRunner runner = (DeqpTestRunner)remote;
2141                 long shardRuntime = (fullRuntimeMs * runner.mTestInstances.size()) / originalSize;
2142                 runner.mRuntimeHint = shardRuntime;
2143             }
2144         }
2145     }
2146
2147     /**
2148      * {@inheritDoc}
2149      */
2150     @Override
2151     public Collection<IRemoteTest> split() {
2152         if (mTestInstances != null) {
2153             throw new AssertionError("Re-splitting or splitting running instance?");
2154         }
2155         // \todo [2015-11-23 kalle] If we split to batches at shard level, we could
2156         // basically get rid of batching. Except that sharding is optional?
2157
2158         // Assume that tests have not been yet loaded.
2159         loadTests();
2160
2161         Collection<IRemoteTest> runners = new ArrayList<>();
2162         // NOTE: Use linked hash map to keep the insertion order in iteration
2163         Map<TestIdentifier, Set<BatchRunConfiguration>> currentSet = new LinkedHashMap<>();
2164         Map<TestIdentifier, Set<BatchRunConfiguration>> iterationSet = this.mTestInstances;
2165
2166         if (iterationSet.keySet().isEmpty()) {
2167             CLog.i("Cannot split deqp tests, no tests to run");
2168             return null;
2169         }
2170
2171         // Go through tests, split
2172         for (TestIdentifier test: iterationSet.keySet()) {
2173             currentSet.put(test, iterationSet.get(test));
2174             if (currentSet.size() >= TESTCASE_BATCH_LIMIT) {
2175                 runners.add(new DeqpTestRunner(this, currentSet));
2176                 // NOTE: Use linked hash map to keep the insertion order in iteration
2177                 currentSet = new LinkedHashMap<>();
2178             }
2179         }
2180         runners.add(new DeqpTestRunner(this, currentSet));
2181
2182         // Compute new runtime hints
2183         updateRuntimeHint(iterationSet.size(), runners);
2184         CLog.i("Split deqp tests into %d shards", runners.size());
2185         return runners;
2186     }
2187
2188     /**
2189      * This sharding should be deterministic for the same input and independent.
2190      * Through this API, each shard could be executed on different machine.
2191      */
2192     @Override
2193     public IRemoteTest getTestShard(int shardCount, int shardIndex) {
2194         // TODO: refactor getTestshard and split to share some logic.
2195         if (mTestInstances == null) {
2196             loadTests();
2197         }
2198
2199         List<IRemoteTest> runners = new ArrayList<>();
2200         // NOTE: Use linked hash map to keep the insertion order in iteration
2201         Map<TestIdentifier, Set<BatchRunConfiguration>> currentSet = new LinkedHashMap<>();
2202         Map<TestIdentifier, Set<BatchRunConfiguration>> iterationSet = this.mTestInstances;
2203
2204         int batchLimit = iterationSet.keySet().size() / shardCount;
2205         int i = 1;
2206         // Go through tests, split
2207         for (TestIdentifier test: iterationSet.keySet()) {
2208             currentSet.put(test, iterationSet.get(test));
2209             if (currentSet.size() >= batchLimit && i < shardCount) {
2210                 runners.add(new DeqpTestRunner(this, currentSet));
2211                 i++;
2212                 // NOTE: Use linked hash map to keep the insertion order in iteration
2213                 currentSet = new LinkedHashMap<>();
2214             }
2215         }
2216         runners.add(new DeqpTestRunner(this, currentSet));
2217
2218         // Compute new runtime hints
2219         updateRuntimeHint(iterationSet.size(), runners);
2220
2221         // If too many shards were requested, we complete with placeholder.
2222         if (runners.size() < shardCount) {
2223             for (int j = runners.size(); j < shardCount; j++) {
2224                 runners.add(new DeqpTestRunner(this,
2225                         new LinkedHashMap<TestIdentifier, Set<BatchRunConfiguration>>()));
2226             }
2227         }
2228
2229         CLog.i("Split deqp tests into %d shards, return shard: %s", runners.size(), shardIndex);
2230         return runners.get(shardIndex);
2231     }
2232
2233     /**
2234      * {@inheritDoc}
2235      */
2236     @Override
2237     public long getRuntimeHint() {
2238         if (mRuntimeHint != -1) {
2239             return mRuntimeHint;
2240         }
2241         if (mTestInstances == null) {
2242             loadTests();
2243         }
2244         // Tests normally take something like ~100ms. Some take a
2245         // second. Let's guess 200ms per test.
2246         return 200 * mTestInstances.size();
2247     }
2248 }