import com.android.tradefed.testtype.IRemoteTest;
import com.android.tradefed.testtype.IRuntimeHintProvider;
import com.android.tradefed.testtype.IShardableTest;
-import com.android.tradefed.testtype.IStrictShardableTest;
import com.android.tradefed.testtype.ITestCollector;
import com.android.tradefed.testtype.ITestFilterReceiver;
import com.android.tradefed.util.AbiUtils;
@OptionClass(alias="deqp-test-runner")
public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
ITestFilterReceiver, IAbiReceiver, IShardableTest, ITestCollector,
- IRuntimeHintProvider, IStrictShardableTest {
+ IRuntimeHintProvider {
private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk";
private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp";
private static final String INCOMPLETE_LOG_MESSAGE = "Crash: Incomplete test log";
}
/**
- * This sharding should be deterministic for the same input and independent.
- * Through this API, each shard could be executed on different machine.
- */
- @Override
- public IRemoteTest getTestShard(int shardCount, int shardIndex) {
- // TODO: refactor getTestshard and split to share some logic.
- if (mTestInstances == null) {
- loadTests();
- }
-
- List<IRemoteTest> runners = new ArrayList<>();
- // NOTE: Use linked hash map to keep the insertion order in iteration
- Map<TestDescription, Set<BatchRunConfiguration>> currentSet = new LinkedHashMap<>();
- Map<TestDescription, Set<BatchRunConfiguration>> iterationSet = this.mTestInstances;
-
- int batchLimit = iterationSet.keySet().size() / shardCount;
- int i = 1;
- // Go through tests, split
- for (TestDescription test: iterationSet.keySet()) {
- currentSet.put(test, iterationSet.get(test));
- if (currentSet.size() >= batchLimit && i < shardCount) {
- runners.add(new DeqpTestRunner(this, currentSet));
- i++;
- // NOTE: Use linked hash map to keep the insertion order in iteration
- currentSet = new LinkedHashMap<>();
- }
- }
- runners.add(new DeqpTestRunner(this, currentSet));
-
- // Compute new runtime hints
- updateRuntimeHint(iterationSet.size(), runners);
-
- // If too many shards were requested, we complete with placeholder.
- if (runners.size() < shardCount) {
- for (int j = runners.size(); j < shardCount; j++) {
- runners.add(new DeqpTestRunner(this,
- new LinkedHashMap<TestDescription, Set<BatchRunConfiguration>>()));
- }
- }
-
- CLog.i("Split deqp tests into %d shards, return shard: %s", runners.size(), shardIndex);
- return runners.get(shardIndex);
- }
-
- /**
* {@inheritDoc}
*/
@Override
((IRuntimeHintProvider)shards.get(1)).getRuntimeHint());
}
- /**
- * Test that strict shardable is able to split deterministically the set of tests.
- */
- public void testGetTestShard() throws Exception {
- final int TEST_COUNT = 2237;
- final int SHARD_COUNT = 4;
-
- ArrayList<TestDescription> testIds = new ArrayList<>(TEST_COUNT);
- for (int i = 0; i < TEST_COUNT; i++) {
- testIds.add(new TestDescription("dEQP-GLES3.funny.group", String.valueOf(i)));
- }
-
- DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir);
- OptionSetter setter = new OptionSetter(deqpTest);
- final long fullRuntimeMs = testIds.size()*100;
- setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs));
-
- DeqpTestRunner shard1 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 0);
- assertEquals(559, shard1.getTestInstance().size());
- int j = 0;
- // Ensure numbers, and that order is stable
- for (TestDescription t : shard1.getTestInstance().keySet()) {
- assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
- String.format("%s#%s", t.getClassName(), t.getTestName()));
- j++;
- }
- DeqpTestRunner shard2 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 1);
- assertEquals(559, shard2.getTestInstance().size());
- for (TestDescription t : shard2.getTestInstance().keySet()) {
- assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
- String.format("%s#%s", t.getClassName(), t.getTestName()));
- j++;
- }
- DeqpTestRunner shard3 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 2);
- assertEquals(559, shard3.getTestInstance().size());
- for (TestDescription t : shard3.getTestInstance().keySet()) {
- assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
- String.format("%s#%s", t.getClassName(), t.getTestName()));
- j++;
- }
- DeqpTestRunner shard4 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 3);
- assertEquals(560, shard4.getTestInstance().size());
- for (TestDescription t : shard4.getTestInstance().keySet()) {
- assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
- String.format("%s#%s", t.getClassName(), t.getTestName()));
- j++;
- }
- assertEquals(TEST_COUNT, j);
- }
-
- /**
- * Test that strict shardable is creating an empty shard of the runner when too many shards
- * are requested.
- */
- public void testGetTestShard_tooManyShardRequested() throws Exception {
- final int TEST_COUNT = 2;
- final int SHARD_COUNT = 3;
-
- ArrayList<TestDescription> testIds = new ArrayList<>(TEST_COUNT);
- for (int i = 0; i < TEST_COUNT; i++) {
- testIds.add(new TestDescription("dEQP-GLES3.funny.group", String.valueOf(i)));
- }
- DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir);
- OptionSetter setter = new OptionSetter(deqpTest);
- final long fullRuntimeMs = testIds.size()*100;
- setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs));
- DeqpTestRunner shard1 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 0);
- assertEquals(1, shard1.getTestInstance().size());
- int j = 0;
- // Ensure numbers, and that order is stable
- for (TestDescription t : shard1.getTestInstance().keySet()) {
- assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
- String.format("%s#%s", t.getClassName(), t.getTestName()));
- j++;
- }
- DeqpTestRunner shard2 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 1);
- assertEquals(1, shard2.getTestInstance().size());
- for (TestDescription t : shard2.getTestInstance().keySet()) {
- assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
- String.format("%s#%s", t.getClassName(), t.getTestName()));
- j++;
- }
- DeqpTestRunner shard3 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 2);
- assertTrue(shard3.getTestInstance().isEmpty());
- assertEquals(TEST_COUNT, j);
- ITestInvocationListener mockListener
- = EasyMock.createStrictMock(ITestInvocationListener.class);
- mockListener.testRunStarted(EasyMock.anyObject(), EasyMock.eq(0));
- mockListener.testRunEnded(EasyMock.anyLong(), (Map<String, String>) EasyMock.anyObject());
- EasyMock.replay(mockListener);
- shard3.run(mockListener);
- EasyMock.verify(mockListener);
- }
-
public void testRuntimeHint_optionNotSet() throws Exception {
final TestDescription[] testIds = {
new TestDescription("dEQP-GLES3.info", "vendor"),