From e7dc520e589c702a3751646ff07d39d438bd120d Mon Sep 17 00:00:00 2001 From: Kalle Raita Date: Fri, 20 Nov 2015 14:17:45 -0800 Subject: [PATCH] Sharding support for deqp CTS v2 integration Change-Id: Ifb31382b6785a8cf1f3cefd67b1d281efbaa443a --- .../drawelements/deqp/runner/DeqpTestRunner.java | 120 +++++++++++-- .../deqp/runner/DeqpTestRunnerTest.java | 197 +++++++++++++++++---- 2 files changed, 266 insertions(+), 51 deletions(-) diff --git a/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java b/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java index 828e6a4..edb5a1d 100644 --- a/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java +++ b/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java @@ -38,6 +38,7 @@ import com.android.tradefed.testtype.IBuildReceiver; import com.android.tradefed.testtype.IAbiReceiver; import com.android.tradefed.testtype.IDeviceTest; import com.android.tradefed.testtype.IRemoteTest; +import com.android.tradefed.testtype.IShardableTest; import com.android.tradefed.testtype.ITestFilterReceiver; import com.android.tradefed.util.IRunUtil; import com.android.tradefed.util.RunInterruptedException; @@ -72,7 +73,8 @@ import java.util.concurrent.TimeUnit; * Supports running drawElements Quality Program tests found under external/deqp. */ @OptionClass(alias="deqp-test-runner") -public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, IRemoteTest, ITestFilterReceiver, IAbiReceiver { +public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, IRemoteTest, + ITestFilterReceiver, IAbiReceiver, IShardableTest { private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk"; private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp"; @@ -90,17 +92,29 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, IRemoteTest, private static final int UNRESPOSIVE_CMD_TIMEOUT_MS = 60000; // one minute - @Option(name="deqp-package", description="Name of the deqp module used. Determines GLES version.", importance=Option.Importance.ALWAYS) + // !NOTE: There's a static method copyOptions() for copying options during split. + // If you add state update copyOptions() as appropriate! + + @Option(name="deqp-package", + description="Name of the deqp module used. Determines GLES version.", + importance=Option.Importance.ALWAYS) private String mDeqpPackage; - @Option(name="deqp-gl-config-name", description="GL render target config. See deqp documentation for syntax. ", importance=Option.Importance.ALWAYS) + @Option(name="deqp-gl-config-name", + description="GL render target config. See deqp documentation for syntax. ", + importance=Option.Importance.ALWAYS) private String mConfigName; - @Option(name="deqp-caselist-file", description="File listing the names of the cases to be run.", importance=Option.Importance.ALWAYS) + @Option(name="deqp-caselist-file", + description="File listing the names of the cases to be run.", + importance=Option.Importance.ALWAYS) private String mCaselistFile; - @Option(name="deqp-screen-rotation", description="Screen orientation. Defaults to 'unspecified'", importance=Option.Importance.NEVER) + @Option(name="deqp-screen-rotation", + description="Screen orientation. Defaults to 'unspecified'", + importance=Option.Importance.NEVER) private String mScreenRotation = "unspecified"; - @Option(name="deqp-surface-type", description="Surface type ('window', 'pbuffer', 'fbo'). Defaults to 'window'", importance=Option.Importance.NEVER) + @Option(name="deqp-surface-type", + description="Surface type ('window', 'pbuffer', 'fbo'). Defaults to 'window'", + importance=Option.Importance.NEVER) private String mSurfaceType = "window"; - @Option(name = "include-filter", description="Test include filter. '*' is zero or more letters. '.' has no special meaning.") private List mIncludeFilters = new ArrayList<>(); @@ -121,11 +135,19 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, IRemoteTest, // When set will override the mCaselistFile for testing purposes. private Reader mCaselistReader = null; - private IRecovery mDeviceRecovery = new Recovery(); - { + private IRecovery mDeviceRecovery = new Recovery(); { mDeviceRecovery.setSleepProvider(new SleepProvider()); } + public DeqpTestRunner() { + } + + private DeqpTestRunner(DeqpTestRunner optionTemplate, + Map> tests) { + copyOptions(this, optionTemplate); + mTestInstances = tests; + } + /** * @param abi the ABI to run the test on */ @@ -1852,7 +1874,7 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, IRemoteTest, List includes = buildPatternList(includeFilters); List excludes = buildPatternList(excludeFilters); - List testList = new ArrayList(tests.keySet()); + List testList = new ArrayList(tests.keySet()); for (TestIdentifier test : testList) { // Remove test if it does not match includes or matches // excludes. @@ -1867,12 +1889,11 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, IRemoteTest, } /** - * {@inheritDoc} + * Loads tests into mTestInstances based on the options. Assumes + * that no tests have been loaded for this instance before. */ - @Override - public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { - final Map emptyMap = Collections.emptyMap(); - final boolean isSupportedApi = !isOpenGlEsPackage() || isSupportedGles(); + private void loadTests() { + if (mTestInstances != null) throw new AssertionError("Re-load of tests not supported"); try { Reader reader = mCaselistReader; @@ -1893,9 +1914,32 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, IRemoteTest, catch (IOException e) { CLog.w("Failed to close test list reader."); } + CLog.d("Filters"); + for (String filter : mIncludeFilters) { + CLog.d("Include: %s", filter); + } + for (String filter : mExcludeFilters) { + CLog.d("Exclude: %s", filter); + } + CLog.i("Num tests before filtering: %d", mTestInstances.size()); if (!mIncludeFilters.isEmpty() || !mExcludeFilters.isEmpty()) { filterTests(mTestInstances, mIncludeFilters, mExcludeFilters); } + CLog.i("Num tests after filtering: %d", mTestInstances.size()); + } + + /** + * {@inheritDoc} + */ + @Override + public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { + final Map emptyMap = Collections.emptyMap(); + final boolean isSupportedApi = !isOpenGlEsPackage() || isSupportedGles(); + + // If sharded, split() will load the tests. + if (mTestInstances == null) + loadTests(); + mRemainingTests = new LinkedList<>(mTestInstances.keySet()); listener.testRunStarted(getId(), mRemainingTests.size()); @@ -1956,4 +2000,50 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, IRemoteTest, public void addAllExcludeFilters(List filters) { mExcludeFilters.addAll(filters); } + + private static void copyOptions(DeqpTestRunner destination, DeqpTestRunner source) { + destination.mDeqpPackage = source.mDeqpPackage; + destination.mConfigName = source.mConfigName; + destination.mCaselistFile = source.mCaselistFile; + destination.mScreenRotation = source.mScreenRotation; + destination.mSurfaceType = source.mSurfaceType; + destination.mIncludeFilters = new ArrayList<>(source.mIncludeFilters); + destination.mExcludeFilters = new ArrayList<>(source.mExcludeFilters); + destination.mAbi = source.mAbi; + destination.mLogData = source.mLogData; + } + + /** + * {@inheritDoc} + */ + @Override + public Collection split() { + if (mTestInstances != null) { + throw new AssertionError("Re-splitting or splitting running instance?"); + } + // \todo [2015-11-23 kalle] If we split to batches at shard level, we could + // basically get rid of batching. Except that sharding is optional? + + // Assume that tests have not been yet loaded. + loadTests(); + + Collection runners = new ArrayList<>(); + // NOTE: Use linked hash map to keep the insertion order in iteration + Map> currentSet = new LinkedHashMap<>(); + Map> iterationSet = this.mTestInstances; + + // Go through tests, split + for (TestIdentifier test: iterationSet.keySet()) { + currentSet.put(test, iterationSet.get(test)); + if (currentSet.size() >= TESTCASE_BATCH_LIMIT) { + runners.add(new DeqpTestRunner(this, currentSet)); + // NOTE: Use linked hash map to keep the insertion order in iteration + currentSet = new LinkedHashMap<>(); + } + } + runners.add(new DeqpTestRunner(this, currentSet)); + + CLog.i("Split deqp tests into %d shards", runners.size()); + return runners; + } } diff --git a/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java b/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java index a84eef5..eb2e590 100644 --- a/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java +++ b/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java @@ -29,6 +29,7 @@ import com.android.tradefed.device.DeviceNotAvailableException; import com.android.tradefed.device.ITestDevice; import com.android.tradefed.result.ITestInvocationListener; import com.android.tradefed.testtype.IAbi; +import com.android.tradefed.testtype.IRemoteTest; import com.android.tradefed.util.IRunUtil; import com.android.tradefed.util.RunInterruptedException; @@ -143,6 +144,12 @@ public class DeqpTestRunnerTest extends TestCase { return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString()); } + private static CompatibilityBuildHelper getMockBuildHelper() { + IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class); + EasyMock.replay(mockIFolderBuildInfo); + return new BuildHelperMock(mockIFolderBuildInfo); + } + private static DeqpTestRunner buildGlesTestRunner(int majorVersion, int minorVersion, String testlist) throws ConfigurationException, FileNotFoundException { @@ -159,13 +166,8 @@ public class DeqpTestRunnerTest extends TestCase { setter.setOptionValue("deqp-surface-type", "window"); runner.setCaselistReader(new StringReader(testlist)); - runner.setAbi(ABI); - - IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class); - EasyMock.replay(mockIFolderBuildInfo); - CompatibilityBuildHelper mockHelper = new BuildHelperMock(mockIFolderBuildInfo); - runner.setBuildHelper(mockHelper); + runner.setBuildHelper(getMockBuildHelper()); return runner; } @@ -589,12 +591,7 @@ public class DeqpTestRunnerTest extends TestCase { EasyMock.verify(mockDevice, mockIDevice); } - private void testFiltering(List includes, - List excludes, - List fullTestList, - String expectedTrie, - List expectedTests) throws Exception{ - + static private String buildTestProcessOutput(List tests) { /* MultiLineReceiver expects "\r\n" line ending. */ final String outputHeader = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" @@ -617,7 +614,7 @@ public class DeqpTestRunnerTest extends TestCase { StringWriter output = new StringWriter(); output.write(outputHeader); - for (TestIdentifier test : expectedTests) { + for (TestIdentifier test : tests) { output.write("INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"); output.write("INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath="); output.write(test.getClassName()); @@ -633,6 +630,15 @@ public class DeqpTestRunnerTest extends TestCase { output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); } output.write(outputEnd); + return output.toString(); + } + + private void testFiltering(List includes, + List excludes, + List fullTestList, + String expectedTrie, + List expectedTests) throws Exception { + ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); ITestInvocationListener mockListener @@ -664,15 +670,8 @@ public class DeqpTestRunnerTest extends TestCase { { expectRenderConfigQuery(mockDevice, 3, 0); - String commandLine = String.format( - "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " - + "--deqp-screen-rotation=unspecified " - + "--deqp-surface-type=window " - + "--deqp-log-images=disable " - + "--deqp-watchdog=enable", - CASE_LIST_FILE_NAME); - - runInstrumentationLineAndAnswer(mockDevice, mockIDevice, expectedTrie, commandLine, output.toString()); + String testOut = buildTestProcessOutput(expectedTests); + runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut); for (int i = 0; i < expectedTests.size(); i++) { mockListener.testStarted(EasyMock.eq(expectedTests.get(i))); @@ -699,7 +698,6 @@ public class DeqpTestRunnerTest extends TestCase { EasyMock.verify(mockListener); EasyMock.verify(mockDevice, mockIDevice); - output.close(); } public void testRun_trivialIncludeFilter() throws Exception { @@ -826,10 +824,10 @@ public class DeqpTestRunnerTest extends TestCase { String expectedTrie = ""; ArrayList excludes = new ArrayList(); - excludes.add("*"); + excludes.add("*"); - testFiltering(null, excludes, allTests, expectedTrie, new ArrayList()); - } + testFiltering(null, excludes, allTests, expectedTrie, new ArrayList()); + } /** * Test running a unexecutable test. @@ -960,9 +958,9 @@ public class DeqpTestRunnerTest extends TestCase { tests.add(testId); DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests); - OptionSetter setter = new OptionSetter(deqpTest); - // Note: If the rotation is the default unspecified, features are not queried at all - setter.setOptionValue("deqp-screen-rotation", "90"); + OptionSetter setter = new OptionSetter(deqpTest); + // Note: If the rotation is the default unspecified, features are not queried at all + setter.setOptionValue("deqp-screen-rotation", "90"); deqpTest.setDevice(mockDevice); @@ -1094,8 +1092,8 @@ public class DeqpTestRunnerTest extends TestCase { tests.add(testId); DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests); - OptionSetter setter = new OptionSetter(deqpTest); - setter.setOptionValue("deqp-screen-rotation", rotation); + OptionSetter setter = new OptionSetter(deqpTest); + setter.setOptionValue("deqp-screen-rotation", rotation); deqpTest.setDevice(mockDevice); @@ -1371,9 +1369,9 @@ public class DeqpTestRunnerTest extends TestCase { Collection tests = new ArrayList(); tests.add(testId); - DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests); - OptionSetter setter = new OptionSetter(deqpTest); - setter.setOptionValue("deqp-gl-config-name", pixelFormat); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests); + OptionSetter setter = new OptionSetter(deqpTest); + setter.setOptionValue("deqp-gl-config-name", pixelFormat); deqpTest.setDevice(mockDevice); @@ -1780,6 +1778,116 @@ public class DeqpTestRunnerTest extends TestCase { EasyMock.verify(mockDevice, mockIDevice); } + private void runShardedTest(TestIdentifier[] testIds, + ArrayList> testsForShard) throws Exception { + Collection tests = new ArrayList(); + for (TestIdentifier id : testIds) tests.add(id); + + DeqpTestRunner runner = buildGlesTestRunner(3, 0, tests); + ArrayList shards = (ArrayList)runner.split(); + + for (int shardIndex = 0; shardIndex < shards.size(); shardIndex++) { + DeqpTestRunner shard = (DeqpTestRunner)shards.get(shardIndex); + shard.setBuildHelper(getMockBuildHelper()); + + ArrayList shardTests = testsForShard.get(shardIndex); + + ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); + ITestInvocationListener mockListener + = EasyMock.createStrictMock(ITestInvocationListener.class); + IDevice mockIDevice = EasyMock.createMock(IDevice.class); + int version = 3 << 16; + EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) + .andReturn(Integer.toString(version)).atLeastOnce(); + + EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") + .once(); + EasyMock.expect(mockDevice.installPackage(EasyMock.anyObject(), + EasyMock.eq(true), EasyMock.eq(AbiUtils.createAbiFlag(ABI.getName())))) + .andReturn(null).once(); + + mockListener.testRunStarted(getTestId(shard), shardTests.size()); + EasyMock.expectLastCall().once(); + + expectRenderConfigQuery(mockDevice, 3, 0); + + String testOut = buildTestProcessOutput(shardTests); + // NOTE: This assumes that there won't be multiple batches per shard! + runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut); + + for (int i = 0; i < shardTests.size(); i++) { + mockListener.testStarted(EasyMock.eq(shardTests.get(i))); + EasyMock.expectLastCall().once(); + + mockListener.testEnded(EasyMock.eq(shardTests.get(i)), + EasyMock.>notNull()); + + EasyMock.expectLastCall().once(); + } + + mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.>notNull()); + EasyMock.expectLastCall().once(); + + EasyMock.expect(mockDevice.uninstallPackage(EasyMock.eq(DEQP_ONDEVICE_PKG))).andReturn("") + .once(); + + EasyMock.replay(mockDevice, mockIDevice); + EasyMock.replay(mockListener); + + shard.setDevice(mockDevice); + shard.run(mockListener); + + EasyMock.verify(mockListener); + EasyMock.verify(mockDevice, mockIDevice); + } + } + + public void testSharding_smallTrivial() throws Exception { + final TestIdentifier[] testIds = { + new TestIdentifier("dEQP-GLES3.info", "vendor"), + new TestIdentifier("dEQP-GLES3.info", "renderer"), + new TestIdentifier("dEQP-GLES3.info", "version"), + new TestIdentifier("dEQP-GLES3.info", "shading_language_version"), + new TestIdentifier("dEQP-GLES3.info", "extensions"), + new TestIdentifier("dEQP-GLES3.info", "render_target") + }; + ArrayList> shardedTests = new ArrayList<>(); + ArrayList shardOne = new ArrayList<>(); + for (int i = 0; i < testIds.length; i++) { + shardOne.add(testIds[i]); + } + shardedTests.add(shardOne); + runShardedTest(testIds, shardedTests); + } + + public void testSharding_twoShards() throws Exception { + final int TEST_COUNT = 1237; + final int SHARD_SIZE = 1000; + + ArrayList testIds = new ArrayList<>(TEST_COUNT); + for (int i = 0; i < TEST_COUNT; i++) { + testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i))); + } + + ArrayList> shardedTests = new ArrayList<>(); + ArrayList shard = new ArrayList<>(); + for (int i = 0; i < testIds.size(); i++) { + if (i == SHARD_SIZE) { + shardedTests.add(shard); + shard = new ArrayList<>(); + } + shard.add(testIds.get(i)); + } + shardedTests.add(shard); + runShardedTest(testIds.toArray(new TestIdentifier[testIds.size()]), shardedTests); + } + + public void testSharding_empty() throws Exception { + DeqpTestRunner runner = buildGlesTestRunner(3, 0, new ArrayList()); + ArrayList shards = (ArrayList)runner.split(); + // \todo [2015-11-23 kalle] What should the result be? The runner or nothing? + } + /** * Test external interruption in testFailed(). */ @@ -1883,6 +1991,18 @@ public class DeqpTestRunnerTest extends TestCase { } private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, + final String output) throws Exception { + String cmd = String.format( + "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " + + "--deqp-screen-rotation=unspecified " + + "--deqp-surface-type=window " + + "--deqp-log-images=disable " + + "--deqp-watchdog=enable", + CASE_LIST_FILE_NAME); + runInstrumentationLineAndAnswer(mockDevice, mockIDevice, null, cmd, output); + } + + private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, final String testTrie, final String cmd, final String output) throws Exception { EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + CASE_LIST_FILE_NAME))) .andReturn("").once(); @@ -1890,8 +2010,13 @@ public class DeqpTestRunnerTest extends TestCase { EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + LOG_FILE_NAME))) .andReturn("").once(); - EasyMock.expect(mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME)) - .andReturn(true).once(); + if (testTrie == null) { + mockDevice.pushString((String)EasyMock.anyObject(), EasyMock.eq(CASE_LIST_FILE_NAME)); + } + else { + mockDevice.pushString(testTrie + "\n", CASE_LIST_FILE_NAME); + } + EasyMock.expectLastCall().andReturn(true).once(); String command = String.format( "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\" " -- 2.7.4