From: Kalle Raita Date: Tue, 21 Feb 2017 18:18:40 +0000 (+0000) Subject: Merge "Restructure copy image test iterations" am: 7ac38a6fa1 am: dae41c30d3 X-Git-Tag: upstream/0.1.0^2^2~228 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=d972c0b2e3b9eee66bc88c02d9354006af6d959e;hp=11a3c5d0a148b9c32abde94c92be56f1d0a73f29;p=platform%2Fupstream%2FVK-GL-CTS.git Merge "Restructure copy image test iterations" am: 7ac38a6fa1 am: dae41c30d3 am: 11a3c5d0a1 Change-Id: I6536bfa6ab9142f82bced640257a16edf9f12beb --- diff --git a/.gitignore b/.gitignore index dbd311c..a7a5af1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *~ *.pyc *.user +*.class .* !.gitignore !.editorconfig diff --git a/android/cts/Android.mk b/android/cts/Android.mk index d43624f..2eb34f7 100644 --- a/android/cts/Android.mk +++ b/android/cts/Android.mk @@ -26,13 +26,16 @@ LOCAL_COMPATIBILITY_SUITE := cts LOCAL_SDK_VERSION := 22 LOCAL_SRC_FILES := $(call all-java-files-under, runner/src) -LOCAL_JAVA_LIBRARIES := cts-tradefed compatibility-host-util tradefed-prebuilt +LOCAL_JAVA_LIBRARIES := cts-tradefed compatibility-host-util tradefed DEQP_CASELISTS:=$(sort $(patsubst master/%,%, \ $(shell cd $(LOCAL_PATH) ; \ find -L master -maxdepth 1 -name "*.txt") \ )) LOCAL_COMPATIBILITY_SUPPORT_FILES := $(foreach file, $(DEQP_CASELISTS), $(LOCAL_PATH)/master/$(file):$(file)) +LOCAL_COMPATIBILITY_SUPPORT_FILES += $(LOCAL_PATH)/nyc/vk-master.txt:nyc-vk-master.txt +LOCAL_COMPATIBILITY_SUPPORT_FILES += $(LOCAL_PATH)/nyc/gles31-master.txt:nyc-gles31-master.txt +LOCAL_COMPATIBILITY_SUPPORT_FILES += $(LOCAL_PATH)/nyc/egl-master.txt:nyc-egl-master.txt include $(BUILD_HOST_JAVA_LIBRARY) diff --git a/android/cts/runner/src/com/drawelements/deqp/runner/BatchRunConfiguration.java b/android/cts/runner/src/com/drawelements/deqp/runner/BatchRunConfiguration.java new file mode 100644 index 0000000..590fbbf --- /dev/null +++ b/android/cts/runner/src/com/drawelements/deqp/runner/BatchRunConfiguration.java @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.drawelements.deqp.runner; + +/** + * Test configuration of dEPQ test instance execution. + */ +public class BatchRunConfiguration { + public static final String ROTATION_UNSPECIFIED = "unspecified"; + public static final String ROTATION_PORTRAIT = "0"; + public static final String ROTATION_LANDSCAPE = "90"; + public static final String ROTATION_REVERSE_PORTRAIT = "180"; + public static final String ROTATION_REVERSE_LANDSCAPE = "270"; + + private final String mGlConfig; + private final String mRotation; + private final String mSurfaceType; + private final boolean mRequired; + + public BatchRunConfiguration(String glConfig, String rotation, String surfaceType, + boolean required) { + mGlConfig = glConfig; + mRotation = rotation; + mSurfaceType = surfaceType; + mRequired = required; + } + + /** + * Get string that uniquely identifies this config + */ + public String getId() { + return String.format("{glformat=%s,rotation=%s,surfacetype=%s,required=%b}", + mGlConfig, mRotation, mSurfaceType, mRequired); + } + + /** + * Get the GL config used in this configuration. + */ + public String getGlConfig() { + return mGlConfig; + } + + /** + * Get the screen rotation used in this configuration. + */ + public String getRotation() { + return mRotation; + } + + /** + * Get the surface type used in this configuration. + */ + public String getSurfaceType() { + return mSurfaceType; + } + + /** + * Is this configuration mandatory to support, if target API is supported? + */ + public boolean isRequired() { + return mRequired; + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } else if (!(other instanceof BatchRunConfiguration)) { + return false; + } else { + return getId().equals(((BatchRunConfiguration)other).getId()); + } + } + + @Override + public int hashCode() { + return getId().hashCode(); + } +} diff --git a/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java b/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java index 465d2da..eb1bb88 100644 --- a/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java +++ b/android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java @@ -16,7 +16,6 @@ package com.drawelements.deqp.runner; import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; -import com.android.compatibility.common.util.AbiUtils; import com.android.ddmlib.AdbCommandRejectedException; import com.android.ddmlib.IShellOutputReceiver; import com.android.ddmlib.MultiLineReceiver; @@ -24,7 +23,6 @@ import com.android.ddmlib.ShellCommandUnresponsiveException; import com.android.ddmlib.TimeoutException; import com.android.ddmlib.testrunner.TestIdentifier; import com.android.tradefed.build.IBuildInfo; -import com.android.tradefed.build.IFolderBuildInfo; import com.android.tradefed.config.Option; import com.android.tradefed.config.OptionClass; import com.android.tradefed.device.DeviceNotAvailableException; @@ -34,14 +32,16 @@ import com.android.tradefed.result.ByteArrayInputStreamSource; import com.android.tradefed.result.ITestInvocationListener; import com.android.tradefed.result.LogDataType; import com.android.tradefed.testtype.IAbi; -import com.android.tradefed.testtype.IBuildReceiver; import com.android.tradefed.testtype.IAbiReceiver; +import com.android.tradefed.testtype.IBuildReceiver; import com.android.tradefed.testtype.IDeviceTest; import com.android.tradefed.testtype.IRemoteTest; import com.android.tradefed.testtype.IRuntimeHintProvider; import com.android.tradefed.testtype.IShardableTest; +import com.android.tradefed.testtype.IStrictShardableTest; import com.android.tradefed.testtype.ITestCollector; import com.android.tradefed.testtype.ITestFilterReceiver; +import com.android.tradefed.util.AbiUtils; import com.android.tradefed.util.IRunUtil; import com.android.tradefed.util.RunInterruptedException; import com.android.tradefed.util.RunUtil; @@ -52,8 +52,8 @@ import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.io.Reader; -import java.lang.reflect.Method; import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -65,9 +65,9 @@ import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.regex.Pattern; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; /** * Test runner for dEQP tests @@ -77,7 +77,7 @@ import java.util.concurrent.TimeUnit; @OptionClass(alias="deqp-test-runner") public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, ITestFilterReceiver, IAbiReceiver, IShardableTest, ITestCollector, - IRuntimeHintProvider { + IRuntimeHintProvider, IStrictShardableTest { private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk"; private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp"; private static final String INCOMPLETE_LOG_MESSAGE = "Crash: Incomplete test log"; @@ -122,9 +122,15 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, @Option(name = "include-filter", description="Test include filter. '*' is zero or more letters. '.' has no special meaning.") private List mIncludeFilters = new ArrayList<>(); + @Option(name = "include-filter-file", + description="Load list of includes from the files given.") + private List mIncludeFilterFiles = new ArrayList<>(); @Option(name = "exclude-filter", description="Test exclude filter. '*' is zero or more letters. '.' has no special meaning.") private List mExcludeFilters = new ArrayList<>(); + @Option(name = "exclude-filter-file", + description="Load list of excludes from the files given.") + private List mExcludeFilterFiles = new ArrayList<>(); @Option(name = "collect-tests-only", description = "Only invoke the instrumentation to collect list of applicable test " + "cases. All test run callbacks will be triggered, but test execution will " @@ -245,82 +251,6 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, } /** - * Test configuration of dEPQ test instance execution. - * Exposed for unit testing - */ - public static final class BatchRunConfiguration { - public static final String ROTATION_UNSPECIFIED = "unspecified"; - public static final String ROTATION_PORTRAIT = "0"; - public static final String ROTATION_LANDSCAPE = "90"; - public static final String ROTATION_REVERSE_PORTRAIT = "180"; - public static final String ROTATION_REVERSE_LANDSCAPE = "270"; - - private final String mGlConfig; - private final String mRotation; - private final String mSurfaceType; - private final boolean mRequired; - - public BatchRunConfiguration(String glConfig, String rotation, String surfaceType, boolean required) { - mGlConfig = glConfig; - mRotation = rotation; - mSurfaceType = surfaceType; - mRequired = required; - } - - /** - * Get string that uniquely identifies this config - */ - public String getId() { - return String.format("{glformat=%s,rotation=%s,surfacetype=%s,required=%b}", - mGlConfig, mRotation, mSurfaceType, mRequired); - } - - /** - * Get the GL config used in this configuration. - */ - public String getGlConfig() { - return mGlConfig; - } - - /** - * Get the screen rotation used in this configuration. - */ - public String getRotation() { - return mRotation; - } - - /** - * Get the surface type used in this configuration. - */ - public String getSurfaceType() { - return mSurfaceType; - } - - /** - * Is this configuration mandatory to support, if target API is supported? - */ - public boolean isRequired() { - return mRequired; - } - - @Override - public boolean equals(Object other) { - if (other == null) { - return false; - } else if (!(other instanceof BatchRunConfiguration)) { - return false; - } else { - return getId().equals(((BatchRunConfiguration)other).getId()); - } - } - - @Override - public int hashCode() { - return getId().hashCode(); - } - } - - /** * dEQP test instance listerer and invocation result forwarded */ private class TestInstanceResultListener { @@ -819,11 +749,9 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, } private static class SleepProvider implements ISleepProvider { + @Override public void sleep(int milliseconds) { - try { - Thread.sleep(milliseconds); - } catch (InterruptedException ex) { - } + RunUtil.getDefault().sleep(milliseconds); } } @@ -858,11 +786,10 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, /** * Tries to recover device after abnormal execution termination or link failure. * - * @param progressedSinceLastCall true if test execution has progressed since last call * @throws DeviceNotAvailableException if recovery did not succeed */ public void recoverComLinkKilled() throws DeviceNotAvailableException; - }; + } /** * State machine for execution failure recovery. @@ -878,7 +805,7 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, RECOVER, // recover by calling recover() REBOOT, // recover by rebooting FAIL, // cannot recover - }; + } private MachineState mState = MachineState.WAIT; private ITestDevice mDevice; @@ -890,6 +817,7 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, /** * {@inheritDoc} */ + @Override public void setSleepProvider(ISleepProvider sleepProvider) { mSleepProvider = sleepProvider; } @@ -946,7 +874,8 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, case FAIL: // Third failure in a row, just fail CLog.w("Cannot recover ADB connection"); - throw new DeviceNotAvailableException("failed to connect after reboot"); + throw new DeviceNotAvailableException("failed to connect after reboot", + mDevice.getSerialNumber()); } } @@ -1009,7 +938,8 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, case FAIL: // Fourth failure in a row, just fail CLog.w("Cannot recover ADB connection"); - throw new DeviceNotAvailableException("link killed after reboot"); + throw new DeviceNotAvailableException("link killed after reboot", + mDevice.getSerialNumber()); } } @@ -1086,7 +1016,8 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, } private static Map> generateTestInstances( - Reader testlist, String configName, String screenRotation, String surfaceType, boolean required) throws FileNotFoundException { + Reader testlist, String configName, String screenRotation, String surfaceType, + boolean required) { // Note: This is specifically a LinkedHashMap to guarantee that tests are iterated // in the insertion order. final Map> instances = new LinkedHashMap<>(); @@ -1111,11 +1042,18 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, return instances; } - private Set getTestRunConfigs (TestIdentifier testId) { + private Set getTestRunConfigs(TestIdentifier testId) { return mTestInstances.get(testId); } /** + * Get the test instance of the runner. Exposed for testing. + */ + Map> getTestInstance() { + return mTestInstances; + } + + /** * Converts dEQP testcase path to TestIdentifier. */ private static TestIdentifier pathToIdentifier(String testPath) { @@ -1883,7 +1821,7 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, /** * Get GL major version (based on package name) */ - private int getGlesMajorVersion() throws DeviceNotAvailableException { + private int getGlesMajorVersion() { if ("dEQP-GLES2".equals(mDeqpPackage)) { return 2; } else if ("dEQP-GLES3".equals(mDeqpPackage)) { @@ -1898,7 +1836,7 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, /** * Get GL minor version (based on package name) */ - private int getGlesMinorVersion() throws DeviceNotAvailableException { + private int getGlesMinorVersion() { if ("dEQP-GLES2".equals(mDeqpPackage)) { return 0; } else if ("dEQP-GLES3".equals(mDeqpPackage)) { @@ -1924,7 +1862,17 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, Set nonPatternFilters = new HashSet(); for (String filter : filters) { if (!filter.contains("*")) { - nonPatternFilters.add(filter); + // Deqp usesly only dots for separating between parts of the names + // Convert last dot to hash if needed. + if (!filter.contains("#")) { + int lastSeparator = filter.lastIndexOf('.'); + String filterWithHash = filter.substring(0, lastSeparator) + "#" + + filter.substring(lastSeparator + 1, filter.length()); + nonPatternFilters.add(filterWithHash); + } + else { + nonPatternFilters.add(filter); + } } } return nonPatternFilters; @@ -1955,7 +1903,7 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, List includePatterns = getPatternFilters(includeFilters); List excludePatterns = getPatternFilters(excludeFilters); - List testList = new ArrayList(tests.keySet()); + List testList = new ArrayList<>(tests.keySet()); for (TestIdentifier test : testList) { if (excludeStrings.contains(test.toString())) { tests.remove(test); // remove test if explicitly excluded @@ -1973,6 +1921,48 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, } /** + * Read a list of filters from a file. + * + * Note: Filters can be numerous so we prefer, for performance + * reasons, to add directly to the target list instead of using + * intermediate return value. + */ + static private void readFilterFile(List filterList, File file) throws FileNotFoundException { + if (!file.canRead()) { + CLog.e("Failed to read filter file '%s'", file.getPath()); + throw new FileNotFoundException(); + } + try (Reader plainReader = new FileReader(file); + BufferedReader reader = new BufferedReader(plainReader)) { + String filter = ""; + while ((filter = reader.readLine()) != null) { + // TOOD: Sanity check filter + filterList.add(filter); + } + // Rely on try block to autoclose + } + catch (IOException e) + { + throw new RuntimeException("Failed to read filter list file '" + file.getPath() + "': " + + e.getMessage()); + } + } + + /** + * Prints filters into debug log stream, limiting to 20 entries. + */ + static private void printFilters(List filters) { + int numPrinted = 0; + for (String filter : filters) { + CLog.d(" %s", filter); + if (++numPrinted == 20) { + CLog.d(" ... AND %d others", filters.size() - numPrinted); + break; + } + } + } + + /** * Loads tests into mTestInstances based on the options. Assumes * that no tests have been loaded for this instance before. */ @@ -1998,14 +1988,29 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, catch (IOException e) { CLog.w("Failed to close test list reader."); } - CLog.d("Filters"); - for (String filter : mIncludeFilters) { - CLog.d("Include: %s", filter); + + try + { + for (String filterFile : mIncludeFilterFiles) { + CLog.d("Read include filter file '%s'", filterFile); + File file = new File(mBuildHelper.getTestsDir(), filterFile); + readFilterFile(mIncludeFilters, file); + } + for (String filterFile : mExcludeFilterFiles) { + CLog.d("Read exclude filter file '%s'", filterFile); + File file = new File(mBuildHelper.getTestsDir(), filterFile); + readFilterFile(mExcludeFilters, file); + } } - for (String filter : mExcludeFilters) { - CLog.d("Exclude: %s", filter); + catch (FileNotFoundException e) { + throw new RuntimeException("Cannot read deqp filter list file:" + e.getMessage()); } + CLog.d("Include filters:"); + printFilters(mIncludeFilters); + CLog.d("Exclude filters:"); + printFilters(mExcludeFilters); + long originalTestCount = mTestInstances.size(); CLog.i("Num tests before filtering: %d", originalTestCount); if ((!mIncludeFilters.isEmpty() || !mExcludeFilters.isEmpty()) && originalTestCount > 0) { @@ -2035,6 +2040,10 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, listener.testRunStarted(getId(), mRemainingTests.size()); try { + if (mRemainingTests.isEmpty()) { + CLog.d("No tests to run."); + return; + } final boolean isSupportedApi = (isOpenGlEsPackage() && isSupportedGles()) || (isVulkanPackage() && isSupportedVulkan()) || (!isOpenGlEsPackage() && !isVulkanPackage()); @@ -2060,9 +2069,9 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, // test cases in "NotExecuted" state CLog.e("Capability query failed - leaving tests unexecuted."); uninstallTestApk(); + } finally { + listener.testRunEnded(System.currentTimeMillis() - startTime, emptyMap); } - - listener.testRunEnded(System.currentTimeMillis() - startTime, emptyMap); } /** @@ -2113,13 +2122,29 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, destination.mSurfaceType = source.mSurfaceType; destination.mConfigRequired = source.mConfigRequired; destination.mIncludeFilters = new ArrayList<>(source.mIncludeFilters); + destination.mIncludeFilterFiles = new ArrayList<>(source.mIncludeFilterFiles); destination.mExcludeFilters = new ArrayList<>(source.mExcludeFilters); + destination.mExcludeFilterFiles = new ArrayList<>(source.mExcludeFilterFiles); destination.mAbi = source.mAbi; destination.mLogData = source.mLogData; destination.mCollectTestsOnly = source.mCollectTestsOnly; } /** + * Helper to update the RuntimeHint of the tests after being sharded. + */ + private void updateRuntimeHint(long originalSize, Collection runners) { + if (originalSize > 0) { + long fullRuntimeMs = getRuntimeHint(); + for (IRemoteTest remote: runners) { + DeqpTestRunner runner = (DeqpTestRunner)remote; + long shardRuntime = (fullRuntimeMs * runner.mTestInstances.size()) / originalSize; + runner.mRuntimeHint = shardRuntime; + } + } + } + + /** * {@inheritDoc} */ @Override @@ -2138,6 +2163,11 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, Map> currentSet = new LinkedHashMap<>(); Map> iterationSet = this.mTestInstances; + if (iterationSet.keySet().isEmpty()) { + CLog.i("Cannot split deqp tests, no tests to run"); + return null; + } + // Go through tests, split for (TestIdentifier test: iterationSet.keySet()) { currentSet.put(test, iterationSet.get(test)); @@ -2145,23 +2175,59 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, runners.add(new DeqpTestRunner(this, currentSet)); // NOTE: Use linked hash map to keep the insertion order in iteration currentSet = new LinkedHashMap<>(); - } + } } runners.add(new DeqpTestRunner(this, currentSet)); // Compute new runtime hints - long originalSize = iterationSet.size(); - if (originalSize > 0) { - long fullRuntimeMs = getRuntimeHint(); - for (IRemoteTest remote: runners) { - DeqpTestRunner runner = (DeqpTestRunner)remote; - long shardRuntime = (fullRuntimeMs * runner.mTestInstances.size()) / originalSize; - runner.mRuntimeHint = shardRuntime; + updateRuntimeHint(iterationSet.size(), runners); + CLog.i("Split deqp tests into %d shards", runners.size()); + return runners; + } + + /** + * This sharding should be deterministic for the same input and independent. + * Through this API, each shard could be executed on different machine. + */ + @Override + public IRemoteTest getTestShard(int shardCount, int shardIndex) { + // TODO: refactor getTestshard and split to share some logic. + if (mTestInstances == null) { + loadTests(); + } + + List runners = new ArrayList<>(); + // NOTE: Use linked hash map to keep the insertion order in iteration + Map> currentSet = new LinkedHashMap<>(); + Map> iterationSet = this.mTestInstances; + + int batchLimit = iterationSet.keySet().size() / shardCount; + int i = 1; + // Go through tests, split + for (TestIdentifier test: iterationSet.keySet()) { + currentSet.put(test, iterationSet.get(test)); + if (currentSet.size() >= batchLimit && i < shardCount) { + runners.add(new DeqpTestRunner(this, currentSet)); + i++; + // NOTE: Use linked hash map to keep the insertion order in iteration + currentSet = new LinkedHashMap<>(); } } + runners.add(new DeqpTestRunner(this, currentSet)); - CLog.i("Split deqp tests into %d shards", runners.size()); - return runners; + // Compute new runtime hints + updateRuntimeHint(iterationSet.size(), runners); + + // If too many shards were requested, we complete with placeholder. + if (runners.size() < shardCount) { + for (int j = runners.size(); j < shardCount; j++) { + runners.add(new DeqpTestRunner(this, + new LinkedHashMap>())); + } + } + + CLog.i("Split deqp tests into %d shards, return shard: %s", runners.size(), shardIndex); + return runners.get(shardIndex); } /** diff --git a/android/cts/runner/tests/Android.mk b/android/cts/runner/tests/Android.mk index 7fc1056..81ba834 100644 --- a/android/cts/runner/tests/Android.mk +++ b/android/cts/runner/tests/Android.mk @@ -21,7 +21,7 @@ LOCAL_SRC_FILES := $(call all-java-files-under, src) LOCAL_MODULE := CtsDeqpRunnerTests LOCAL_MODULE_TAGS := optional -LOCAL_JAVA_LIBRARIES := cts-tradefed compatibility-host-util tradefed-prebuilt CtsDeqpTestCases +LOCAL_JAVA_LIBRARIES := cts-tradefed compatibility-host-util tradefed CtsDeqpTestCases LOCAL_STATIC_JAVA_LIBRARIES := easymock include $(BUILD_HOST_JAVA_LIBRARY) diff --git a/android/cts/runner/tests/run_tests.sh b/android/cts/runner/tests/run_tests.sh index 70a9ce7..4832995 100755 --- a/android/cts/runner/tests/run_tests.sh +++ b/android/cts/runner/tests/run_tests.sh @@ -47,7 +47,7 @@ JARS=" hosttestlib\ CtsDeqpTestCases\ CtsDeqpRunnerTests\ - tradefed-prebuilt" + tradefed" JAR_PATH= for JAR in $JARS; do checkFile ${JAR_DIR}/${JAR}.jar diff --git a/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java b/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java index 5c2ac0b..ee2810c 100644 --- a/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java +++ b/android/cts/runner/tests/src/com/drawelements/deqp/runner/DeqpTestRunnerTest.java @@ -16,10 +16,8 @@ package com.drawelements.deqp.runner; import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; -import com.android.compatibility.common.util.AbiUtils; import com.android.ddmlib.IDevice; import com.android.ddmlib.IShellOutputReceiver; -import com.android.ddmlib.ShellCommandUnresponsiveException; import com.android.ddmlib.testrunner.TestIdentifier; import com.android.tradefed.build.IFolderBuildInfo; import com.android.tradefed.config.ConfigurationException; @@ -31,6 +29,8 @@ import com.android.tradefed.testtype.Abi; import com.android.tradefed.testtype.IAbi; import com.android.tradefed.testtype.IRemoteTest; import com.android.tradefed.testtype.IRuntimeHintProvider; +import com.android.tradefed.util.AbiUtils; +import com.android.tradefed.util.FileUtil; import com.android.tradefed.util.IRunUtil; import com.android.tradefed.util.RunInterruptedException; @@ -42,6 +42,9 @@ import org.easymock.IMocksControl; import java.io.File; import java.io.FileNotFoundException; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; import java.io.StringReader; import java.io.StringWriter; import java.util.ArrayList; @@ -81,50 +84,17 @@ public class DeqpTestRunnerTest extends TestCase { DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window"); } - private static class StubRecovery implements DeqpTestRunner.IRecovery { - /** - * {@inheritDoc} - */ - @Override - public void setSleepProvider(DeqpTestRunner.ISleepProvider sleepProvider) { - } - - /** - * {@inheritDoc} - */ - @Override - public void setDevice(ITestDevice device) { - } - - /** - * {@inheritDoc} - */ - @Override - public void onExecutionProgressed() { - } - - /** - * {@inheritDoc} - */ - @Override - public void recoverConnectionRefused() throws DeviceNotAvailableException { - } - - /** - * {@inheritDoc} - */ - @Override - public void recoverComLinkKilled() throws DeviceNotAvailableException { - } - }; + private File mTestsDir = null; public static class BuildHelperMock extends CompatibilityBuildHelper { - public BuildHelperMock(IFolderBuildInfo buildInfo) { + private File mTestsDir = null; + public BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir) { super(buildInfo); + mTestsDir = testsDir; } @Override public File getTestsDir() throws FileNotFoundException { - return new File("logs"); + return mTestsDir; } } @@ -135,27 +105,39 @@ public class DeqpTestRunnerTest extends TestCase { @Override protected void setUp() throws Exception { super.setUp(); + mTestsDir = FileUtil.createTempDir("deqp-test-cases"); + } + + /** + * {@inheritDoc} + */ + @Override + protected void tearDown() throws Exception { + FileUtil.recursiveDelete(mTestsDir); + super.tearDown(); } private static DeqpTestRunner buildGlesTestRunner(int majorVersion, int minorVersion, - Collection tests) throws ConfigurationException, FileNotFoundException { + Collection tests, + File testsDir) throws ConfigurationException, FileNotFoundException { StringWriter testlist = new StringWriter(); for (TestIdentifier test : tests) { testlist.write(test.getClassName() + "." + test.getTestName() + "\n"); } - return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString()); + return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString(), testsDir); } - private static CompatibilityBuildHelper getMockBuildHelper() { + private static CompatibilityBuildHelper getMockBuildHelper(File testsDir) { IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class); EasyMock.replay(mockIFolderBuildInfo); - return new BuildHelperMock(mockIFolderBuildInfo); + return new BuildHelperMock(mockIFolderBuildInfo, testsDir); } private static DeqpTestRunner buildGlesTestRunner(int majorVersion, int minorVersion, - String testlist) throws ConfigurationException, FileNotFoundException { + String testlist, + File testsDir) throws ConfigurationException, FileNotFoundException { DeqpTestRunner runner = new DeqpTestRunner(); OptionSetter setter = new OptionSetter(runner); @@ -170,7 +152,7 @@ public class DeqpTestRunnerTest extends TestCase { runner.setCaselistReader(new StringReader(testlist)); runner.setAbi(ABI); - runner.setBuildHelper(getMockBuildHelper()); + runner.setBuildHelper(getMockBuildHelper(testsDir)); return runner; } @@ -232,7 +214,7 @@ public class DeqpTestRunnerTest extends TestCase { Collection tests = new ArrayList(); tests.add(testId); - DeqpTestRunner deqpTest = buildGlesTestRunner(requiredMajorVersion, requiredMinorVersion, tests); + DeqpTestRunner deqpTest = buildGlesTestRunner(requiredMajorVersion, requiredMinorVersion, tests, mTestsDir); int version = (majorVersion << 16) | minorVersion; EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) @@ -374,7 +356,7 @@ public class DeqpTestRunnerTest extends TestCase { Collection tests = new ArrayList(); tests.add(testId); - DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); int version = 3 << 16; EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) @@ -541,7 +523,7 @@ public class DeqpTestRunnerTest extends TestCase { tests.add(id); } - DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); int version = 3 << 16; EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) @@ -636,30 +618,15 @@ public class DeqpTestRunnerTest extends TestCase { return output.toString(); } - private void testFiltering(Set includes, - Set excludes, - List fullTestList, + private void testFiltering(DeqpTestRunner deqpTest, String expectedTrie, List expectedTests) throws Exception { - - boolean thereAreTests = !expectedTests.isEmpty(); - ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); - ITestInvocationListener mockListener - = EasyMock.createStrictMock(ITestInvocationListener.class); - IDevice mockIDevice = EasyMock.createMock(IDevice.class); - - DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, fullTestList); - if (includes != null) { - deqpTest.addAllIncludeFilters(includes); - } - if (excludes != null) { - deqpTest.addAllExcludeFilters(excludes); - } - int version = 3 << 16; + ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) .andReturn(Integer.toString(version)).atLeastOnce(); + boolean thereAreTests = !expectedTests.isEmpty(); if (thereAreTests) { // only expect to install/uninstall packages if there are any tests @@ -670,10 +637,12 @@ public class DeqpTestRunnerTest extends TestCase { .andReturn(null).once(); } - + ITestInvocationListener mockListener + = EasyMock.createStrictMock(ITestInvocationListener.class); mockListener.testRunStarted(getTestId(deqpTest), expectedTests.size()); EasyMock.expectLastCall().once(); + IDevice mockIDevice = EasyMock.createMock(IDevice.class); if (thereAreTests) { expectRenderConfigQuery(mockDevice, 3, 0); @@ -734,9 +703,9 @@ public class DeqpTestRunnerTest extends TestCase { String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; - Set includes = new HashSet(); - includes.add("dEQP-GLES3.pick_me#*"); - testFiltering(includes, null, allTests, expectedTrie, activeTests); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); + deqpTest.addIncludeFilter("dEQP-GLES3.pick_me#*"); + testFiltering(deqpTest, expectedTrie, activeTests); } public void testRun_trivialExcludeFilter() throws Exception { @@ -761,9 +730,9 @@ public class DeqpTestRunnerTest extends TestCase { String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; - Set excludes = new HashSet(); - excludes.add("dEQP-GLES3.missing#*"); - testFiltering(null, excludes, allTests, expectedTrie, activeTests); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); + deqpTest.addExcludeFilter("dEQP-GLES3.missing#*"); + testFiltering(deqpTest, expectedTrie, activeTests); } public void testRun_includeAndExcludeFilter() throws Exception { @@ -786,13 +755,17 @@ public class DeqpTestRunnerTest extends TestCase { String expectedTrie = "{dEQP-GLES3{group2{yes}}}"; - Set includes = new HashSet(); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); + + Set includes = new HashSet<>(); includes.add("dEQP-GLES3.group2#*"); + deqpTest.addAllIncludeFilters(includes); - Set excludes = new HashSet(); + Set excludes = new HashSet<>(); excludes.add("*foo"); excludes.add("*thoushallnotpass"); - testFiltering(includes, excludes, allTests, expectedTrie, activeTests); + deqpTest.addAllExcludeFilters(excludes); + testFiltering(deqpTest, expectedTrie, activeTests); } public void testRun_includeAll() throws Exception { @@ -812,10 +785,9 @@ public class DeqpTestRunnerTest extends TestCase { String expectedTrie = "{dEQP-GLES3{group1{mememe,yeah,takeitall},group2{jeba,yes,granted}}}"; - Set includes = new HashSet(); - includes.add("*"); - - testFiltering(includes, null, allTests, expectedTrie, allTests); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); + deqpTest.addIncludeFilter("*"); + testFiltering(deqpTest, expectedTrie, allTests); } public void testRun_excludeAll() throws Exception { @@ -833,12 +805,18 @@ public class DeqpTestRunnerTest extends TestCase { allTests.add(id); } - String expectedTrie = ""; - - Set excludes = new HashSet(); - excludes.add("*"); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); + deqpTest.addExcludeFilter("*"); + ITestInvocationListener mockListener + = EasyMock.createStrictMock(ITestInvocationListener.class); + mockListener.testRunStarted(getTestId(deqpTest), 0); + EasyMock.expectLastCall().once(); + mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.>notNull()); + EasyMock.expectLastCall().once(); - testFiltering(null, excludes, allTests, expectedTrie, new ArrayList()); + EasyMock.replay(mockListener); + deqpTest.run(mockListener); + EasyMock.verify(mockListener); } /** @@ -887,7 +865,7 @@ public class DeqpTestRunnerTest extends TestCase { tests.add(id); } - DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); int version = 3 << 16; EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) @@ -969,7 +947,7 @@ public class DeqpTestRunnerTest extends TestCase { Collection tests = new ArrayList(); tests.add(testId); - DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); OptionSetter setter = new OptionSetter(deqpTest); // Note: If the rotation is the default unspecified, features are not queried at all setter.setOptionValue("deqp-screen-rotation", "90"); @@ -1021,7 +999,7 @@ public class DeqpTestRunnerTest extends TestCase { Collection tests = new ArrayList(); tests.add(testId); - DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); deqpTest.setDevice(mockDevice); @@ -1103,7 +1081,7 @@ public class DeqpTestRunnerTest extends TestCase { Collection tests = new ArrayList(); tests.add(testId); - DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); OptionSetter setter = new OptionSetter(deqpTest); setter.setOptionValue("deqp-screen-rotation", rotation); @@ -1113,19 +1091,19 @@ public class DeqpTestRunnerTest extends TestCase { EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) .andReturn(Integer.toString(version)).atLeastOnce(); - if (!rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED)) { + if (!rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED)) { EasyMock.expect(mockDevice.executeShellCommand("pm list features")) .andReturn(featureString); } final boolean isPortraitOrientation = - rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_PORTRAIT) || - rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT); + rotation.equals(BatchRunConfiguration.ROTATION_PORTRAIT) || + rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT); final boolean isLandscapeOrientation = - rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_LANDSCAPE) || - rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE); + rotation.equals(BatchRunConfiguration.ROTATION_LANDSCAPE) || + rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE); final boolean executable = - rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED) || + rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED) || (isPortraitOrientation && featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) || (isLandscapeOrientation && @@ -1345,34 +1323,6 @@ public class DeqpTestRunnerTest extends TestCase { public void testRun_unsupportedPixelFormat() throws Exception { final String pixelFormat = "rgba5658d16m4"; final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.pixelformat", "test"); - final String testPath = "dEQP-GLES3.pixelformat.test"; - final String testTrie = "{dEQP-GLES3{pixelformat{test}}}"; - final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" - + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" - + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" - + "INSTRUMENTATION_STATUS_CODE: 0\r\n" - + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" - + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" - + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" - + "INSTRUMENTATION_STATUS_CODE: 0\r\n" - + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" - + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" - + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" - + "INSTRUMENTATION_STATUS_CODE: 0\r\n" - + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" - + "INSTRUMENTATION_STATUS_CODE: 0\r\n" - + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" - + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" - + "INSTRUMENTATION_STATUS_CODE: 0\r\n" - + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" - + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" - + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" - + "INSTRUMENTATION_STATUS_CODE: 0\r\n" - + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" - + "INSTRUMENTATION_STATUS_CODE: 0\r\n" - + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" - + "INSTRUMENTATION_STATUS_CODE: 0\r\n" - + "INSTRUMENTATION_CODE: 0\r\n"; ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); ITestInvocationListener mockListener @@ -1381,7 +1331,7 @@ public class DeqpTestRunnerTest extends TestCase { Collection tests = new ArrayList(); tests.add(testId); - DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); OptionSetter setter = new OptionSetter(deqpTest); setter.setOptionValue("deqp-gl-config-name", pixelFormat); @@ -1435,7 +1385,7 @@ public class DeqpTestRunnerTest extends TestCase { PROGRESS, FAIL_CONNECTION_REFUSED, FAIL_LINK_KILLED, - }; + } private void runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events) throws DeviceNotAvailableException { @@ -1551,6 +1501,7 @@ public class DeqpTestRunnerTest extends TestCase { DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery(); IMocksControl orderedControl = EasyMock.createStrictControl(); RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class); + EasyMock.expect(mockDevice.getSerialNumber()).andStubReturn("SERIAL"); DeqpTestRunner.ISleepProvider mockSleepProvider = orderedControl.createMock(DeqpTestRunner.ISleepProvider.class); @@ -1748,7 +1699,7 @@ public class DeqpTestRunnerTest extends TestCase { IDevice mockIDevice = EasyMock.createMock(IDevice.class); IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); - DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); deqpTest.setDevice(mockDevice); deqpTest.setRunUtil(mockRunUtil); @@ -1775,6 +1726,8 @@ public class DeqpTestRunnerTest extends TestCase { mockListener.testRunStarted(getTestId(deqpTest), 1); EasyMock.expectLastCall().once(); + mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.anyObject()); + EasyMock.expectLastCall().once(); EasyMock.replay(mockDevice, mockIDevice); EasyMock.replay(mockListener); @@ -1795,12 +1748,12 @@ public class DeqpTestRunnerTest extends TestCase { Collection tests = new ArrayList(); for (TestIdentifier id : testIds) tests.add(id); - DeqpTestRunner runner = buildGlesTestRunner(3, 0, tests); + DeqpTestRunner runner = buildGlesTestRunner(3, 0, tests, mTestsDir); ArrayList shards = (ArrayList)runner.split(); for (int shardIndex = 0; shardIndex < shards.size(); shardIndex++) { DeqpTestRunner shard = (DeqpTestRunner)shards.get(shardIndex); - shard.setBuildHelper(getMockBuildHelper()); + shard.setBuildHelper(getMockBuildHelper(mTestsDir)); ArrayList shardTests = testsForShard.get(shardIndex); @@ -1895,9 +1848,10 @@ public class DeqpTestRunnerTest extends TestCase { } public void testSharding_empty() throws Exception { - DeqpTestRunner runner = buildGlesTestRunner(3, 0, new ArrayList()); + DeqpTestRunner runner = buildGlesTestRunner(3, 0, new ArrayList(), mTestsDir); ArrayList shards = (ArrayList)runner.split(); - // \todo [2015-11-23 kalle] What should the result be? The runner or nothing? + // Returns null when cannot be sharded. + assertNull(shards); } /** @@ -1943,7 +1897,7 @@ public class DeqpTestRunnerTest extends TestCase { IDevice mockIDevice = EasyMock.createMock(IDevice.class); IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); - DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); deqpTest.setDevice(mockDevice); deqpTest.setRunUtil(mockRunUtil); @@ -1988,6 +1942,8 @@ public class DeqpTestRunnerTest extends TestCase { mockListener.testFailed(EasyMock.eq(testId), EasyMock.notNull()); EasyMock.expectLastCall().andThrow(new RunInterruptedException()); + mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.anyObject()); + EasyMock.expectLastCall().once(); EasyMock.replay(mockDevice, mockIDevice); EasyMock.replay(mockListener); EasyMock.replay(mockRunUtil); @@ -2108,7 +2064,7 @@ public class DeqpTestRunnerTest extends TestCase { tests.add(id); } - DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); OptionSetter setter = new OptionSetter(deqpTest); final long runtimeMs = 123456; setter.setOptionValue("runtime-hint", String.valueOf(runtimeMs)); @@ -2177,7 +2133,7 @@ public class DeqpTestRunnerTest extends TestCase { testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i))); } - DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir); OptionSetter setter = new OptionSetter(deqpTest); final long fullRuntimeMs = testIds.size()*100; setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs)); @@ -2191,6 +2147,100 @@ public class DeqpTestRunnerTest extends TestCase { ((IRuntimeHintProvider)shards.get(1)).getRuntimeHint()); } + /** + * Test that strict shardable is able to split deterministically the set of tests. + */ + public void testGetTestShard() throws Exception { + final int TEST_COUNT = 2237; + final int SHARD_COUNT = 4; + + ArrayList testIds = new ArrayList<>(TEST_COUNT); + for (int i = 0; i < TEST_COUNT; i++) { + testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i))); + } + + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir); + OptionSetter setter = new OptionSetter(deqpTest); + final long fullRuntimeMs = testIds.size()*100; + setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs)); + + DeqpTestRunner shard1 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 0); + assertEquals(559, shard1.getTestInstance().size()); + int j = 0; + // Ensure numbers, and that order is stable + for (TestIdentifier t : shard1.getTestInstance().keySet()) { + assertEquals(String.format("dEQP-GLES3.funny.group#%s", j), + String.format("%s#%s", t.getClassName(), t.getTestName())); + j++; + } + DeqpTestRunner shard2 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 1); + assertEquals(559, shard2.getTestInstance().size()); + for (TestIdentifier t : shard2.getTestInstance().keySet()) { + assertEquals(String.format("dEQP-GLES3.funny.group#%s", j), + String.format("%s#%s", t.getClassName(), t.getTestName())); + j++; + } + DeqpTestRunner shard3 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 2); + assertEquals(559, shard3.getTestInstance().size()); + for (TestIdentifier t : shard3.getTestInstance().keySet()) { + assertEquals(String.format("dEQP-GLES3.funny.group#%s", j), + String.format("%s#%s", t.getClassName(), t.getTestName())); + j++; + } + DeqpTestRunner shard4 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 3); + assertEquals(560, shard4.getTestInstance().size()); + for (TestIdentifier t : shard4.getTestInstance().keySet()) { + assertEquals(String.format("dEQP-GLES3.funny.group#%s", j), + String.format("%s#%s", t.getClassName(), t.getTestName())); + j++; + } + assertEquals(TEST_COUNT, j); + } + + /** + * Test that strict shardable is creating an empty shard of the runner when too many shards + * are requested. + */ + public void testGetTestShard_tooManyShardRequested() throws Exception { + final int TEST_COUNT = 2; + final int SHARD_COUNT = 3; + + ArrayList testIds = new ArrayList<>(TEST_COUNT); + for (int i = 0; i < TEST_COUNT; i++) { + testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i))); + } + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir); + OptionSetter setter = new OptionSetter(deqpTest); + final long fullRuntimeMs = testIds.size()*100; + setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs)); + DeqpTestRunner shard1 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 0); + assertEquals(1, shard1.getTestInstance().size()); + int j = 0; + // Ensure numbers, and that order is stable + for (TestIdentifier t : shard1.getTestInstance().keySet()) { + assertEquals(String.format("dEQP-GLES3.funny.group#%s", j), + String.format("%s#%s", t.getClassName(), t.getTestName())); + j++; + } + DeqpTestRunner shard2 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 1); + assertEquals(1, shard2.getTestInstance().size()); + for (TestIdentifier t : shard2.getTestInstance().keySet()) { + assertEquals(String.format("dEQP-GLES3.funny.group#%s", j), + String.format("%s#%s", t.getClassName(), t.getTestName())); + j++; + } + DeqpTestRunner shard3 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 2); + assertTrue(shard3.getTestInstance().isEmpty()); + assertEquals(TEST_COUNT, j); + ITestInvocationListener mockListener + = EasyMock.createStrictMock(ITestInvocationListener.class); + mockListener.testRunStarted(EasyMock.anyObject(), EasyMock.eq(0)); + mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.anyObject()); + EasyMock.replay(mockListener); + shard3.run(mockListener); + EasyMock.verify(mockListener); + } + public void testRuntimeHint_optionNotSet() throws Exception { final TestIdentifier[] testIds = { new TestIdentifier("dEQP-GLES3.info", "vendor"), @@ -2206,7 +2256,7 @@ public class DeqpTestRunnerTest extends TestCase { tests.add(id); } - DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests); + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); long runtime = deqpTest.getRuntimeHint(); assertTrue("Runtime for tests must be positive", runtime > 0); @@ -2266,4 +2316,166 @@ public class DeqpTestRunnerTest extends TestCase { } }); } + + static private void writeStringsToFile(File target, Set strings) throws IOException { + try (PrintWriter out = new PrintWriter(new FileWriter(target))) { + out.print(String.join(System.lineSeparator(), strings)); + out.println(); + } + } + + private void addFilterFileForOption(DeqpTestRunner test, Set filters, String option) + throws IOException, ConfigurationException { + String filterFile = option + ".txt"; + writeStringsToFile(new File(mTestsDir, filterFile), filters); + OptionSetter setter = new OptionSetter(test); + setter.setOptionValue(option, filterFile); + } + + public void testIncludeFilterFile() throws Exception { + final TestIdentifier[] testIds = { + new TestIdentifier("dEQP-GLES3.missing", "no"), + new TestIdentifier("dEQP-GLES3.missing", "nope"), + new TestIdentifier("dEQP-GLES3.missing", "donotwant"), + new TestIdentifier("dEQP-GLES3.pick_me", "yes"), + new TestIdentifier("dEQP-GLES3.pick_me", "ok"), + new TestIdentifier("dEQP-GLES3.pick_me", "accepted"), + }; + + List allTests = new ArrayList(); + for (TestIdentifier id : testIds) { + allTests.add(id); + } + + List activeTests = new ArrayList(); + activeTests.add(testIds[3]); + activeTests.add(testIds[4]); + activeTests.add(testIds[5]); + + String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; + + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); + Set includes = new HashSet<>(); + includes.add("dEQP-GLES3.pick_me#*"); + addFilterFileForOption(deqpTest, includes, "include-filter-file"); + testFiltering(deqpTest, expectedTrie, activeTests); + } + + public void testMissingIncludeFilterFile() throws Exception { + final TestIdentifier[] testIds = { + new TestIdentifier("dEQP-GLES3.pick_me", "yes"), + new TestIdentifier("dEQP-GLES3.pick_me", "ok"), + new TestIdentifier("dEQP-GLES3.pick_me", "accepted"), + }; + + List allTests = new ArrayList(); + for (TestIdentifier id : testIds) { + allTests.add(id); + } + + String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; + + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); + OptionSetter setter = new OptionSetter(deqpTest); + setter.setOptionValue("include-filter-file", "not-a-file.txt"); + try { + testFiltering(deqpTest, expectedTrie, allTests); + fail("Test execution should have aborted with exception."); + } catch (RuntimeException e) { + } + } + + public void testExcludeFilterFile() throws Exception { + final TestIdentifier[] testIds = { + new TestIdentifier("dEQP-GLES3.missing", "no"), + new TestIdentifier("dEQP-GLES3.missing", "nope"), + new TestIdentifier("dEQP-GLES3.missing", "donotwant"), + new TestIdentifier("dEQP-GLES3.pick_me", "yes"), + new TestIdentifier("dEQP-GLES3.pick_me", "ok"), + new TestIdentifier("dEQP-GLES3.pick_me", "accepted"), + }; + + List allTests = new ArrayList(); + for (TestIdentifier id : testIds) { + allTests.add(id); + } + + List activeTests = new ArrayList(); + activeTests.add(testIds[3]); + activeTests.add(testIds[4]); + activeTests.add(testIds[5]); + + String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; + + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); + Set excludes = new HashSet<>(); + excludes.add("dEQP-GLES3.missing#*"); + addFilterFileForOption(deqpTest, excludes, "exclude-filter-file"); + testFiltering(deqpTest, expectedTrie, activeTests); + } + + public void testFilterComboWithFiles() throws Exception { + final TestIdentifier[] testIds = { + new TestIdentifier("dEQP-GLES3.group1", "footah"), + new TestIdentifier("dEQP-GLES3.group1", "foo"), + new TestIdentifier("dEQP-GLES3.group1", "nope"), + new TestIdentifier("dEQP-GLES3.group1", "nonotwant"), + new TestIdentifier("dEQP-GLES3.group2", "foo"), + new TestIdentifier("dEQP-GLES3.group2", "yes"), + new TestIdentifier("dEQP-GLES3.group2", "thoushallnotpass"), + }; + + List allTests = new ArrayList(); + for (TestIdentifier id : testIds) { + allTests.add(id); + } + + List activeTests = new ArrayList(); + activeTests.add(testIds[0]); + activeTests.add(testIds[5]); + + String expectedTrie = "{dEQP-GLES3{group1{footah}group2{yes}}}"; + + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); + + Set includes = new HashSet<>(); + includes.add("dEQP-GLES3.group2#*"); + deqpTest.addAllIncludeFilters(includes); + + Set fileIncludes = new HashSet<>(); + fileIncludes.add("dEQP-GLES3.group1#no*"); + fileIncludes.add("dEQP-GLES3.group1#foo*"); + addFilterFileForOption(deqpTest, fileIncludes, "include-filter-file"); + + Set fileExcludes = new HashSet<>(); + fileExcludes.add("*foo"); + fileExcludes.add("*thoushallnotpass"); + addFilterFileForOption(deqpTest, fileExcludes, "exclude-filter-file"); + + deqpTest.addExcludeFilter("dEQP-GLES3.group1#no*"); + + testFiltering(deqpTest, expectedTrie, activeTests); + } + + public void testDotToHashConversionInFilters() throws Exception { + final TestIdentifier[] testIds = { + new TestIdentifier("dEQP-GLES3.missing", "no"), + new TestIdentifier("dEQP-GLES3.pick_me", "donotwant"), + new TestIdentifier("dEQP-GLES3.pick_me", "yes") + }; + + List allTests = new ArrayList(); + for (TestIdentifier id : testIds) { + allTests.add(id); + } + + List activeTests = new ArrayList(); + activeTests.add(testIds[2]); + + String expectedTrie = "{dEQP-GLES3{pick_me{yes}}}"; + + DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); + deqpTest.addIncludeFilter("dEQP-GLES3.pick_me.yes"); + testFiltering(deqpTest, expectedTrie, activeTests); + } } diff --git a/external/vulkancts/modules/vulkan/api/vktApiFeatureInfo.cpp b/external/vulkancts/modules/vulkan/api/vktApiFeatureInfo.cpp index 8b98f08..847d07f 100644 --- a/external/vulkancts/modules/vulkan/api/vktApiFeatureInfo.cpp +++ b/external/vulkancts/modules/vulkan/api/vktApiFeatureInfo.cpp @@ -2108,28 +2108,12 @@ tcu::TestStatus imageFormatProperties (Context& context, const VkFormat format, results.check(imageType != VK_IMAGE_TYPE_3D || (properties.maxExtent.width >= 1 && properties.maxExtent.height >= 1 && properties.maxExtent.depth >= 1), "Invalid dimensions for 3D image"); results.check(imageType != VK_IMAGE_TYPE_3D || properties.maxArrayLayers == 1, "Invalid maxArrayLayers for 3D image"); - if (tiling == VK_IMAGE_TILING_OPTIMAL) + if (tiling == VK_IMAGE_TILING_OPTIMAL && imageType == VK_IMAGE_TYPE_2D && !(curCreateFlags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) && + ((supportedFeatures & (VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT)) || + ((supportedFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && deviceFeatures.shaderStorageImageMultisample))) { - // Vulkan API specification has changed since initial Android Nougat release. - // For NYC CTS we need to tolerate old behavior as well and issue compatibility - // warning instead. - // - // See spec issues 272, 282, 302, 445 and CTS issues 369, 440. - const bool requiredByNewSpec = (imageType == VK_IMAGE_TYPE_2D && !(curCreateFlags & VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT) && - ((supportedFeatures & (VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT)) || - ((supportedFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && deviceFeatures.shaderStorageImageMultisample))); - - if (requiredByNewSpec) - { - const VkSampleCountFlags requiredSampleCounts = getRequiredOptimalTilingSampleCounts(deviceLimits, format, curUsageFlags); - - results.check((properties.sampleCounts & requiredSampleCounts) == requiredSampleCounts, "Required sample counts not supported"); - } - else if (properties.sampleCounts != VK_SAMPLE_COUNT_1_BIT) - { - results.addResult(QP_TEST_RESULT_COMPATIBILITY_WARNING, - "Implementation supports more sample counts than allowed by the spec"); - } + const VkSampleCountFlags requiredSampleCounts = getRequiredOptimalTilingSampleCounts(deviceLimits, format, curUsageFlags); + results.check((properties.sampleCounts & requiredSampleCounts) == requiredSampleCounts, "Required sample counts not supported"); } else results.check(properties.sampleCounts == VK_SAMPLE_COUNT_1_BIT, "sampleCounts != VK_SAMPLE_COUNT_1_BIT");