package com.drawelements.deqp.runner;
import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
-import com.android.compatibility.common.util.AbiUtils;
import com.android.ddmlib.AdbCommandRejectedException;
import com.android.ddmlib.IShellOutputReceiver;
import com.android.ddmlib.MultiLineReceiver;
import com.android.ddmlib.TimeoutException;
import com.android.ddmlib.testrunner.TestIdentifier;
import com.android.tradefed.build.IBuildInfo;
-import com.android.tradefed.build.IFolderBuildInfo;
import com.android.tradefed.config.Option;
import com.android.tradefed.config.OptionClass;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.result.LogDataType;
import com.android.tradefed.testtype.IAbi;
-import com.android.tradefed.testtype.IBuildReceiver;
import com.android.tradefed.testtype.IAbiReceiver;
+import com.android.tradefed.testtype.IBuildReceiver;
import com.android.tradefed.testtype.IDeviceTest;
import com.android.tradefed.testtype.IRemoteTest;
import com.android.tradefed.testtype.IRuntimeHintProvider;
import com.android.tradefed.testtype.IShardableTest;
+import com.android.tradefed.testtype.IStrictShardableTest;
import com.android.tradefed.testtype.ITestCollector;
import com.android.tradefed.testtype.ITestFilterReceiver;
+import com.android.tradefed.util.AbiUtils;
import com.android.tradefed.util.IRunUtil;
import com.android.tradefed.util.RunInterruptedException;
import com.android.tradefed.util.RunUtil;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
-import java.lang.reflect.Method;
import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.regex.Pattern;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
/**
* Test runner for dEQP tests
@OptionClass(alias="deqp-test-runner")
public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
ITestFilterReceiver, IAbiReceiver, IShardableTest, ITestCollector,
- IRuntimeHintProvider {
+ IRuntimeHintProvider, IStrictShardableTest {
private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk";
private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp";
private static final String INCOMPLETE_LOG_MESSAGE = "Crash: Incomplete test log";
@Option(name = "include-filter",
description="Test include filter. '*' is zero or more letters. '.' has no special meaning.")
private List<String> mIncludeFilters = new ArrayList<>();
+ @Option(name = "include-filter-file",
+ description="Load list of includes from the files given.")
+ private List<String> mIncludeFilterFiles = new ArrayList<>();
@Option(name = "exclude-filter",
description="Test exclude filter. '*' is zero or more letters. '.' has no special meaning.")
private List<String> mExcludeFilters = new ArrayList<>();
+ @Option(name = "exclude-filter-file",
+ description="Load list of excludes from the files given.")
+ private List<String> mExcludeFilterFiles = new ArrayList<>();
@Option(name = "collect-tests-only",
description = "Only invoke the instrumentation to collect list of applicable test "
+ "cases. All test run callbacks will be triggered, but test execution will "
}
/**
- * Test configuration of dEPQ test instance execution.
- * Exposed for unit testing
- */
- public static final class BatchRunConfiguration {
- public static final String ROTATION_UNSPECIFIED = "unspecified";
- public static final String ROTATION_PORTRAIT = "0";
- public static final String ROTATION_LANDSCAPE = "90";
- public static final String ROTATION_REVERSE_PORTRAIT = "180";
- public static final String ROTATION_REVERSE_LANDSCAPE = "270";
-
- private final String mGlConfig;
- private final String mRotation;
- private final String mSurfaceType;
- private final boolean mRequired;
-
- public BatchRunConfiguration(String glConfig, String rotation, String surfaceType, boolean required) {
- mGlConfig = glConfig;
- mRotation = rotation;
- mSurfaceType = surfaceType;
- mRequired = required;
- }
-
- /**
- * Get string that uniquely identifies this config
- */
- public String getId() {
- return String.format("{glformat=%s,rotation=%s,surfacetype=%s,required=%b}",
- mGlConfig, mRotation, mSurfaceType, mRequired);
- }
-
- /**
- * Get the GL config used in this configuration.
- */
- public String getGlConfig() {
- return mGlConfig;
- }
-
- /**
- * Get the screen rotation used in this configuration.
- */
- public String getRotation() {
- return mRotation;
- }
-
- /**
- * Get the surface type used in this configuration.
- */
- public String getSurfaceType() {
- return mSurfaceType;
- }
-
- /**
- * Is this configuration mandatory to support, if target API is supported?
- */
- public boolean isRequired() {
- return mRequired;
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == null) {
- return false;
- } else if (!(other instanceof BatchRunConfiguration)) {
- return false;
- } else {
- return getId().equals(((BatchRunConfiguration)other).getId());
- }
- }
-
- @Override
- public int hashCode() {
- return getId().hashCode();
- }
- }
-
- /**
* dEQP test instance listerer and invocation result forwarded
*/
private class TestInstanceResultListener {
}
private static class SleepProvider implements ISleepProvider {
+ @Override
public void sleep(int milliseconds) {
- try {
- Thread.sleep(milliseconds);
- } catch (InterruptedException ex) {
- }
+ RunUtil.getDefault().sleep(milliseconds);
}
}
/**
* Tries to recover device after abnormal execution termination or link failure.
*
- * @param progressedSinceLastCall true if test execution has progressed since last call
* @throws DeviceNotAvailableException if recovery did not succeed
*/
public void recoverComLinkKilled() throws DeviceNotAvailableException;
- };
+ }
/**
* State machine for execution failure recovery.
RECOVER, // recover by calling recover()
REBOOT, // recover by rebooting
FAIL, // cannot recover
- };
+ }
private MachineState mState = MachineState.WAIT;
private ITestDevice mDevice;
/**
* {@inheritDoc}
*/
+ @Override
public void setSleepProvider(ISleepProvider sleepProvider) {
mSleepProvider = sleepProvider;
}
case FAIL:
// Third failure in a row, just fail
CLog.w("Cannot recover ADB connection");
- throw new DeviceNotAvailableException("failed to connect after reboot");
+ throw new DeviceNotAvailableException("failed to connect after reboot",
+ mDevice.getSerialNumber());
}
}
case FAIL:
// Fourth failure in a row, just fail
CLog.w("Cannot recover ADB connection");
- throw new DeviceNotAvailableException("link killed after reboot");
+ throw new DeviceNotAvailableException("link killed after reboot",
+ mDevice.getSerialNumber());
}
}
}
private static Map<TestIdentifier, Set<BatchRunConfiguration>> generateTestInstances(
- Reader testlist, String configName, String screenRotation, String surfaceType, boolean required) throws FileNotFoundException {
+ Reader testlist, String configName, String screenRotation, String surfaceType,
+ boolean required) {
// Note: This is specifically a LinkedHashMap to guarantee that tests are iterated
// in the insertion order.
final Map<TestIdentifier, Set<BatchRunConfiguration>> instances = new LinkedHashMap<>();
return instances;
}
- private Set<BatchRunConfiguration> getTestRunConfigs (TestIdentifier testId) {
+ private Set<BatchRunConfiguration> getTestRunConfigs(TestIdentifier testId) {
return mTestInstances.get(testId);
}
/**
+ * Get the test instance of the runner. Exposed for testing.
+ */
+ Map<TestIdentifier, Set<BatchRunConfiguration>> getTestInstance() {
+ return mTestInstances;
+ }
+
+ /**
* Converts dEQP testcase path to TestIdentifier.
*/
private static TestIdentifier pathToIdentifier(String testPath) {
/**
* Get GL major version (based on package name)
*/
- private int getGlesMajorVersion() throws DeviceNotAvailableException {
+ private int getGlesMajorVersion() {
if ("dEQP-GLES2".equals(mDeqpPackage)) {
return 2;
} else if ("dEQP-GLES3".equals(mDeqpPackage)) {
/**
* Get GL minor version (based on package name)
*/
- private int getGlesMinorVersion() throws DeviceNotAvailableException {
+ private int getGlesMinorVersion() {
if ("dEQP-GLES2".equals(mDeqpPackage)) {
return 0;
} else if ("dEQP-GLES3".equals(mDeqpPackage)) {
Set<String> nonPatternFilters = new HashSet<String>();
for (String filter : filters) {
if (!filter.contains("*")) {
- nonPatternFilters.add(filter);
+ // Deqp usesly only dots for separating between parts of the names
+ // Convert last dot to hash if needed.
+ if (!filter.contains("#")) {
+ int lastSeparator = filter.lastIndexOf('.');
+ String filterWithHash = filter.substring(0, lastSeparator) + "#" +
+ filter.substring(lastSeparator + 1, filter.length());
+ nonPatternFilters.add(filterWithHash);
+ }
+ else {
+ nonPatternFilters.add(filter);
+ }
}
}
return nonPatternFilters;
List<Pattern> includePatterns = getPatternFilters(includeFilters);
List<Pattern> excludePatterns = getPatternFilters(excludeFilters);
- List<TestIdentifier> testList = new ArrayList(tests.keySet());
+ List<TestIdentifier> testList = new ArrayList<>(tests.keySet());
for (TestIdentifier test : testList) {
if (excludeStrings.contains(test.toString())) {
tests.remove(test); // remove test if explicitly excluded
}
/**
+ * Read a list of filters from a file.
+ *
+ * Note: Filters can be numerous so we prefer, for performance
+ * reasons, to add directly to the target list instead of using
+ * intermediate return value.
+ */
+ static private void readFilterFile(List<String> filterList, File file) throws FileNotFoundException {
+ if (!file.canRead()) {
+ CLog.e("Failed to read filter file '%s'", file.getPath());
+ throw new FileNotFoundException();
+ }
+ try (Reader plainReader = new FileReader(file);
+ BufferedReader reader = new BufferedReader(plainReader)) {
+ String filter = "";
+ while ((filter = reader.readLine()) != null) {
+ // TOOD: Sanity check filter
+ filterList.add(filter);
+ }
+ // Rely on try block to autoclose
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException("Failed to read filter list file '" + file.getPath() + "': " +
+ e.getMessage());
+ }
+ }
+
+ /**
+ * Prints filters into debug log stream, limiting to 20 entries.
+ */
+ static private void printFilters(List<String> filters) {
+ int numPrinted = 0;
+ for (String filter : filters) {
+ CLog.d(" %s", filter);
+ if (++numPrinted == 20) {
+ CLog.d(" ... AND %d others", filters.size() - numPrinted);
+ break;
+ }
+ }
+ }
+
+ /**
* Loads tests into mTestInstances based on the options. Assumes
* that no tests have been loaded for this instance before.
*/
catch (IOException e) {
CLog.w("Failed to close test list reader.");
}
- CLog.d("Filters");
- for (String filter : mIncludeFilters) {
- CLog.d("Include: %s", filter);
+
+ try
+ {
+ for (String filterFile : mIncludeFilterFiles) {
+ CLog.d("Read include filter file '%s'", filterFile);
+ File file = new File(mBuildHelper.getTestsDir(), filterFile);
+ readFilterFile(mIncludeFilters, file);
+ }
+ for (String filterFile : mExcludeFilterFiles) {
+ CLog.d("Read exclude filter file '%s'", filterFile);
+ File file = new File(mBuildHelper.getTestsDir(), filterFile);
+ readFilterFile(mExcludeFilters, file);
+ }
}
- for (String filter : mExcludeFilters) {
- CLog.d("Exclude: %s", filter);
+ catch (FileNotFoundException e) {
+ throw new RuntimeException("Cannot read deqp filter list file:" + e.getMessage());
}
+ CLog.d("Include filters:");
+ printFilters(mIncludeFilters);
+ CLog.d("Exclude filters:");
+ printFilters(mExcludeFilters);
+
long originalTestCount = mTestInstances.size();
CLog.i("Num tests before filtering: %d", originalTestCount);
if ((!mIncludeFilters.isEmpty() || !mExcludeFilters.isEmpty()) && originalTestCount > 0) {
listener.testRunStarted(getId(), mRemainingTests.size());
try {
+ if (mRemainingTests.isEmpty()) {
+ CLog.d("No tests to run.");
+ return;
+ }
final boolean isSupportedApi = (isOpenGlEsPackage() && isSupportedGles())
|| (isVulkanPackage() && isSupportedVulkan())
|| (!isOpenGlEsPackage() && !isVulkanPackage());
// test cases in "NotExecuted" state
CLog.e("Capability query failed - leaving tests unexecuted.");
uninstallTestApk();
+ } finally {
+ listener.testRunEnded(System.currentTimeMillis() - startTime, emptyMap);
}
-
- listener.testRunEnded(System.currentTimeMillis() - startTime, emptyMap);
}
/**
destination.mSurfaceType = source.mSurfaceType;
destination.mConfigRequired = source.mConfigRequired;
destination.mIncludeFilters = new ArrayList<>(source.mIncludeFilters);
+ destination.mIncludeFilterFiles = new ArrayList<>(source.mIncludeFilterFiles);
destination.mExcludeFilters = new ArrayList<>(source.mExcludeFilters);
+ destination.mExcludeFilterFiles = new ArrayList<>(source.mExcludeFilterFiles);
destination.mAbi = source.mAbi;
destination.mLogData = source.mLogData;
destination.mCollectTestsOnly = source.mCollectTestsOnly;
}
/**
+ * Helper to update the RuntimeHint of the tests after being sharded.
+ */
+ private void updateRuntimeHint(long originalSize, Collection<IRemoteTest> runners) {
+ if (originalSize > 0) {
+ long fullRuntimeMs = getRuntimeHint();
+ for (IRemoteTest remote: runners) {
+ DeqpTestRunner runner = (DeqpTestRunner)remote;
+ long shardRuntime = (fullRuntimeMs * runner.mTestInstances.size()) / originalSize;
+ runner.mRuntimeHint = shardRuntime;
+ }
+ }
+ }
+
+ /**
* {@inheritDoc}
*/
@Override
Map<TestIdentifier, Set<BatchRunConfiguration>> currentSet = new LinkedHashMap<>();
Map<TestIdentifier, Set<BatchRunConfiguration>> iterationSet = this.mTestInstances;
+ if (iterationSet.keySet().isEmpty()) {
+ CLog.i("Cannot split deqp tests, no tests to run");
+ return null;
+ }
+
// Go through tests, split
for (TestIdentifier test: iterationSet.keySet()) {
currentSet.put(test, iterationSet.get(test));
runners.add(new DeqpTestRunner(this, currentSet));
// NOTE: Use linked hash map to keep the insertion order in iteration
currentSet = new LinkedHashMap<>();
- }
+ }
}
runners.add(new DeqpTestRunner(this, currentSet));
// Compute new runtime hints
- long originalSize = iterationSet.size();
- if (originalSize > 0) {
- long fullRuntimeMs = getRuntimeHint();
- for (IRemoteTest remote: runners) {
- DeqpTestRunner runner = (DeqpTestRunner)remote;
- long shardRuntime = (fullRuntimeMs * runner.mTestInstances.size()) / originalSize;
- runner.mRuntimeHint = shardRuntime;
+ updateRuntimeHint(iterationSet.size(), runners);
+ CLog.i("Split deqp tests into %d shards", runners.size());
+ return runners;
+ }
+
+ /**
+ * This sharding should be deterministic for the same input and independent.
+ * Through this API, each shard could be executed on different machine.
+ */
+ @Override
+ public IRemoteTest getTestShard(int shardCount, int shardIndex) {
+ // TODO: refactor getTestshard and split to share some logic.
+ if (mTestInstances == null) {
+ loadTests();
+ }
+
+ List<IRemoteTest> runners = new ArrayList<>();
+ // NOTE: Use linked hash map to keep the insertion order in iteration
+ Map<TestIdentifier, Set<BatchRunConfiguration>> currentSet = new LinkedHashMap<>();
+ Map<TestIdentifier, Set<BatchRunConfiguration>> iterationSet = this.mTestInstances;
+
+ int batchLimit = iterationSet.keySet().size() / shardCount;
+ int i = 1;
+ // Go through tests, split
+ for (TestIdentifier test: iterationSet.keySet()) {
+ currentSet.put(test, iterationSet.get(test));
+ if (currentSet.size() >= batchLimit && i < shardCount) {
+ runners.add(new DeqpTestRunner(this, currentSet));
+ i++;
+ // NOTE: Use linked hash map to keep the insertion order in iteration
+ currentSet = new LinkedHashMap<>();
}
}
+ runners.add(new DeqpTestRunner(this, currentSet));
- CLog.i("Split deqp tests into %d shards", runners.size());
- return runners;
+ // Compute new runtime hints
+ updateRuntimeHint(iterationSet.size(), runners);
+
+ // If too many shards were requested, we complete with placeholder.
+ if (runners.size() < shardCount) {
+ for (int j = runners.size(); j < shardCount; j++) {
+ runners.add(new DeqpTestRunner(this,
+ new LinkedHashMap<TestIdentifier, Set<BatchRunConfiguration>>()));
+ }
+ }
+
+ CLog.i("Split deqp tests into %d shards, return shard: %s", runners.size(), shardIndex);
+ return runners.get(shardIndex);
}
/**
package com.drawelements.deqp.runner;
import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
-import com.android.compatibility.common.util.AbiUtils;
import com.android.ddmlib.IDevice;
import com.android.ddmlib.IShellOutputReceiver;
-import com.android.ddmlib.ShellCommandUnresponsiveException;
import com.android.ddmlib.testrunner.TestIdentifier;
import com.android.tradefed.build.IFolderBuildInfo;
import com.android.tradefed.config.ConfigurationException;
import com.android.tradefed.testtype.IAbi;
import com.android.tradefed.testtype.IRemoteTest;
import com.android.tradefed.testtype.IRuntimeHintProvider;
+import com.android.tradefed.util.AbiUtils;
+import com.android.tradefed.util.FileUtil;
import com.android.tradefed.util.IRunUtil;
import com.android.tradefed.util.RunInterruptedException;
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.ArrayList;
DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window");
}
- private static class StubRecovery implements DeqpTestRunner.IRecovery {
- /**
- * {@inheritDoc}
- */
- @Override
- public void setSleepProvider(DeqpTestRunner.ISleepProvider sleepProvider) {
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void setDevice(ITestDevice device) {
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void onExecutionProgressed() {
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void recoverConnectionRefused() throws DeviceNotAvailableException {
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void recoverComLinkKilled() throws DeviceNotAvailableException {
- }
- };
+ private File mTestsDir = null;
public static class BuildHelperMock extends CompatibilityBuildHelper {
- public BuildHelperMock(IFolderBuildInfo buildInfo) {
+ private File mTestsDir = null;
+ public BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir) {
super(buildInfo);
+ mTestsDir = testsDir;
}
@Override
public File getTestsDir() throws FileNotFoundException {
- return new File("logs");
+ return mTestsDir;
}
}
@Override
protected void setUp() throws Exception {
super.setUp();
+ mTestsDir = FileUtil.createTempDir("deqp-test-cases");
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void tearDown() throws Exception {
+ FileUtil.recursiveDelete(mTestsDir);
+ super.tearDown();
}
private static DeqpTestRunner buildGlesTestRunner(int majorVersion,
int minorVersion,
- Collection<TestIdentifier> tests) throws ConfigurationException, FileNotFoundException {
+ Collection<TestIdentifier> tests,
+ File testsDir) throws ConfigurationException, FileNotFoundException {
StringWriter testlist = new StringWriter();
for (TestIdentifier test : tests) {
testlist.write(test.getClassName() + "." + test.getTestName() + "\n");
}
- return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString());
+ return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString(), testsDir);
}
- private static CompatibilityBuildHelper getMockBuildHelper() {
+ private static CompatibilityBuildHelper getMockBuildHelper(File testsDir) {
IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class);
EasyMock.replay(mockIFolderBuildInfo);
- return new BuildHelperMock(mockIFolderBuildInfo);
+ return new BuildHelperMock(mockIFolderBuildInfo, testsDir);
}
private static DeqpTestRunner buildGlesTestRunner(int majorVersion,
int minorVersion,
- String testlist) throws ConfigurationException, FileNotFoundException {
+ String testlist,
+ File testsDir) throws ConfigurationException, FileNotFoundException {
DeqpTestRunner runner = new DeqpTestRunner();
OptionSetter setter = new OptionSetter(runner);
runner.setCaselistReader(new StringReader(testlist));
runner.setAbi(ABI);
- runner.setBuildHelper(getMockBuildHelper());
+ runner.setBuildHelper(getMockBuildHelper(testsDir));
return runner;
}
Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
tests.add(testId);
- DeqpTestRunner deqpTest = buildGlesTestRunner(requiredMajorVersion, requiredMinorVersion, tests);
+ DeqpTestRunner deqpTest = buildGlesTestRunner(requiredMajorVersion, requiredMinorVersion, tests, mTestsDir);
int version = (majorVersion << 16) | minorVersion;
EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
tests.add(testId);
- DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
int version = 3 << 16;
EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
tests.add(id);
}
- DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
int version = 3 << 16;
EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
return output.toString();
}
- private void testFiltering(Set<String> includes,
- Set<String> excludes,
- List<TestIdentifier> fullTestList,
+ private void testFiltering(DeqpTestRunner deqpTest,
String expectedTrie,
List<TestIdentifier> expectedTests) throws Exception {
-
- boolean thereAreTests = !expectedTests.isEmpty();
- ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
- ITestInvocationListener mockListener
- = EasyMock.createStrictMock(ITestInvocationListener.class);
- IDevice mockIDevice = EasyMock.createMock(IDevice.class);
-
- DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, fullTestList);
- if (includes != null) {
- deqpTest.addAllIncludeFilters(includes);
- }
- if (excludes != null) {
- deqpTest.addAllExcludeFilters(excludes);
- }
-
int version = 3 << 16;
+ ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
.andReturn(Integer.toString(version)).atLeastOnce();
+ boolean thereAreTests = !expectedTests.isEmpty();
if (thereAreTests)
{
// only expect to install/uninstall packages if there are any tests
.andReturn(null).once();
}
-
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
mockListener.testRunStarted(getTestId(deqpTest), expectedTests.size());
EasyMock.expectLastCall().once();
+ IDevice mockIDevice = EasyMock.createMock(IDevice.class);
if (thereAreTests)
{
expectRenderConfigQuery(mockDevice, 3, 0);
String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
- Set<String> includes = new HashSet();
- includes.add("dEQP-GLES3.pick_me#*");
- testFiltering(includes, null, allTests, expectedTrie, activeTests);
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
+ deqpTest.addIncludeFilter("dEQP-GLES3.pick_me#*");
+ testFiltering(deqpTest, expectedTrie, activeTests);
}
public void testRun_trivialExcludeFilter() throws Exception {
String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
- Set<String> excludes = new HashSet();
- excludes.add("dEQP-GLES3.missing#*");
- testFiltering(null, excludes, allTests, expectedTrie, activeTests);
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
+ deqpTest.addExcludeFilter("dEQP-GLES3.missing#*");
+ testFiltering(deqpTest, expectedTrie, activeTests);
}
public void testRun_includeAndExcludeFilter() throws Exception {
String expectedTrie = "{dEQP-GLES3{group2{yes}}}";
- Set<String> includes = new HashSet();
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
+
+ Set<String> includes = new HashSet<>();
includes.add("dEQP-GLES3.group2#*");
+ deqpTest.addAllIncludeFilters(includes);
- Set<String> excludes = new HashSet();
+ Set<String> excludes = new HashSet<>();
excludes.add("*foo");
excludes.add("*thoushallnotpass");
- testFiltering(includes, excludes, allTests, expectedTrie, activeTests);
+ deqpTest.addAllExcludeFilters(excludes);
+ testFiltering(deqpTest, expectedTrie, activeTests);
}
public void testRun_includeAll() throws Exception {
String expectedTrie = "{dEQP-GLES3{group1{mememe,yeah,takeitall},group2{jeba,yes,granted}}}";
- Set<String> includes = new HashSet();
- includes.add("*");
-
- testFiltering(includes, null, allTests, expectedTrie, allTests);
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
+ deqpTest.addIncludeFilter("*");
+ testFiltering(deqpTest, expectedTrie, allTests);
}
public void testRun_excludeAll() throws Exception {
allTests.add(id);
}
- String expectedTrie = "";
-
- Set<String> excludes = new HashSet();
- excludes.add("*");
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
+ deqpTest.addExcludeFilter("*");
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ mockListener.testRunStarted(getTestId(deqpTest), 0);
+ EasyMock.expectLastCall().once();
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<Map<String, String>>notNull());
+ EasyMock.expectLastCall().once();
- testFiltering(null, excludes, allTests, expectedTrie, new ArrayList<TestIdentifier>());
+ EasyMock.replay(mockListener);
+ deqpTest.run(mockListener);
+ EasyMock.verify(mockListener);
}
/**
tests.add(id);
}
- DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
int version = 3 << 16;
EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
tests.add(testId);
- DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
OptionSetter setter = new OptionSetter(deqpTest);
// Note: If the rotation is the default unspecified, features are not queried at all
setter.setOptionValue("deqp-screen-rotation", "90");
Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
tests.add(testId);
- DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
deqpTest.setDevice(mockDevice);
Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
tests.add(testId);
- DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
OptionSetter setter = new OptionSetter(deqpTest);
setter.setOptionValue("deqp-screen-rotation", rotation);
EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
.andReturn(Integer.toString(version)).atLeastOnce();
- if (!rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED)) {
+ if (!rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED)) {
EasyMock.expect(mockDevice.executeShellCommand("pm list features"))
.andReturn(featureString);
}
final boolean isPortraitOrientation =
- rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_PORTRAIT) ||
- rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT);
+ rotation.equals(BatchRunConfiguration.ROTATION_PORTRAIT) ||
+ rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT);
final boolean isLandscapeOrientation =
- rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_LANDSCAPE) ||
- rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE);
+ rotation.equals(BatchRunConfiguration.ROTATION_LANDSCAPE) ||
+ rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE);
final boolean executable =
- rotation.equals(DeqpTestRunner.BatchRunConfiguration.ROTATION_UNSPECIFIED) ||
+ rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED) ||
(isPortraitOrientation &&
featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) ||
(isLandscapeOrientation &&
public void testRun_unsupportedPixelFormat() throws Exception {
final String pixelFormat = "rgba5658d16m4";
final TestIdentifier testId = new TestIdentifier("dEQP-GLES3.pixelformat", "test");
- final String testPath = "dEQP-GLES3.pixelformat.test";
- final String testTrie = "{dEQP-GLES3{pixelformat{test}}}";
- final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
- + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
- + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
- + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
- + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
- + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
- + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
- + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
- + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
- + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
- + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
- + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
- + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
- + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
- + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
- + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
- + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
- + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
- + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
- + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
- + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
- + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
- + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
- + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
- + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
- + "INSTRUMENTATION_CODE: 0\r\n";
ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
ITestInvocationListener mockListener
Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
tests.add(testId);
- DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
OptionSetter setter = new OptionSetter(deqpTest);
setter.setOptionValue("deqp-gl-config-name", pixelFormat);
PROGRESS,
FAIL_CONNECTION_REFUSED,
FAIL_LINK_KILLED,
- };
+ }
private void runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events)
throws DeviceNotAvailableException {
DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery();
IMocksControl orderedControl = EasyMock.createStrictControl();
RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class);
+ EasyMock.expect(mockDevice.getSerialNumber()).andStubReturn("SERIAL");
DeqpTestRunner.ISleepProvider mockSleepProvider =
orderedControl.createMock(DeqpTestRunner.ISleepProvider.class);
IDevice mockIDevice = EasyMock.createMock(IDevice.class);
IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class);
- DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
deqpTest.setDevice(mockDevice);
deqpTest.setRunUtil(mockRunUtil);
mockListener.testRunStarted(getTestId(deqpTest), 1);
EasyMock.expectLastCall().once();
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.anyObject());
+ EasyMock.expectLastCall().once();
EasyMock.replay(mockDevice, mockIDevice);
EasyMock.replay(mockListener);
Collection<TestIdentifier> tests = new ArrayList<TestIdentifier>();
for (TestIdentifier id : testIds) tests.add(id);
- DeqpTestRunner runner = buildGlesTestRunner(3, 0, tests);
+ DeqpTestRunner runner = buildGlesTestRunner(3, 0, tests, mTestsDir);
ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split();
for (int shardIndex = 0; shardIndex < shards.size(); shardIndex++) {
DeqpTestRunner shard = (DeqpTestRunner)shards.get(shardIndex);
- shard.setBuildHelper(getMockBuildHelper());
+ shard.setBuildHelper(getMockBuildHelper(mTestsDir));
ArrayList<TestIdentifier> shardTests = testsForShard.get(shardIndex);
}
public void testSharding_empty() throws Exception {
- DeqpTestRunner runner = buildGlesTestRunner(3, 0, new ArrayList<TestIdentifier>());
+ DeqpTestRunner runner = buildGlesTestRunner(3, 0, new ArrayList<TestIdentifier>(), mTestsDir);
ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split();
- // \todo [2015-11-23 kalle] What should the result be? The runner or nothing?
+ // Returns null when cannot be sharded.
+ assertNull(shards);
}
/**
IDevice mockIDevice = EasyMock.createMock(IDevice.class);
IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class);
- DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
deqpTest.setDevice(mockDevice);
deqpTest.setRunUtil(mockRunUtil);
mockListener.testFailed(EasyMock.eq(testId), EasyMock.<String>notNull());
EasyMock.expectLastCall().andThrow(new RunInterruptedException());
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.anyObject());
+ EasyMock.expectLastCall().once();
EasyMock.replay(mockDevice, mockIDevice);
EasyMock.replay(mockListener);
EasyMock.replay(mockRunUtil);
tests.add(id);
}
- DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
OptionSetter setter = new OptionSetter(deqpTest);
final long runtimeMs = 123456;
setter.setOptionValue("runtime-hint", String.valueOf(runtimeMs));
testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i)));
}
- DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds);
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir);
OptionSetter setter = new OptionSetter(deqpTest);
final long fullRuntimeMs = testIds.size()*100;
setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs));
((IRuntimeHintProvider)shards.get(1)).getRuntimeHint());
}
+ /**
+ * Test that strict shardable is able to split deterministically the set of tests.
+ */
+ public void testGetTestShard() throws Exception {
+ final int TEST_COUNT = 2237;
+ final int SHARD_COUNT = 4;
+
+ ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT);
+ for (int i = 0; i < TEST_COUNT; i++) {
+ testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i)));
+ }
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir);
+ OptionSetter setter = new OptionSetter(deqpTest);
+ final long fullRuntimeMs = testIds.size()*100;
+ setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs));
+
+ DeqpTestRunner shard1 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 0);
+ assertEquals(559, shard1.getTestInstance().size());
+ int j = 0;
+ // Ensure numbers, and that order is stable
+ for (TestIdentifier t : shard1.getTestInstance().keySet()) {
+ assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
+ String.format("%s#%s", t.getClassName(), t.getTestName()));
+ j++;
+ }
+ DeqpTestRunner shard2 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 1);
+ assertEquals(559, shard2.getTestInstance().size());
+ for (TestIdentifier t : shard2.getTestInstance().keySet()) {
+ assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
+ String.format("%s#%s", t.getClassName(), t.getTestName()));
+ j++;
+ }
+ DeqpTestRunner shard3 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 2);
+ assertEquals(559, shard3.getTestInstance().size());
+ for (TestIdentifier t : shard3.getTestInstance().keySet()) {
+ assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
+ String.format("%s#%s", t.getClassName(), t.getTestName()));
+ j++;
+ }
+ DeqpTestRunner shard4 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 3);
+ assertEquals(560, shard4.getTestInstance().size());
+ for (TestIdentifier t : shard4.getTestInstance().keySet()) {
+ assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
+ String.format("%s#%s", t.getClassName(), t.getTestName()));
+ j++;
+ }
+ assertEquals(TEST_COUNT, j);
+ }
+
+ /**
+ * Test that strict shardable is creating an empty shard of the runner when too many shards
+ * are requested.
+ */
+ public void testGetTestShard_tooManyShardRequested() throws Exception {
+ final int TEST_COUNT = 2;
+ final int SHARD_COUNT = 3;
+
+ ArrayList<TestIdentifier> testIds = new ArrayList<>(TEST_COUNT);
+ for (int i = 0; i < TEST_COUNT; i++) {
+ testIds.add(new TestIdentifier("dEQP-GLES3.funny.group", String.valueOf(i)));
+ }
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir);
+ OptionSetter setter = new OptionSetter(deqpTest);
+ final long fullRuntimeMs = testIds.size()*100;
+ setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs));
+ DeqpTestRunner shard1 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 0);
+ assertEquals(1, shard1.getTestInstance().size());
+ int j = 0;
+ // Ensure numbers, and that order is stable
+ for (TestIdentifier t : shard1.getTestInstance().keySet()) {
+ assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
+ String.format("%s#%s", t.getClassName(), t.getTestName()));
+ j++;
+ }
+ DeqpTestRunner shard2 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 1);
+ assertEquals(1, shard2.getTestInstance().size());
+ for (TestIdentifier t : shard2.getTestInstance().keySet()) {
+ assertEquals(String.format("dEQP-GLES3.funny.group#%s", j),
+ String.format("%s#%s", t.getClassName(), t.getTestName()));
+ j++;
+ }
+ DeqpTestRunner shard3 = (DeqpTestRunner)deqpTest.getTestShard(SHARD_COUNT, 2);
+ assertTrue(shard3.getTestInstance().isEmpty());
+ assertEquals(TEST_COUNT, j);
+ ITestInvocationListener mockListener
+ = EasyMock.createStrictMock(ITestInvocationListener.class);
+ mockListener.testRunStarted(EasyMock.anyObject(), EasyMock.eq(0));
+ mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.anyObject());
+ EasyMock.replay(mockListener);
+ shard3.run(mockListener);
+ EasyMock.verify(mockListener);
+ }
+
public void testRuntimeHint_optionNotSet() throws Exception {
final TestIdentifier[] testIds = {
new TestIdentifier("dEQP-GLES3.info", "vendor"),
tests.add(id);
}
- DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests);
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
long runtime = deqpTest.getRuntimeHint();
assertTrue("Runtime for tests must be positive", runtime > 0);
}
});
}
+
+ static private void writeStringsToFile(File target, Set<String> strings) throws IOException {
+ try (PrintWriter out = new PrintWriter(new FileWriter(target))) {
+ out.print(String.join(System.lineSeparator(), strings));
+ out.println();
+ }
+ }
+
+ private void addFilterFileForOption(DeqpTestRunner test, Set<String> filters, String option)
+ throws IOException, ConfigurationException {
+ String filterFile = option + ".txt";
+ writeStringsToFile(new File(mTestsDir, filterFile), filters);
+ OptionSetter setter = new OptionSetter(test);
+ setter.setOptionValue(option, filterFile);
+ }
+
+ public void testIncludeFilterFile() throws Exception {
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.missing", "no"),
+ new TestIdentifier("dEQP-GLES3.missing", "nope"),
+ new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
+ };
+
+ List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
+ for (TestIdentifier id : testIds) {
+ allTests.add(id);
+ }
+
+ List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
+ activeTests.add(testIds[3]);
+ activeTests.add(testIds[4]);
+ activeTests.add(testIds[5]);
+
+ String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
+ Set<String> includes = new HashSet<>();
+ includes.add("dEQP-GLES3.pick_me#*");
+ addFilterFileForOption(deqpTest, includes, "include-filter-file");
+ testFiltering(deqpTest, expectedTrie, activeTests);
+ }
+
+ public void testMissingIncludeFilterFile() throws Exception {
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
+ };
+
+ List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
+ for (TestIdentifier id : testIds) {
+ allTests.add(id);
+ }
+
+ String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
+ OptionSetter setter = new OptionSetter(deqpTest);
+ setter.setOptionValue("include-filter-file", "not-a-file.txt");
+ try {
+ testFiltering(deqpTest, expectedTrie, allTests);
+ fail("Test execution should have aborted with exception.");
+ } catch (RuntimeException e) {
+ }
+ }
+
+ public void testExcludeFilterFile() throws Exception {
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.missing", "no"),
+ new TestIdentifier("dEQP-GLES3.missing", "nope"),
+ new TestIdentifier("dEQP-GLES3.missing", "donotwant"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "yes"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "ok"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "accepted"),
+ };
+
+ List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
+ for (TestIdentifier id : testIds) {
+ allTests.add(id);
+ }
+
+ List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
+ activeTests.add(testIds[3]);
+ activeTests.add(testIds[4]);
+ activeTests.add(testIds[5]);
+
+ String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
+ Set<String> excludes = new HashSet<>();
+ excludes.add("dEQP-GLES3.missing#*");
+ addFilterFileForOption(deqpTest, excludes, "exclude-filter-file");
+ testFiltering(deqpTest, expectedTrie, activeTests);
+ }
+
+ public void testFilterComboWithFiles() throws Exception {
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.group1", "footah"),
+ new TestIdentifier("dEQP-GLES3.group1", "foo"),
+ new TestIdentifier("dEQP-GLES3.group1", "nope"),
+ new TestIdentifier("dEQP-GLES3.group1", "nonotwant"),
+ new TestIdentifier("dEQP-GLES3.group2", "foo"),
+ new TestIdentifier("dEQP-GLES3.group2", "yes"),
+ new TestIdentifier("dEQP-GLES3.group2", "thoushallnotpass"),
+ };
+
+ List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
+ for (TestIdentifier id : testIds) {
+ allTests.add(id);
+ }
+
+ List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
+ activeTests.add(testIds[0]);
+ activeTests.add(testIds[5]);
+
+ String expectedTrie = "{dEQP-GLES3{group1{footah}group2{yes}}}";
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
+
+ Set<String> includes = new HashSet<>();
+ includes.add("dEQP-GLES3.group2#*");
+ deqpTest.addAllIncludeFilters(includes);
+
+ Set<String> fileIncludes = new HashSet<>();
+ fileIncludes.add("dEQP-GLES3.group1#no*");
+ fileIncludes.add("dEQP-GLES3.group1#foo*");
+ addFilterFileForOption(deqpTest, fileIncludes, "include-filter-file");
+
+ Set<String> fileExcludes = new HashSet<>();
+ fileExcludes.add("*foo");
+ fileExcludes.add("*thoushallnotpass");
+ addFilterFileForOption(deqpTest, fileExcludes, "exclude-filter-file");
+
+ deqpTest.addExcludeFilter("dEQP-GLES3.group1#no*");
+
+ testFiltering(deqpTest, expectedTrie, activeTests);
+ }
+
+ public void testDotToHashConversionInFilters() throws Exception {
+ final TestIdentifier[] testIds = {
+ new TestIdentifier("dEQP-GLES3.missing", "no"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "donotwant"),
+ new TestIdentifier("dEQP-GLES3.pick_me", "yes")
+ };
+
+ List<TestIdentifier> allTests = new ArrayList<TestIdentifier>();
+ for (TestIdentifier id : testIds) {
+ allTests.add(id);
+ }
+
+ List<TestIdentifier> activeTests = new ArrayList<TestIdentifier>();
+ activeTests.add(testIds[2]);
+
+ String expectedTrie = "{dEQP-GLES3{pick_me{yes}}}";
+
+ DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
+ deqpTest.addIncludeFilter("dEQP-GLES3.pick_me.yes");
+ testFiltering(deqpTest, expectedTrie, activeTests);
+ }
}
projects / platform / upstream / VK-GL-CTS.git / commitdiff