Add a "playlists" flag to runtests.sh that will run a list of tests

diff --git a/netci.groovy b/netci.groovy
index 9bb44419c4f5e0996cc9b9979ea97cf2e18c2f55..65273892f779f6c1db7250365f588a2179a2557c 100755 (executable)
--- a/netci.groovy
+++ b/netci.groovy
@@ -1451,10 +1451,9 @@ combinedScenarios.each { scenario ->
                     // of in parallel. Only used for long GC tests.
                     def sequentialString = ''
                     
-                    // Whether or not this test run should only run failing test.
-                    // Only used for long GC tests, because they are all ignored in
-                    // normal test runs.
-                    def runFailingString = ''
+                    // Whether or not this test run should run a specific playlist.
+                    // Only used for long GC tests.
+                    def playlistString = ''
                      
                     if (os == 'Ubuntu' && isPR){
                         serverGCString = '--useServerGC'
@@ -1471,11 +1470,9 @@ combinedScenarios.each { scenario ->
                         // the only test running (many of them allocate until OOM).
                         sequentialString = '--sequential'
                         
-                        // Long GC tests all exist in the ignore list because
-                        // they can't run during normal test runs. This is not
-                        // particularly pretty but, until we get a more generalized
-                        // mechanism for coming up with test playlists, it works.
-                        runFailingString = '--runFailingTestsOnly'
+                        // The Long GC playlist contains all of the tests that are
+                        // going to be run.
+                        playlistString = '--playlist ./tests/longRunningGcTests.txt'
                     }
                     
 
@@ -1613,7 +1610,7 @@ combinedScenarios.each { scenario ->
                 --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
                 --coreFxBinDir=\"\${WORKSPACE}/bin/${osGroup}.AnyCPU.Release\" \\
                 --coreFxNativeBinDir=\"\${WORKSPACE}/bin/${osGroup}.${architecture}.Release\" \\
-                ${testEnvOpt} ${serverGCString} ${crossgenStr} ${sequentialString} ${runFailingString}""")
+                ${testEnvOpt} ${serverGCString} ${crossgenStr} ${sequentialString} ${playlistString}""")
                             }
                         }
                     }

diff --git a/tests/longRunningGcTests.txt b/tests/longRunningGcTests.txt
new file mode 100644 (file)
index 0000000..8259df9
--- /dev/null
+++ b/tests/longRunningGcTests.txt
@@ -0,0 +1,9 @@
+GC/Features/BackgroundGC/concurrentspin2/concurrentspin2.sh
+GC/Features/LOHCompaction/lohcompactapi2/lohcompactapi2.sh
+GC/Features/LOHCompaction/lohcompactscenariorepro/lohcompactscenariorepro.sh
+GC/Features/LOHCompaction/lohcompact_stress/lohcompact_stress.sh
+GC/Features/PartialCompaction/eco1/eco1.sh
+GC/Features/PartialCompaction/partialcompactiontest/partialcompactiontest.sh
+GC/Features/PartialCompaction/partialcompactionwloh/partialcompactionwloh.sh
+GC/Features/SustainedLowLatency/sustainedlowlatency_race_reverse/sustainedlowlatency_race_reverse.sh
+GC/Features/SustainedLowLatency/sustainedlowlatency_race/sustainedlowlatency_race.sh
\ No newline at end of file

diff --git a/tests/runtest.sh b/tests/runtest.sh
index 8cf2603cee500230a0e2108e52d3243533179c88..369d5f2f0d565644a1f44a867f21651931e94f0b 100755 (executable)
--- a/tests/runtest.sh
+++ b/tests/runtest.sh
@@ -37,6 +37,8 @@ function print_usage {
     echo '                                 line, as paths to .sh files relative to the directory specified by --testRootDir.'
     echo '  --disableEventLogging        : Disable the events logged by both VM and Managed Code'
     echo '  --sequential                 : Run tests sequentially (default is to run in parallel).'
+    echo '  --playlist=<path>            : Run only the tests that are specified in the file at <path>, in the same format as'
+    echo '                                 runFailingTestsOnly'
     echo '  -v, --verbose                : Show output from each test.'
     echo '  -h|--help                    : Show usage information.'
     echo '  --useServerGC                : Enable server GC for this test run'
@@ -409,6 +411,7 @@ function copy_test_native_bin_to_test_root {
 # Variables for unsupported and failing tests
 declare -a unsupportedTests
 declare -a failingTests
+declare -a playlistTests
 ((runFailingTestsOnly = 0))
 
 function load_unsupported_tests {
@@ -427,6 +430,13 @@ function load_failing_tests {
     done <"$(dirname "$0")/testsFailingOutsideWindows.txt"
 }
 
+function load_playlist_tests {
+    # Load the list of tests that are enabled as a part of this test playlist.
+    while IFS='' read -r line || [ -n "$line" ]; do
+        playlistTests[${#playlistTests[@]}]=$line
+    done <"${playlistFile}"
+}
+
 function is_unsupported_test {
     for unsupportedTest in "${unsupportedTests[@]}"; do
         if [ "$1" == "$unsupportedTest" ]; then
@@ -445,6 +455,15 @@ function is_failing_test {
     return 1
 }
 
+function is_playlist_test {
+    for playlistTest in "${playlistTests[@]}"; do
+        if [ "$1" == "$playlistTest" ]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
 function skip_unsupported_test {
     # This function runs in a background process. It should not echo anything, and should not use global variables. This
     # function is analogous to run_test, and causes the test to be skipped with the message below.
@@ -467,6 +486,17 @@ function skip_failing_test {
     return 2 # skip the test
 }
 
+function skip_non_playlist_test {
+    # This function runs in a background process. It should not echo anything, and should not use global variables. This
+    # function is analogous to run_test, and causes the test to be skipped with the message below.
+
+    local scriptFilePath=$1
+    local outputFilePath=$2
+
+    echo "Test is not included in the running playlist." >"$outputFilePath"
+    return 2 # skip the test
+}
+
 function run_test {
     # This function runs in a background process. It should not echo anything, and should not use global variables.
 
@@ -553,10 +583,15 @@ function finish_remaining_tests {
 
 function start_test {
     local scriptFilePath=$1
-
     if ((runFailingTestsOnly == 1)) && ! is_failing_test "$scriptFilePath"; then
         return
     fi
+    
+    # Skip any test that's not in the current playlist, if a playlist was
+    # given to us.
+    if [ -n "$playlistFile" ] && ! is_playlist_test "$scriptFilePath"; then
+        return
+    fi
 
     if ((nextProcessIndex < processCount)); then
         finish_test
@@ -655,6 +690,7 @@ coreClrObjs=
 coreClrSrc=
 coverageOutputDir=
 testEnv=
+playlistFile=
 
 ((disableEventLogging = 0))
 ((serverGC = 0))
@@ -715,6 +751,9 @@ do
         --useServerGC)
             ((serverGC = 1))
             ;;
+        --playlist=*)
+            playlistFile=${i#*=}
+            ;;
         --coreclr-coverage)
             CoreClrCoverage=ON
             ;;
@@ -797,8 +836,18 @@ xunit_output_begin
 create_core_overlay
 precompile_overlay_assemblies
 copy_test_native_bin_to_test_root
-load_unsupported_tests
-load_failing_tests
+
+if [ -n "$playlistFile" ]
+then
+    # Use a playlist file exclusively, if it was provided
+    echo "Executing playlist $playlistFile"
+    load_playlist_tests
+else
+    load_unsupported_tests
+    load_failing_tests
+fi
+
+
 
 scriptPath=$(dirname $0)
 

diff --git a/tests/src/CLRTest.Execute.Bash.targets b/tests/src/CLRTest.Execute.Bash.targets
index 7ec2e1566d2525bb99e93cf31c918bef12fed520..d4bef4728321c1bf0989d8a4e7ad10d9802d0eb8 100644 (file)
--- a/tests/src/CLRTest.Execute.Bash.targets
+++ b/tests/src/CLRTest.Execute.Bash.targets
@@ -165,10 +165,6 @@ then
   exit 0
 fi
       ]]></BashCLRTestEnvironmentCompatibilityCheck>
-      <BashCLRTestGCLongTestSkipCondition Condition="'$(GCLongRunning)' == 'true' And '$(IsLongRunningGCTest)' != 'true'"><![CDATA[
-echo Skipping execution because this is not a long-running GC test
-exit 0
-      ]]></BashCLRTestGCLongTestSkipCondition>
       <BashCLRTestExitCodePrep Condition="$(_CLRTestNeedsToRun)">
 <![CDATA[CLRTestExpectedExitCode=$(CLRTestExitCode)
 echo BEGIN EXECUTION]]>
@@ -316,7 +312,6 @@ $__TestEnv
 
 $(BashEnvironmentVariables)
 $(BashCLRTestEnvironmentCompatibilityCheck)
-$(BashCLRTestGCLongTestSkipCondition)
 $(BashCLRTestArgPrep)
 $(BashCLRTestExitCodePrep)
 # CrossGen Script (when /p:CrossGen=true)