From 57246c0812e94e0ad98cb313d2d97ba1a458afb8 Mon Sep 17 00:00:00 2001 From: Sean Gillespie Date: Fri, 11 Mar 2016 15:48:56 -0800 Subject: [PATCH] Create Long-running GC test job for the CI --- netci.groovy | 75 ++++++++- tests/buildtest.cmd | 7 + tests/issues.targets | 3 - tests/runtest.cmd | 18 +- tests/src/CLRTest.Execute.Bash.targets | 5 + tests/src/CLRTest.Execute.Batch.targets | 8 +- .../Coreclr.TestWrapper/CoreclrTestWrapperLib.cs | 8 +- tests/src/GC/Coverage/concurrentspin2.cs | 184 --------------------- tests/src/GC/Coverage/concurrentspin2.csproj | 45 ----- .../Features/BackgroundGC/concurrentspin2.csproj | 1 + .../LOHCompaction/lohcompact_stress.csproj | 1 + .../Features/LOHCompaction/lohcompactapi2.csproj | 1 + .../LOHCompaction/lohcompactscenariorepro.csproj | 1 + .../src/GC/Features/PartialCompaction/eco1.csproj | 1 + .../PartialCompaction/partialcompactiontest.csproj | 1 + .../PartialCompaction/partialcompactionwloh.csproj | 1 + .../sustainedlowlatency_race.csproj | 1 + .../sustainedlowlatency_race_reverse.csproj | 1 + tests/src/GC/Scenarios/BaseFinal/basefinal.csproj | 1 + .../Scenarios/DoublinkList/doublinknoleak.csproj | 1 + tests/tests.targets | 5 +- tests/testsFailingOutsideWindows.txt | 2 - tests/x86_legacy_backend_issues.targets | 3 - 23 files changed, 126 insertions(+), 248 deletions(-) delete mode 100644 tests/src/GC/Coverage/concurrentspin2.cs delete mode 100644 tests/src/GC/Coverage/concurrentspin2.csproj diff --git a/netci.groovy b/netci.groovy index 53295e8..6889de1 100755 --- a/netci.groovy +++ b/netci.groovy @@ -66,7 +66,7 @@ class Constants { 'gcstress0xc_minopts_heapverify1' : ['COMPlus_GCStress' : '0xC', 'COMPlus_JITMinOpts' : '1', 'COMPlus_HeapVerify' : '1'] ] // This is the basic set of scenarios - def static basicScenarios = ['default', 'pri1', 'ilrt', 'r2r', 'pri1r2r', 'gcstress15_pri1r2r'] + def static basicScenarios = ['default', 'pri1', 'ilrt', 'r2r', 'pri1r2r', 'gcstress15_pri1r2r', 'longgc'] // This is the set of configurations def static configurationList = ['Debug', 'Checked', 'Release'] // This is the set of architectures @@ -348,6 +348,9 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os, } } break + case 'longgc': + // long GC jobs are triggered by phrase only. + break case 'ilrt': assert !(os in bidailyCrossList) // ILASM/ILDASM roundtrip one gets a daily build, and only for release @@ -479,6 +482,11 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os, Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*") } break + case 'longgc': + if (configuration == 'Release' || configuration == 'Checked') { + Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*") + } + break case 'minopts': assert (os == 'Windows_NT') || (os in Constants.crossList) Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - MinOpts)", @@ -636,6 +644,11 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os, Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*") } break + case 'longgc': + if (configuration == 'Release' || configuration == 'Checked') { + Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${configuration}\\W+${scenario}.*") + } + break case 'minopts': assert (os == 'Windows_NT') || (os in Constants.crossList) Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Build and Test (Jit - MinOpts)", @@ -816,6 +829,12 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os, Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} GCStress 15 R2R pri1 Build & Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*") } } + } else if (scenario == 'longgc') { + if (os == 'Windows_NT'){ + if (configuration == 'Release' || configuration == 'Checked'){ + Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} Long-Running GC Build & Test", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*") + } + } } break default: @@ -950,6 +969,16 @@ combinedScenarios.each { scenario -> return } break + case 'longgc': + if (os != 'Windows_NT' && !(os in Constants.crossList)) { + return + } + + // only x64 or x86 for now + if (architecture != 'x64' && architecture != 'x86') { + return + } + break case 'default': // Nothing skipped break @@ -1014,6 +1043,9 @@ combinedScenarios.each { scenario -> //Build pri1 R2R tests with GC stress level 15 buildCommands += "build.cmd ${lowerConfiguration} ${architecture} docrossgen skiptests" buildCommands += "set __TestIntermediateDir=int&&tests\\buildtest.cmd ${lowerConfiguration} ${architecture} crossgen Priority 1 gcstresslevel 15" + } else if (scenario == 'longgc') { + buildCommands += "build.cmd ${lowerConfiguration} ${architecture} skiptests" + buildCommands += "set __TestIntermediateDir=int&&tests\\buildtest.cmd ${lowerConfiguration} ${architecture} longgctests" } else { println("Unknown scenario: ${scenario}") @@ -1053,10 +1085,18 @@ combinedScenarios.each { scenario -> } } else if (architecture == 'x64') { - buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} ${crossgenStr}" + if (scenario == 'longgc') { + buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} longgctests sequential Exclude0" + } else { + buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture}" + } } else if (architecture == 'x86') { - buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} ${crossgenStr} Exclude0 x86_legacy_backend_issues.targets" + if (scenario == 'longgc') { + buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} longgctests sequential Exclude0" + } else { + buildCommands += "tests\\runtest.cmd ${lowerConfiguration} ${architecture} Exclude0 x86_legacy_backend_issues.targets" + } } } @@ -1295,6 +1335,12 @@ combinedScenarios.each { scenario -> return } break + case 'longgc': + //Skip configs that aren't Checked or Release (so just Debug, for now) + if (configuration != 'Checked' && configuration != 'Release') { + return + } + break case 'default': // Nothing skipped break @@ -1324,6 +1370,15 @@ combinedScenarios.each { scenario -> } // Enable Server GC for Ubuntu PR builds def serverGCString = '' + + // Whether or not this test run should be run sequentially instead + // of in parallel. Only used for long GC tests. + def sequentialString = '' + + // Whether or not this test run should only run failing test. + // Only used for long GC tests, because they are all ignored in + // normal test runs. + def runFailingString = '' if (os == 'Ubuntu' && isPR){ serverGCString = '--useServerGC' @@ -1334,6 +1389,18 @@ combinedScenarios.each { scenario -> if (scenario == 'r2r' || scenario == 'pri1r2r' || scenario == 'gcstress15_pri1r2r'){ crossgenStr = '--crossgen' } + + if (scenario == 'longgc') { + // Long GC tests behave very poorly when they are not + // the only test running (many of them allocate until OOM). + sequentialString = 'sequential' + + // Long GC tests all exist in the ignore list because + // they can't run during normal test runs. This is not + // particularly pretty but, until we get a more generalized + // mechanism for coming up with test playlists, it works. + runFailingString = '--runFailingTestsOnly' + } def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) { @@ -1412,7 +1479,7 @@ combinedScenarios.each { scenario -> --mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\ --coreFxBinDir=\"\${WORKSPACE}/bin/${osGroup}.AnyCPU.Release\" \\ --coreFxNativeBinDir=\"\${WORKSPACE}/bin/${osGroup}.${architecture}.Release\" \\ - ${testEnvOpt} ${serverGCString} ${crossgenStr}""") + ${testEnvOpt} ${serverGCString} ${crossgenStr} ${sequentialString} ${runFailingString}""") } } diff --git a/tests/buildtest.cmd b/tests/buildtest.cmd index f85941d..8f23e82 100644 --- a/tests/buildtest.cmd +++ b/tests/buildtest.cmd @@ -31,6 +31,7 @@ set __crossgen= set __ILAsmRoundtrip= set __BuildSequential= set __TestPriority= +set __LongGCTests= set __msbuildCleanBuildArgs= set __msbuildExtraArgs= set __verbosity=normal @@ -65,6 +66,7 @@ if /i "%1" == "ilasmroundtrip" (set __ILAsmRoundtrip=true&shift&goto Arg_Lo if /i "%1" == "sequential" (set __BuildSequential=1&shift&goto Arg_Loop) if /i "%1" == "priority" (set __TestPriority=%2&shift&shift&goto Arg_Loop) if /i "%1" == "gcstresslevel" (set __GCStressLevel=%2&shift&shift&goto Arg_Loop) +if /i "%1" == "longgctests" (set __LongGCTests=1&shift&goto Arg_Loop) if /i "%1" == "verbose" (set __verbosity=detailed&shift&goto Arg_Loop) @@ -275,6 +277,11 @@ if %__GCStressLevel% GTR 0 ( set __msbuildManagedBuildArgs=%__msbuildManagedBuildArgs% /p:GCStressLevel=%__GCStressLevel% ) +if defined __LongGCTests ( + echo Building tests with Long GC tests enabled. + set __msbuildManagedBuildArgs=%__msbuildManagedBuildArgs% /p:GCLongRunning=true +) + set __BuildLogRootName=Tests_Managed call :msbuild "%__ProjectFilesDir%\build.proj" %__msbuildManagedBuildArgs% if errorlevel 1 exit /b 1 diff --git a/tests/issues.targets b/tests/issues.targets index 331e721..03545dc 100644 --- a/tests/issues.targets +++ b/tests/issues.targets @@ -319,9 +319,6 @@ 3392 - - 3392 - 3392 diff --git a/tests/runtest.cmd b/tests/runtest.cmd index eeca3a9..d894c8e 100644 --- a/tests/runtest.cmd +++ b/tests/runtest.cmd @@ -26,8 +26,9 @@ set "__LogsDir=%__RootBinDir%\Logs" :: Default __Exclude to issues.targets set __Exclude0=%~dp0\issues.targets -set __BuildSequential= +set __Sequential= set __msbuildExtraArgs= +set __LongGCTests= :Arg_Loop if "%1" == "" goto ArgsDone @@ -53,8 +54,9 @@ if /i "%1" == "SkipWrapperGeneration" (set __SkipWrapperGeneration=true&shift&go if /i "%1" == "Exclude" (set __Exclude=%2&shift&shift&goto Arg_Loop) if /i "%1" == "Exclude0" (set __Exclude0=%2&shift&shift&goto Arg_Loop) if /i "%1" == "TestEnv" (set __TestEnv=%2&shift&shift&goto Arg_Loop) -if /i "%1" == "sequential" (set __BuildSequential=1&shift&goto Arg_Loop) +if /i "%1" == "sequential" (set __Sequential=1&shift&goto Arg_Loop) if /i "%1" == "crossgen" (set __DoCrossgen=1&shift&goto Arg_Loop) +if /i "%1" == "longgctests" (set __LongGCTests=1&shift&goto Arg_Loop) if /i "%1" == "GenerateLayoutOnly" (set __GenerateLayoutOnly=1&set __SkipWrapperGeneration=true&shift&goto Arg_Loop) if /i "%1" == "PerfTests" (set __PerfTests=true&set __SkipWrapperGeneration=true&shift&goto Arg_Loop) @@ -125,8 +127,10 @@ if not defined VSINSTALLDIR ( :: assembly. set __msbuildCommonArgs=/nologo /nodeReuse:false %__msbuildExtraArgs% -if not defined __BuildSequential ( +if not defined __Sequential ( set __msbuildCommonArgs=%__msbuildCommonArgs% /maxcpucount +) else ( + set __msbuildCommonArgs=%__msbuildCommonArgs% /p:ParallelRun=false ) if defined CORE_ROOT goto SkipCoreRootSetup @@ -309,6 +313,14 @@ if "%__IsGCTest%"=="true" ( ) ) +:: Long GC tests take about 10 minutes per test on average, so +:: they often bump up against the default 10 minute timeout. +:: 20 minutes is more than enough time for a test to complete successfully. +if defined __LongGCTests ( + echo Running Long GC tests, extending timeout to 20 minutes + set __TestTimeout=1200000 +) + set __BuildLogRootName=Tests_GenerateRuntimeLayout call :msbuild "%__ProjectFilesDir%\runtest.proj" /p:GenerateRuntimeLayout=true if errorlevel 1 ( diff --git a/tests/src/CLRTest.Execute.Bash.targets b/tests/src/CLRTest.Execute.Bash.targets index d4bef47..7ec2e15 100644 --- a/tests/src/CLRTest.Execute.Bash.targets +++ b/tests/src/CLRTest.Execute.Bash.targets @@ -165,6 +165,10 @@ then exit 0 fi ]]> + @@ -312,6 +316,7 @@ $__TestEnv $(BashEnvironmentVariables) $(BashCLRTestEnvironmentCompatibilityCheck) +$(BashCLRTestGCLongTestSkipCondition) $(BashCLRTestArgPrep) $(BashCLRTestExitCodePrep) # CrossGen Script (when /p:CrossGen=true) diff --git a/tests/src/CLRTest.Execute.Batch.targets b/tests/src/CLRTest.Execute.Batch.targets index 12820ac..7fbe32c 100644 --- a/tests/src/CLRTest.Execute.Batch.targets +++ b/tests/src/CLRTest.Execute.Batch.targets @@ -153,7 +153,10 @@ IF "%COMPlus_JitStress%"=="" IF "%COMPlus_JitStressRegs%"=="" IF "%COMPlus_JITMi Exit /b 0 :Compatible1 ]]> - + "); - } - - public static int[] ParseArgs(string[] args) - { - int[] parameters = new int[2]; - - // set defaults - parameters[0] = 100; - parameters[1] = 4; - - if (args.Length == 0) - { - //use defaults - Console.WriteLine("Using defaults: 100 iterations, 4 threads"); - return parameters; - } - if (args.Length == parameters.Length) - { - for (int i = 0; i < args.Length; i++) - { - int j = 0; - if (!int.TryParse(args[i], out j)) - { - Usage(); - return null; - } - parameters[i] = j; - } - - return parameters; - } - - // incorrect number of arguments - Usage(); - return null; - } - - - public static int Main(string[] args) - { - - // parse arguments - int[] parameters = ParseArgs(args); - if (parameters == null) - { - return 0; - } - - // set process affinity to 1 to repro bug easier - //Process.GetCurrentProcess().ProcessorAffinity = (IntPtr)1; - - - PriorityTest priorityTest = new PriorityTest(1000000, 5000, parameters[0], 17, 30, 3); - ThreadStart startDelegate = new ThreadStart(priorityTest.RunTest); - - // create threads - Thread[] threads = new Thread[parameters[1]]; - for (int i = 0; i < threads.Length; i++) - { - threads[i] = new Thread(startDelegate); - threads[i].Name = String.Format("Thread{0}", i); - //if (i % 2 == 0) - //{ - // threads[i].Priority = ThreadPriority.Lowest; - //} - threads[i].Start(); - } - - // wait for threads to complete - for (int i = 0; i < threads.Length; i++) - { - threads[i].Join(); - } - - return 100; - } -} - - diff --git a/tests/src/GC/Coverage/concurrentspin2.csproj b/tests/src/GC/Coverage/concurrentspin2.csproj deleted file mode 100644 index 7e63aff..0000000 --- a/tests/src/GC/Coverage/concurrentspin2.csproj +++ /dev/null @@ -1,45 +0,0 @@ - - - - - Debug - AnyCPU - 2.0 - {95DFC527-4DC1-495E-97D7-E94EE1F7140D} - Exe - Properties - 512 - {786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} - $(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages - ..\..\ - 7a9bfb7d - - - - - - - - - False - - - - - - - - - - - - $(GCPackagesConfigFileDirectory)minimal\project.json - $(GCPackagesConfigFileDirectory)minimal\project.lock.json - - - - - - - - \ No newline at end of file diff --git a/tests/src/GC/Features/BackgroundGC/concurrentspin2.csproj b/tests/src/GC/Features/BackgroundGC/concurrentspin2.csproj index 5e585cf..3158d18 100644 --- a/tests/src/GC/Features/BackgroundGC/concurrentspin2.csproj +++ b/tests/src/GC/Features/BackgroundGC/concurrentspin2.csproj @@ -13,6 +13,7 @@ $(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages ..\..\ 7a9bfb7d + true diff --git a/tests/src/GC/Features/LOHCompaction/lohcompact_stress.csproj b/tests/src/GC/Features/LOHCompaction/lohcompact_stress.csproj index f3adce1..3da854d 100644 --- a/tests/src/GC/Features/LOHCompaction/lohcompact_stress.csproj +++ b/tests/src/GC/Features/LOHCompaction/lohcompact_stress.csproj @@ -13,6 +13,7 @@ $(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages ..\..\ 7a9bfb7d + true diff --git a/tests/src/GC/Features/LOHCompaction/lohcompactapi2.csproj b/tests/src/GC/Features/LOHCompaction/lohcompactapi2.csproj index a360930..9dc2e8a 100644 --- a/tests/src/GC/Features/LOHCompaction/lohcompactapi2.csproj +++ b/tests/src/GC/Features/LOHCompaction/lohcompactapi2.csproj @@ -13,6 +13,7 @@ $(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages ..\..\ 7a9bfb7d + true diff --git a/tests/src/GC/Features/LOHCompaction/lohcompactscenariorepro.csproj b/tests/src/GC/Features/LOHCompaction/lohcompactscenariorepro.csproj index e4962bb..6e0f5e7 100644 --- a/tests/src/GC/Features/LOHCompaction/lohcompactscenariorepro.csproj +++ b/tests/src/GC/Features/LOHCompaction/lohcompactscenariorepro.csproj @@ -13,6 +13,7 @@ $(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages ..\..\ 7a9bfb7d + true diff --git a/tests/src/GC/Features/PartialCompaction/eco1.csproj b/tests/src/GC/Features/PartialCompaction/eco1.csproj index c6021a3..74314e8 100644 --- a/tests/src/GC/Features/PartialCompaction/eco1.csproj +++ b/tests/src/GC/Features/PartialCompaction/eco1.csproj @@ -13,6 +13,7 @@ $(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages ..\..\ 7a9bfb7d + true diff --git a/tests/src/GC/Features/PartialCompaction/partialcompactiontest.csproj b/tests/src/GC/Features/PartialCompaction/partialcompactiontest.csproj index 8ccc632..ec9e594 100644 --- a/tests/src/GC/Features/PartialCompaction/partialcompactiontest.csproj +++ b/tests/src/GC/Features/PartialCompaction/partialcompactiontest.csproj @@ -13,6 +13,7 @@ $(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages ..\..\ 7a9bfb7d + true diff --git a/tests/src/GC/Features/PartialCompaction/partialcompactionwloh.csproj b/tests/src/GC/Features/PartialCompaction/partialcompactionwloh.csproj index 905da65..9c2b2dd 100644 --- a/tests/src/GC/Features/PartialCompaction/partialcompactionwloh.csproj +++ b/tests/src/GC/Features/PartialCompaction/partialcompactionwloh.csproj @@ -13,6 +13,7 @@ $(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages ..\..\ 7a9bfb7d + true diff --git a/tests/src/GC/Features/SustainedLowLatency/sustainedlowlatency_race.csproj b/tests/src/GC/Features/SustainedLowLatency/sustainedlowlatency_race.csproj index 7048a0c..92728a0 100644 --- a/tests/src/GC/Features/SustainedLowLatency/sustainedlowlatency_race.csproj +++ b/tests/src/GC/Features/SustainedLowLatency/sustainedlowlatency_race.csproj @@ -13,6 +13,7 @@ $(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages ..\..\ 7a9bfb7d + true diff --git a/tests/src/GC/Features/SustainedLowLatency/sustainedlowlatency_race_reverse.csproj b/tests/src/GC/Features/SustainedLowLatency/sustainedlowlatency_race_reverse.csproj index 868ee9c..b9843ce 100644 --- a/tests/src/GC/Features/SustainedLowLatency/sustainedlowlatency_race_reverse.csproj +++ b/tests/src/GC/Features/SustainedLowLatency/sustainedlowlatency_race_reverse.csproj @@ -13,6 +13,7 @@ $(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages ..\..\ 7a9bfb7d + true diff --git a/tests/src/GC/Scenarios/BaseFinal/basefinal.csproj b/tests/src/GC/Scenarios/BaseFinal/basefinal.csproj index a6b0e5d..a11454a 100644 --- a/tests/src/GC/Scenarios/BaseFinal/basefinal.csproj +++ b/tests/src/GC/Scenarios/BaseFinal/basefinal.csproj @@ -13,6 +13,7 @@ $(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages ..\..\ 7a9bfb7d + true diff --git a/tests/src/GC/Scenarios/DoublinkList/doublinknoleak.csproj b/tests/src/GC/Scenarios/DoublinkList/doublinknoleak.csproj index 5742139..5ac2e52 100644 --- a/tests/src/GC/Scenarios/DoublinkList/doublinknoleak.csproj +++ b/tests/src/GC/Scenarios/DoublinkList/doublinknoleak.csproj @@ -13,6 +13,7 @@ $(ProgramFiles)\Common Files\microsoft shared\VSTT\11.0\UITestExtensionPackages ..\..\ 7a9bfb7d + true diff --git a/tests/tests.targets b/tests/tests.targets index 79b60bd..56c6c5d 100644 --- a/tests/tests.targets +++ b/tests/tests.targets @@ -37,6 +37,7 @@ category=outerloop;category=failing + true @@ -44,8 +45,8 @@ 3392 - - 3392 - 3392 -- 2.7.4