Collect Windows test run logs into testReports.zip file (#21813)

* Collect Windows test run logs into testReports.zip file

And upload these to Jenkins, so we can see how the tests ran.

* Archive test Reports even if the test run fails

* Need to use "call" when invoking runtest.cmd batch file

diff --git a/netci.groovy b/netci.groovy
index b8016e7..906238b 100755 (executable)
--- a/netci.groovy
+++ b/netci.groovy
@@ -2266,7 +2266,18 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                             buildCommands += "tests\\scripts\\run-gc-reliability-framework.cmd ${arch} ${configuration}"
                         }
                         else {
-                            buildCommands += "tests\\runtest.cmd ${runtestArguments}"
+                            def buildCommandsStr = "call tests\\runtest.cmd ${runtestArguments}\r\n"
+                            if (!isBuildOnly) {
+                                // If we ran the tests, collect the test logs collected by xunit. We want to do this even if the tests fail, so we
+                                // must do it in the same batch file as the test run.
+
+                                buildCommandsStr += "set saved_errorlevel=%errorlevel%\r\n"
+                                buildCommandsStr += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}\\Reports', '.\\bin\\tests\\testReports.zip')\"\r\n";
+                                buildCommandsStr += "exit /b %saved_errorlevel%\r\n"
+
+                                Utilities.addArchival(newJob, "bin/tests/testReports.zip", "")
+                            }
+                            buildCommands += buildCommandsStr
                         }
                     } // end if (!isBuildOnly)
 
@@ -2280,11 +2291,11 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR
                             buildCommands += "build.cmd ${lowerConfiguration} arm64 linuxmscorlib"
                         }
 
-                        // Zip up the tests directory so that we don't use so much space/time copying
-                        // 10s of thousands of files around.
-                        buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
-
                         if (!isJitStressScenario(scenario)) {
+                            // Zip up the tests directory so that we don't use so much space/time copying
+                            // 10s of thousands of files around.
+                            buildCommands += "powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${arch}.${configuration}', '.\\bin\\tests\\tests.zip')\"";
+
                             // For Windows, pull full test results and test drops for x86/x64.
                             // No need to pull for stress mode scenarios (downstream builds use the default scenario)
                             Utilities.addArchival(newJob, "bin/Product/**,bin/tests/tests.zip", "bin/Product/**/.nuget/**")
@@ -3131,12 +3142,18 @@ def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture
                 // Run runtest.cmd
                 // Do not run generate layout. It will delete the correct CORE_ROOT, and we do not have a correct product
                 // dir to copy from.
-                def runtestCommand = "%WORKSPACE%\\tests\\runtest.cmd ${architecture} ${configuration} skipgeneratelayout"
+                def runtestCommand = "call %WORKSPACE%\\tests\\runtest.cmd ${architecture} ${configuration} skipgeneratelayout"
 
                 addCommand("${runtestCommand}")
+                addCommand("set saved_errorlevel=%errorlevel%")
+
+                // Collect the test logs collected by xunit. Ignore errors here. We want to collect these even if the run
+                // failed for some reason, so it needs to be in this batch file.
+
+                addCommand("powershell -NoProfile -Command \"Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('.\\bin\\tests\\${osGroup}.${architecture}.${configuration}\\Reports', '.\\bin\\tests\\testReports.zip')\"");
 
-                // Use the smarty errorlevel as the script errorlevel.
-                addCommand("exit /b %errorlevel%")
+                // Use the runtest.cmd errorlevel as the script errorlevel.
+                addCommand("exit /b %saved_errorlevel%")
 
                 batchFile(buildCommands)
             } // non-corefx testing
@@ -3144,6 +3161,7 @@ def static CreateWindowsArmTestJob(def dslFactory, def project, def architecture
     } // job
 
     if (!isCoreFxScenario(scenario)) {
+        Utilities.addArchival(newJob, "bin/tests/testReports.zip", "")
         Utilities.addXUnitDotNETResults(newJob, 'bin/**/TestRun*.xml', true)
     }