Increase test scenario invocation parallelism (#80409)

The MSBuild code that invokes Helix to execute coreclr and libraries tests
spawns an MSBuild task for each "configuration" (which is a particular set of
environment variables, such as a set of JIT stress modes), and each of those
MSBuild invocations creates a (large) set of HelixWorkItem that are submitted
to Helix for running for that particular configuration. (One HelixWorkItem
is created for each partition of the test tree). But the MSBuild task also
waits for all the work items to finish before letting another configuration
start. The MSBuild invocations are run in parallel, but only to the level
of parallelism allowed by MSBuild settings. MSBuild is invoked with "/maxcpucount",
which means one MSBuild "node" per processor on the machine, thus one
configuration in parallel per processor. The AzDO machines we run on appear
to have 3 processors allocated to them (at least in some cases). Many test configurations
have far more than 3 configurations (e.g., jitstress-isas-x86 has 55 configurations),
so that's a lot of waiting.

These MSBuild tasks are doing very little: creating HelixWorkItems, invoking Helix,
and waiting. We can run many, many more MSBuild tasks in parallel,
to "flood" Helix with more work that could be done in parallel.

This change passes a larger `/maxcpucount` argument when invoking MSBuild in
cases where this would be beneficial.

(Note that ideally we wouldn't depend on MSBuild parallelism, and a single MSBuild
instance, or other code not written in MSBuild, could create and wait for all the
needed HelixWorkItems for all scenarios in a single process. But that's not how
it's currently architected.)

diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml
index d3e3454..6b23801 100644 (file)
--- a/eng/pipelines/common/templates/runtimes/run-test-job.yml
+++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml
@@ -433,6 +433,17 @@ jobs:
 
         helixProjectArguments: '$(Build.SourcesDirectory)/src/tests/Common/helixpublishwitharcade.proj'
 
+        # helixpublishwitharcade.proj processes one scenario per parallel MSBuild invocation. Each invocation only
+        # creates Helix work items and them waits for their completion on the remote Helix machines, so is not
+        # computationally intensive. We want Helix to be provided with all the possible work items in up front,
+        # so can do as much work in parallel as possible. Thus, increase the amount of allowed MSBuild parallelism
+        # to at least the maximum number of scenarios to be processed in a testGroup.
+        #
+        # We only need to do this for a testGroup with more than one or two scenarios, though it isn't functionally
+        # a problem to always do it. Thus, we only exempt this setting for a few cases.
+        ${{ if notIn(parameters.testGroup, 'innerloop', 'outerloop', 'clrinterpreter', 'ilasm', 'gc-longrunning', 'gc-simulator', 'gc-standalone', 'gc-standalone-server') }}:
+          msbuildParallelism: '/maxcpucount:55'
+
         ${{ if in(parameters.testGroup, 'innerloop', 'outerloop') }}:
           ${{ if eq(parameters.runtimeFlavor, 'mono') }}:
             # tiered compilation isn't done on mono yet

diff --git a/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml b/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml
index dfad8c5..5467dd4 100644 (file)
--- a/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml
+++ b/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml
@@ -12,6 +12,7 @@ parameters:
   helixSource: ''
   helixQueues: ''
   helixType: ''
+  msbuildParallelism: '/maxcpucount'
   scenarios: ''
   timeoutPerTestCollectionInMinutes: ''
   timeoutPerTestInMinutes: ''
@@ -34,7 +35,7 @@ steps:
   parameters:
     osGroup: ${{ parameters.osGroup }}
     restoreParams: /p:DotNetPublishToBlobFeed=true -restore -projects $(Build.SourcesDirectory)$(dir)eng$(dir)empty.csproj
-    sendParams: ${{ parameters.helixProjectArguments }} /maxcpucount /bl:$(Build.SourcesDirectory)/artifacts/log/SendToHelix.binlog /p:TargetArchitecture=${{ parameters.archType }} /p:TargetOS=${{ parameters.osGroup }} /p:TargetOSSubgroup=${{ parameters.osSubgroup }} /p:Configuration=${{ parameters.buildConfig }}
+    sendParams: ${{ parameters.helixProjectArguments }} ${{ parameters.msbuildParallelism }} /bl:$(Build.SourcesDirectory)/artifacts/log/SendToHelix.binlog /p:TargetArchitecture=${{ parameters.archType }} /p:TargetOS=${{ parameters.osGroup }} /p:TargetOSSubgroup=${{ parameters.osSubgroup }} /p:Configuration=${{ parameters.buildConfig }}
     condition: and(succeeded(), ${{ parameters.condition }})
     shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
     displayName: ${{ parameters.displayName }}

diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml
index 3b7efa0..bd4a1e2 100644 (file)
--- a/eng/pipelines/libraries/run-test-job.yml
+++ b/eng/pipelines/libraries/run-test-job.yml
@@ -109,7 +109,6 @@ jobs:
             shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
             creator: dotnet-bot
             testRunNamePrefixSuffix: $(_testRunNamePrefixSuffix)
-            extraHelixArguments: $(_extraHelixArguments)
 
             # coreclrTestGroup: The following mappings of 'coreclrTestGroup' to 'scenarios' is copied from
             # eng/pipelines/common/templates/runtimes/run-test-job.yml (with 'testGroup' replaced by 'coreclrTestGroup'
@@ -123,6 +122,17 @@ jobs:
             # is run in a regular CI job, so there is no need to duplicate it here. So, add 'no_tiered_compilation'
             # to the 'jitstress' job instead of adding a new job just for 'no_tiered_compilation'.
 
+            # src/libraries/sendtohelix.proj processes one scenario per parallel MSBuild invocation. Each invocation only
+            # creates Helix work items and them waits for their completion on the remote Helix machines, so is not
+            # computationally intensive. We want Helix to be provided with all the possible work items in up front,
+            # so can do as much work in parallel as possible. Thus, increase the amount of allowed MSBuild parallelism
+            # to at least the maximum number of scenarios to be processed in a coreclrTestGroup. If there is no
+            # coreclrTestGroup then there is only one scenario (the default scenario), so don't change the MSBuild argument.
+            ${{ if ne(parameters.coreclrTestGroup, '') }}:
+              extraHelixArguments: $(_extraHelixArguments) /maxcpucount:10
+            ${{ else }}:
+              extraHelixArguments: $(_extraHelixArguments)
+
             ${{ if in(parameters.coreclrTestGroup, 'jitstress') }}:
               scenarios:
               - no_tiered_compilation