Add back superpmi pipeline files (#43323)

* Add back superpmi pipeline files

* add the missing import

* add another missing import

I am not sure how this worked in the past with missing imports.

* remove the workaround for a bug

* Revert "remove the workaround for a bug"

This reverts commit 5c674210cc8f20fda3e73704c7f72d6e168c68d9.

diff --git a/eng/pipelines/coreclr/templates/run-superpmi-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-job.yml
index 703c384..286acbc 100644 (file)
--- a/eng/pipelines/coreclr/templates/run-superpmi-job.yml
+++ b/eng/pipelines/coreclr/templates/run-superpmi-job.yml
@@ -89,7 +89,7 @@ jobs:
       displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }}
 
       # Run superpmi collection in helix
-    - template: /eng/common/templates/steps/superpmi-send-to-helix.yml
+    - template: /eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml
       parameters:
         HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/
         HelixType: 'test/superpmi/$(Kind)/$(_Framework)/$(Architecture)'

diff --git a/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml b/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml
new file mode 100644 (file)
index 0000000..77a25dc
--- /dev/null
+++ b/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml
@@ -0,0 +1,53 @@
+# Please remember to update the documentation if you make changes to these parameters!
+parameters:
+  HelixSource: 'pr/default'              # required -- sources must start with pr/, official/, prodcon/, or agent/
+  HelixType: 'tests/default/'            # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
+  HelixBuild: $(Build.BuildNumber)       # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
+  HelixTargetQueues: ''                  # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues
+  HelixAccessToken: ''                   # required -- access token to make Helix API requests; should be provided by the appropriate variable group
+  HelixPreCommands: ''                   # optional -- commands to run before Helix work item execution
+  HelixPostCommands: ''                  # optional -- commands to run after Helix work item execution
+  WorkItemDirectory: ''                  # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
+  CorrelationPayloadDirectory: ''        # optional -- a directory to zip up and send to Helix as a correlation payload
+  IncludeDotNetCli: false                # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
+  DotNetCliPackageType: ''               # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
+  DotNetCliVersion: ''                   # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
+  EnableXUnitReporter: false             # optional -- true enables XUnit result reporting to Mission Control
+  WaitForWorkItemCompletion: true        # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
+  Creator: ''                            # optional -- if the build is external, use this to specify who is sending the job
+  DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO 
+  condition: succeeded()                 # optional -- condition for step to execute; defaults to succeeded()
+  continueOnError: false                 # optional -- determines whether to continue the build if the step errors; defaults to false
+  BuildConfig: 'checked'                 # optional -- Mostly, superpmi will be run on checked builds
+  LibrariesArtifacts: ''
+  TestsArtifacts: ''
+
+steps:
+- template: /eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml
+  parameters:
+    osGroup: ${{ parameters.osGroup }}
+    sendParams: $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.proj /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(BuildConfig)/SendToHelix.binlog
+    displayName: ${{ parameters.DisplayNamePrefix }}
+    condition: ${{ parameters.condition }}
+    continueOnError: ${{ parameters.continueOnError }}
+    environment:
+      MchFileTag: $(MchFileTag)
+      BuildConfig:  ${{ parameters.BuildConfig }}
+      LibrariesArtifacts: ${{ parameters.LibrariesArtifacts }}
+      TestsArtifacts: ${{ parameters.TestsArtifacts }}
+      HelixSource: ${{ parameters.HelixSource }}
+      HelixType: ${{ parameters.HelixType }}
+      HelixBuild: ${{ parameters.HelixBuild }}
+      HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
+      HelixAccessToken: ${{ parameters.HelixAccessToken }}
+      HelixPreCommands: ${{ parameters.HelixPreCommands }}
+      HelixPostCommands: ${{ parameters.HelixPostCommands }}
+      WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
+      CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
+      IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
+      DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
+      DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
+      EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
+      WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
+      Creator: ${{ parameters.Creator }}
+      SYSTEM_ACCESSTOKEN: $(System.AccessToken)
\ No newline at end of file

diff --git a/src/coreclr/scripts/superpmi-setup.py b/src/coreclr/scripts/superpmi-setup.py
index 73de8cf..db4fca2 100644 (file)
--- a/src/coreclr/scripts/superpmi-setup.py
+++ b/src/coreclr/scripts/superpmi-setup.py
@@ -33,8 +33,10 @@
 ################################################################################
 
 
-import subprocess
 import argparse
+import shutil
+import subprocess
+import tempfile
 
 from os import listdir, path, walk
 from os.path import isfile, join, getsize

diff --git a/src/coreclr/scripts/superpmi.proj b/src/coreclr/scripts/superpmi.proj
new file mode 100644 (file)
index 0000000..75797de
--- /dev/null
+++ b/src/coreclr/scripts/superpmi.proj
@@ -0,0 +1,102 @@
+<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
+
+  <PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
+    <FileSeparatorChar>\</FileSeparatorChar>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
+    <FileSeparatorChar>/</FileSeparatorChar>
+  </PropertyGroup>
+
+    <!-- The directory structure of pmiAssembliesPayload is
+         source\workitem\pmiAssembliesDirectory\<machine_specific_load_0>\binaries
+         source\workitem\pmiAssembliesDirectory\<machine_specific_load_1>\binaries
+         ...
+         source\workitem\pmiAssembliesDirectory\<machine_specific_load_N>\binaries
+
+         PmiAssembliesPayload - Path that will be sent to helix machine to run collection on
+         PmiAssembliesDirectory - Path on helix machine itself where superpmi.py will discover the sent assemblies.
+    -->
+  <PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
+    <Python>%HELIX_PYTHONPATH%</Python>
+    <PmiAssembliesPayload>$(WorkItemDirectory)\pmiAssembliesDirectory</PmiAssembliesPayload>
+    <PmiAssembliesDirectory>%HELIX_WORKITEM_PAYLOAD%\binaries</PmiAssembliesDirectory>
+    <SuperPMIDirectory>%HELIX_CORRELATION_PAYLOAD%\superpmi</SuperPMIDirectory>
+    <OutputMchPath>%HELIX_WORKITEM_UPLOAD_ROOT%</OutputMchPath>
+    <!-- Workaround until https://github.com/dotnet/arcade/pull/6179 is not available -->
+    <HelixResultsDestinationDir>$(BUILD_SOURCESDIRECTORY)\artifacts\helixresults</HelixResultsDestinationDir>
+    <WorkItemCommand>$(SuperPMIDirectory)\superpmi.py collect --pmi -pmi_location $(SuperPMIDirectory)\pmi.dll </WorkItemCommand>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
+    <Python>$HELIX_PYTHONPATH</Python>
+    <PmiAssembliesPayload>$(WorkItemDirectory)/pmiAssembliesDirectory</PmiAssembliesPayload>
+    <PmiAssembliesDirectory>$HELIX_WORKITEM_PAYLOAD/binaries</PmiAssembliesDirectory>
+    <SuperPMIDirectory>$HELIX_CORRELATION_PAYLOAD/superpmi</SuperPMIDirectory>
+    <OutputMchPath>$HELIX_WORKITEM_UPLOAD_ROOT</OutputMchPath>
+    <!-- Workaround until https://github.com/dotnet/arcade/pull/6179 is not available -->
+    <HelixResultsDestinationDir>$(BUILD_SOURCESDIRECTORY)/artifacts/helixresults</HelixResultsDestinationDir>
+    <WorkItemCommand>$(SuperPMIDirectory)/superpmi.py collect --pmi -pmi_location $(SuperPMIDirectory)/pmi.dll </WorkItemCommand>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(WorkItemCommand)' != ''">
+    <WorkItemCommand>$(Python) $(WorkItemCommand) -pmi_assemblies $(PmiAssembliesDirectory) -arch $(Architecture) -build_type $(BuildConfig) -core_root $(SuperPMIDirectory)</WorkItemCommand>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <EnableAzurePipelinesReporter>false</EnableAzurePipelinesReporter>
+    <EnableXUnitReporter>false</EnableXUnitReporter>
+    <WorkItemTimeout>5:00</WorkItemTimeout>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
+      <PayloadDirectory>%(Identity)</PayloadDirectory>
+    </HelixCorrelationPayload>
+  </ItemGroup>
+
+  <ItemGroup>
+    <!-- libraries payload -->
+    <LibraryPartitions Include="$([System.IO.Directory]::GetDirectories($(LibrariesArtifacts)))"/>
+    <Partition Include="@(LibraryPartitions -> '%(Filename)')" PmiAssemblies="Core_Root$(FileSeparatorChar)%(Filename)" OutputMchName="libraries.pmi" PartitionId="%(Filename)" />
+
+    <!-- tests payload -->
+    <!-- TODO: Disable SPMI for P1 tests -->
+    <!-- <TestPartitions Include="$([System.IO.Directory]::GetDirectories($(TestsArtifacts)))"/>
+    <Partition Include="@(TestPartitions -> '%(Filename)')" PmiAssemblies="Tests$(FileSeparatorChar)%(Filename)" OutputMchName="tests.pmi" PartitionId="%(Filename)" /> -->
+  </ItemGroup>
+
+  <ItemGroup>
+    <HelixWorkItem Include="@(Partition)">
+      <OutputFileName>%(HelixWorkItem.OutputMchName).$(MchFileTag).%(HelixWorkItem.PartitionId)</OutputFileName>
+      <PayloadDirectory>$(PmiAssembliesPayload)$(FileSeparatorChar)%(HelixWorkItem.PmiAssemblies)</PayloadDirectory>
+      <Command>$(WorkItemCommand) -output_mch_path $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).mch -log_file $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).log</Command>
+      <Timeout>$(WorkItemTimeout)</Timeout>
+      <DownloadFilesFromResults>%(OutputFileName).mch;%(OutputFileName).mch.mct;%(OutputFileName).log</DownloadFilesFromResults>
+    </HelixWorkItem>
+  </ItemGroup>
+
+    <!--
+    This is useful for local testing to print the produced helix items
+    To use this when you are changing how items are produced, uncomment the target
+    and replace the Project item at the top of the file with this:
+    <Project DefaultTargets="printItems">
+
+    Once you've done that you can run this to see the results:
+    dotnet msbuild .\scenarios.proj /v:n
+   -->
+<!--    
+   <PropertyGroup>
+     <HelixTargetQueues>Some_Queue</HelixTargetQueues>
+     <LibrariesArtifacts>E:\temp\power\test1</LibrariesArtifacts>
+     <TestsArtifacts>E:\temp\power\test1</TestsArtifacts>
+   </PropertyGroup>
+  <Target Name="printItems">
+        <Message Text="@(HelixWorkItem -> 'name: %(HelixWorkItem.Identity)
+     dir: %(HelixWorkItem.PayloadDirectory)
+     pre: %(HelixWorkItem.PreCommands)
+     command: %(HelixWorkItem.Command)
+     post: %(HelixWorkItem.PostCommands)
+     timeout: %(HelixWorkItem.Timeout)  '"/>
+  </Target>
+   -->
+
+</Project>
\ No newline at end of file