Use helix to run package tests (dotnet/corefx#34764)

* Use helix to run package tests

* PR Feedback and fix typo in helix.yml

* Define RestoreSources in packageTest.targets

* Remove restore sources item group and move the property to packageTest.targets

* Use less busy queue, change sendtohelix conditions to use targetgroup

* Move HelixCommand assignment out of target now that is not needed

* Disable azure pipelines reporter for all configurations test results

* PR Feedback and use dotnet blob feed as restore source

Commit migrated from https://github.com/dotnet/corefx/commit/ebfa30454ba9a1dcca6293166cbe486175ec17f1

diff --git a/eng/pipelines/libraries/corefx-base.yml b/eng/pipelines/libraries/corefx-base.yml
index b572853..5bb2ddc 100644 (file)
--- a/eng/pipelines/libraries/corefx-base.yml
+++ b/eng/pipelines/libraries/corefx-base.yml
@@ -166,12 +166,16 @@ jobs:
                 helixQueues: $(_helixQueues)
                 msbuildScript: $(_msbuildCommand)
                 framework: $(_framework)
+
                 ${{ if eq(parameters.isOfficialBuild, 'true') }}:
                   isExternal: false
                   waitForCompletion: false
                   officialBuildId: $(Build.BuildNumber)
                   helixToken: $(HelixApiAccessToken)
-                  enableAzurePipelinesReporter: true
+                  ${{ if ne(job.enableAzurePipelinesReporter, '') }}:
+                    enableAzurePipelinesReporter: ${{ job.enableAzurePipelinesReporter }}
+                  ${{ if eq(job.enableAzurePipelinesReporter, '') }}:
+                    enableAzurePipelinesReporter: true
 
                 ${{ if eq(parameters.isOfficialBuild, 'false') }}:
                   # TODO: SET Creator to the PR owner whenever Azure DevOps supports a good way to retrieve it.

diff --git a/eng/pipelines/libraries/windows.yml b/eng/pipelines/libraries/windows.yml
index fb21b8f..ab03869 100644 (file)
--- a/eng/pipelines/libraries/windows.yml
+++ b/eng/pipelines/libraries/windows.yml
@@ -124,6 +124,7 @@ jobs:
               _BuildConfig: Debug
               _architecture: x64
               _framework: allConfigurations
+              _helixQueues: $(allConfigurationsQueues)
 
           # Official Build Matrix
           ${{ if eq(parameters.isOfficialBuild, 'true') }}:
@@ -131,16 +132,25 @@ jobs:
               _BuildConfig: Release
               _architecture: x64
               _framework: allConfigurations
+              _helixQueues: $(allConfigurationsQueues)
 
       pool:
         ${{ if eq(parameters.isOfficialBuild, 'true') }}:
           name: dotnet-internal-temp
         ${{ if eq(parameters.isOfficialBuild, 'false') }}:
-          name: dotnet-external-temp
+          name: Hosted VS2017
 
-      submitToHelix: false
+      submitToHelix: true
+      # azure pipelines reporter only supports xunit results based tests.
+      enableAzurePipelinesReporter: false
       enableMicrobuild: ${{ parameters.isOfficialBuild }}
 
+      variables:
+        - ${{ if eq(parameters.isOfficialBuild, 'false') }}:
+          - allConfigurationsQueues: Windows.10.Amd64.ClientRS2.Open
+        - ${{ if eq(parameters.isOfficialBuild, 'true') }}:
+          - allConfigurationsQueues: Windows.10.Amd64.ClientRS2
+
       customBuildSteps:
         - script: build.cmd
                   -ci
@@ -151,6 +161,14 @@ jobs:
                   $(_windowsOfficialBuildArguments)
                   $(_msbuildCommonParameters)
           displayName: Build Packages
+        - script: build.cmd
+                  -ci
+                  -test
+                  -$(_framework)
+                  /p:ArchGroup=$(_architecture)
+                  /p:ConfigurationGroup=$(_BuildConfig)
+                  /p:ArchiveTests=true
+          displayName: Build test projects
         - ${{ if eq(parameters.isOfficialBuild, 'false') }}:
           - script: build.cmd
                     -ci
@@ -160,13 +178,6 @@ jobs:
                     /p:ConfigurationGroup=$(_BuildConfig)
                     /p:SkipTests=true
             displayName: Build Netstandard Test Suite
-          - script: build.cmd
-                    -ci
-                    -test
-                    -$(_framework)
-                    /p:ArchGroup=$(_architecture)
-                    /p:ConfigurationGroup=$(_BuildConfig)
-            displayName: Run Package Tests
 
     # TODO: UAPAOT official builds should send to helix using continuation runner.
     # Legs without HELIX testing

diff --git a/eng/sendtohelix.proj b/eng/sendtohelix.proj
index b6821aa..925c3af 100644 (file)
--- a/eng/sendtohelix.proj
+++ b/eng/sendtohelix.proj
@@ -13,10 +13,12 @@
     <TimeoutInSeconds Condition="'$(TimeoutInSeconds)' == ''">600</TimeoutInSeconds>
     <_timeoutSpan>$([System.TimeSpan]::FromSeconds($(TimeoutInSeconds)))</_timeoutSpan>
     
-    <!-- We need to enable xunit reporter so that it parses test results -->
-    <EnableXunitReporter>true</EnableXunitReporter>
+    <!-- We need to enable xunit reporter so that it parses test results
+         Package testing doesn't run on xunit. -->
+    <EnableXunitReporter Condition="'$(TargetGroup)' != 'AllConfigurations'">true</EnableXunitReporter>
     
     <TestArchiveRuntimeFile>$(TestArchiveRuntimeRoot)test-runtime-$(BuildConfiguration).zip</TestArchiveRuntimeFile>
+    <TestArchiveRuntimeFile Condition="'$(TargetGroup)' == 'AllConfigurations'">$(TestArchiveRuntimeRoot)packages-testPayload-$(ConfigurationGroup).zip</TestArchiveRuntimeFile>
 
     <!-- The helix runtime payload and the tests to run -->
     <HelixCorrelationPayload Condition="'$(HelixCorrelationPayload)' == ''">$(TestArchiveRuntimeFile)</HelixCorrelationPayload>
@@ -28,6 +30,7 @@
     <!-- This property is used to show the tests results in Azure Dev Ops. By setting this property the
     test run name will be displayed as $(BuildConfiguration)-$(HelixTargetQueue) -->
     <TestRunNamePrefix>$(BuildConfiguration)-</TestRunNamePrefix>
+    <TestRunNamePrefix Condition="'$(IsPackageTesting)' == 'true'">PackageTests-$(ConfigurationGroup)-$(ArchGroup)</TestRunNamePrefix>
 
     <MaxRetryCount Condition="'$(MaxRetryCount)' == ''">4</MaxRetryCount>
   </PropertyGroup>
@@ -45,13 +48,24 @@
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(HelixType)' == ''">
-    <!-- For PRs we want helixtype to be the same for all frameworks -->
+    <!-- For PRs we want helixtype to be the same for all frameworks except package testing-->
     <HelixType>test/functional/cli/</HelixType>
+    <HelixType Condition="'$(TargetGroup)' == 'AllConfigurations'">test/functional/packaging/</HelixType>
     <HelixType Condition="'$(TargetGroup)' == 'netfx' AND '$(OfficialBuildId)' != ''">test/functional/desktop/cli/</HelixType>
     <HelixType Condition="'$(TargetGroup)' == 'uap' AND '$(OfficialBuildId)' != ''">test/functional/uwp/</HelixType>
     <HelixType Condition="'$(TargetGroup)' == 'uapaot' AND '$(OfficialBuildId)' != ''">test/functional/ilc/</HelixType>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(HelixCommand)' == '' AND '$(TargetGroup)' == 'AllConfigurations'">
+    <HelixPreCommands>set DOTNET_CLI_TELEMETRY_OPTOUT=1;set DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1;set DOTNET_MULTILEVEL_LOOKUP=0</HelixPreCommands>
+
+    <HelixCommand>%HELIX_CORRELATION_PAYLOAD%\tools\dotnet.exe msbuild %HELIX_CORRELATION_PAYLOAD%\test.msbuild</HelixCommand>
+    <HelixCommand>$(HelixCommand) /p:PackageTestProjectsDir=%HELIX_WORKITEM_PAYLOAD%</HelixCommand>
+    <HelixCommand>$(HelixCommand) /p:RestorePackagesPath=%HELIX_WORKITEM_PAYLOAD%\packages</HelixCommand>
+    <HelixCommand>$(HelixCommand) /p:LocalPackagesPath="%HELIX_CORRELATION_PAYLOAD%\packages\</HelixCommand>
+  </PropertyGroup>
+
+
   <PropertyGroup Condition="'$(HelixCommand)' == ''">
     <!-- For windows we need to use call, since the command is going to be called from a bat script created by Helix
     and we exit /b at the end of RunTests.cmd, Helix runs some other commands after ours within the bat script,
@@ -68,7 +82,8 @@
 
   <Target Name="CompressRuntimeDirectory"
           Inputs="@(_RuntimeInputs)"
-          Outputs="$(TestArchiveRuntimeFile)">
+          Outputs="$(TestArchiveRuntimeFile)"
+          Condition="'$(TargetGroup)' != 'AllConfigurations'">
 
     <MakeDir Directories="$(TestArchiveRuntimeRoot)" />
     <ZipDirectory
@@ -78,7 +93,8 @@
 
   </Target>
 
-  <Target Name="BuildHelixWorkItems" DependsOnTargets="CompressRuntimeDirectory">
+  <Target Name="BuildHelixWorkItems"
+          DependsOnTargets="CompressRuntimeDirectory">
     <ItemGroup>
 
       <HelixCorrelationPayload Include="$(HelixCorrelationPayload)" />

diff --git a/src/libraries/pkg/test/frameworkSettings/net/settings.targets b/src/libraries/pkg/test/frameworkSettings/net/settings.targets
index e6467ef..a92c2e2 100644 (file)
--- a/src/libraries/pkg/test/frameworkSettings/net/settings.targets
+++ b/src/libraries/pkg/test/frameworkSettings/net/settings.targets
@@ -7,6 +7,10 @@
   </PropertyGroup>
   
   <ItemGroup>
+    <!-- For full framework we reference these package which has the same layout as the SDK, with this package,
+    tests can run in any machine without having to install full framework. -->
+    <PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.0-alpha-5" />
+
     <!-- these are part of the targeting pack but not actual assemblies -->
     <ExcludeReference Include="System.EnterpriseServices.Thunk;System.EnterpriseServices.Wrapper" />
     

diff --git a/src/libraries/pkg/test/packageTest.targets b/src/libraries/pkg/test/packageTest.targets
index 1329bb9..96e646a 100644 (file)
--- a/src/libraries/pkg/test/packageTest.targets
+++ b/src/libraries/pkg/test/packageTest.targets
@@ -5,6 +5,13 @@
     <_targetFrameworkVersionIndex>$(TargetFramework.IndexOfAny(".-0123456789"))</_targetFrameworkVersionIndex>
     <_targetFrameworkIdentifier Condition="'$(_runtimeOSVersionIndex)' != '-1'">$(TargetFramework.SubString(0, $(_targetFrameworkVersionIndex)))</_targetFrameworkIdentifier>
     <ToolsDir>$(MSBuildThisFileDirectory)tools\</ToolsDir>
+
+    <RestoreSources>
+      https://api.nuget.org/v3/index.json;
+      https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json;
+      https://dotnet.myget.org/F/roslyn-tools/api/v3/index.json;
+      $(LocalPackagesPath)
+    </RestoreSources>
   </PropertyGroup>
 
   <Import Project="tools\Packaging.common.targets" />

diff --git a/src/libraries/pkg/test/project.csproj.template b/src/libraries/pkg/test/project.csproj.template
index 20afcca..d78e153 100644 (file)
--- a/src/libraries/pkg/test/project.csproj.template
+++ b/src/libraries/pkg/test/project.csproj.template
@@ -5,11 +5,13 @@
     <RuntimeIdentifiers>{RIDs}</RuntimeIdentifiers>
     <TestPackageId>{PackageId}</TestPackageId>
     <TestPackageVersion>{PackageVersion}</TestPackageVersion>
+
+    <SupportFilesDir Condition="'$(SupportFilesDir)' == ''">$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), Directory.Build.props))\support</SupportFilesDir>
   </PropertyGroup>
   
   <ItemGroup>
     <PackageReference Include="{PackageId}" Version="{PackageVersion}" />
   </ItemGroup>
 
-  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), packageTest.targets))\packageTest.targets" />
+  <Import Project="$(SupportFilesDir)\packageTest.targets" />
 </Project>

diff --git a/src/libraries/pkg/test/test.msbuild b/src/libraries/pkg/test/test.msbuild
index 6c09548..aa3cb05 100644 (file)
--- a/src/libraries/pkg/test/test.msbuild
+++ b/src/libraries/pkg/test/test.msbuild
@@ -4,10 +4,15 @@
     <TestPackages Condition="'$(TestPackages)' != ''" Include="$(TestPackages)" />
   </ItemGroup>
 
+  <PropertyGroup>
+    <PackageTestProjectsDir Condition="'$(PackageTestProjectsDir)' == ''">$(MSBuildThisFileDirectory)..\projects</PackageTestProjectsDir>
+    <_projectProperties>SupportFilesDir=$(MSBuildThisFileDirectory)</_projectProperties>
+  </PropertyGroup>
+
   <Target Name="GetProjects">
     <ItemGroup>
-      <Project Condition="'@(TestPackages)' == ''" Include="projects\**\*.csproj" />
-      <Project Condition="'@(TestPackages)' != ''" Include="projects\%(TestPackages.Identity)\**\*.csproj" />
+      <Project Condition="'@(TestPackages)' == ''" Include="$(PackageTestProjectsDir)\**\*.csproj" />
+      <Project Condition="'@(TestPackages)' != ''" Include="$(PackageTestProjectsDir)\%(TestPackages.Identity)\**\*.csproj" />
     </ItemGroup>
   </Target>
 
@@ -15,11 +20,13 @@
           Inputs="@(Project)"
           Outputs="@(Project->'%(RootDir)%(Directory)obj\project.assets.json')"
           DependsOnTargets="GetProjects">
-    <MSBuild Projects="@(Project)" Targets="Restore" BuildInParallel="true" />
+    <MSBuild Projects="@(Project)" Targets="Restore" BuildInParallel="true" Properties="$(_projectProperties)" />
     <Touch Files="@(Project->'%(RootDir)%(Directory)obj\project.assets.json')" />
   </Target>
 
   <Target Name="Test" DependsOnTargets="GetProjects">
-    <MSBuild Projects="@(Project)" Targets="Test" BuildInParallel="true" />
+    <!-- We need to pass a dummy global property to force msbuild reevaluation after restoring the project to get implicit references added
+    and all generated props and targets imported in the test step. This is used to be able to call RestoreAndTest target when running in helix. -->
+    <MSBuild Projects="@(Project)" Targets="Test" BuildInParallel="true" Properties="$(_projectProperties);ForceMsbuildEvaluation=true" />
   </Target>
 </Project>
\ No newline at end of file

diff --git a/src/libraries/pkg/test/testPackages.proj b/src/libraries/pkg/test/testPackages.proj
index d9960ae..35558f8 100644 (file)
--- a/src/libraries/pkg/test/testPackages.proj
+++ b/src/libraries/pkg/test/testPackages.proj
@@ -14,14 +14,6 @@
     <RuntimesToInclude Condition="'$(RuntimesToInclude)' != ''" Include="$(RuntimesToInclude)" />
     <TargetFrameworksToInclude Condition="'$(TargetFrameworksToInclude)' != ''" Include="$(TargetFrameworksToInclude)" />
 
-    <RestoreSource Include="https:%2F%2Fapi.nuget.org/v3/index.json" />
-
-    <!-- needed for SQLClient's SNI and core-setup packages -->
-    <RestoreSource Include="https:%2F%2Fdotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
-
-    <!-- ideally this would be first, but we need to list last to workaround https://github.com/NuGet/Home/issues/6678 -->
-    <RestoreSource Include="$(PackageOutputPath)" />
-
     <!-- we don't build alpine or older OSX versions -->
     <RuntimesToExclude Include="alpine.3.4.3-x64;osx.10.10-x64;osx.10.11-x64" />
 
@@ -37,25 +29,15 @@
 
   <PropertyGroup>
     <TestDir>$(BinDir)testPkg/</TestDir>
+    <TestSupportDir>$(TestDir)support/</TestSupportDir>
     <TestProjectName>test.msbuild</TestProjectName>
-    <TestProject>$(TestDir)$(TestProjectName)</TestProject>
-    <TestToolsDir>$(TestDir)tools/</TestToolsDir>
+    <TestProject>$(TestSupportDir)$(TestProjectName)</TestProject>
+    <TestToolsDir>$(TestSupportDir)tools/</TestToolsDir>
     <TestProjectDir>$(TestDir)projects/</TestProjectDir>
     <TestPackageDir>$(BinDir)testPackages</TestPackageDir>
     <TestDotNetPath>$(TestToolsDir)/dotnet</TestDotNetPath>
 
     <ProjectTemplate>project.csproj.template</ProjectTemplate>
-
-    <TestRestoreCommand>$(TestDotNetPath)</TestRestoreCommand>
-    <TestRestoreCommand>$(TestRestoreCommand) restore</TestRestoreCommand>
-    <TestRestoreCommand>$(TestRestoreCommand) @(RestoreSource->'-s %(Identity)', ' ')</TestRestoreCommand>
-    <TestRestoreCommand>$(TestRestoreCommand) --packages "$(TestPackageDir)"</TestRestoreCommand>
-    <TestRestoreCommand  Condition="'$(TestPackages)' != ''">$(TestRestoreCommand) /p:TestPackages=$(TestPackages)</TestRestoreCommand>
-
-    <TestBuildCommand>$(TestDotNetPath)</TestBuildCommand>
-    <TestBuildCommand>$(TestBuildCommand) msbuild</TestBuildCommand>
-    <TestBuildCommand>$(TestBuildCommand) /t:Test</TestBuildCommand>
-    <TestBuildCommand  Condition="'$(TestPackages)' != ''">$(TestBuildCommand) /p:TestPackages=$(TestPackages)</TestBuildCommand>
   </PropertyGroup>
 
   <ItemGroup>
@@ -87,10 +69,10 @@
       <DestinationFolder>$(TestDir)</DestinationFolder>
     </TestSupportFiles>
     <TestSupportFiles Include="**\*.targets">
-      <DestinationFolder>$(TestDir)%(RecursiveDir)</DestinationFolder>
+      <DestinationFolder>$(TestSupportDir)%(RecursiveDir)</DestinationFolder>
     </TestSupportFiles>
     <TestSupportFiles Include="$(TestProjectName)">
-      <DestinationFolder>$(TestDir)</DestinationFolder>
+      <DestinationFolder>$(TestSupportDir)</DestinationFolder>
     </TestSupportFiles>
   </ItemGroup>
 
@@ -176,19 +158,68 @@
     <Message Text="Generated $(_projectFile)" />
   </Target>
 
+  <Target Name="CopyProducedPackages">
+    <!-- Copy over package local cache to our helix runtime path -->
+    <ItemGroup>
+     <_localPackages Include="$(PackageOutputPath)*.nupkg" />
+    </ItemGroup>
+
+    <Copy SourceFiles="@(_localPackages)"
+          UseHardlinksIfPossible="true"
+          DestinationFolder="$(TestSupportDir)packages" />
+  </Target>
+
+  <Target Name="ArchiveHelixItems"
+          DependsOnTargets="GenerateProjects;CopyProducedPackages"
+          Condition="'$(ArchiveTests)' == 'true'">
+
+    <MakeDir Directories="$(TestArchiveTestsRoot)" />
+    <ZipDirectory
+        SourceDirectory="$(TestProjectDir)%(SupportedPackage.Identity)"
+        DestinationFile="$(TestArchiveTestsRoot)%(SupportedPackage.Identity).zip"
+        Overwrite="true" />
+
+    <MakeDir Directories="$(TestArchiveRuntimeRoot)" />
+    <ZipDirectory
+        SourceDirectory="$(TestSupportDir)"
+        DestinationFile="$(TestArchiveRuntimeRoot)packages-testPayload-$(ConfigurationGroup).zip"
+        Overwrite="true" />
+
+  </Target>
+
   <Target Name="RestoreProjects"
-          DependsOnTargets="GenerateProjects">
+          DependsOnTargets="GenerateProjects"
+          Condition="'$(ArchiveTests)' != 'true'">
+
+    <PropertyGroup>
+      <TestRestoreCommand>$(TestDotNetPath)</TestRestoreCommand>
+      <TestRestoreCommand>$(TestRestoreCommand) restore</TestRestoreCommand>
+      <TestRestoreCommand>$(TestRestoreCommand) --packages "$(TestPackageDir)"</TestRestoreCommand>
+      <TestRestoreCommand>$(TestRestoreCommand) /p:LocalPackagesPath=$(PackageOutputPath)</TestRestoreCommand>
+      <TestRestoreCommand  Condition="'$(TestPackages)' != ''">$(TestRestoreCommand) /p:TestPackages=$(TestPackages)</TestRestoreCommand>
+    </PropertyGroup>
+
     <Message Importance="High" Text="*** Restoring ***" />
     <Exec Command="$(TestRestoreCommand) &quot;$(TestProject)&quot;" EnvironmentVariables="@(CliEnvironment)" StandardOutputImportance="High" />
   </Target>
 
   <Target Name="BuildProjects"
-          DependsOnTargets="RestoreProjects">
-    <Message Importance="High" Text="*** Testing ***" />
-    <Exec Command="$(TestBuildCommand) &quot;$(TestProject)&quot; /p:TestPackages=%(SupportedPackage.Identity)" EnvironmentVariables="@(CliEnvironment)" StandardOutputImportance="High" />
+          DependsOnTargets="RestoreProjects"
+          Condition="'$(ArchiveTests)' != 'true'">
+
+    <PropertyGroup>
+      <TestBuildCommand>$(TestDotNetPath)</TestBuildCommand>
+      <TestBuildCommand>$(TestBuildCommand) msbuild</TestBuildCommand>
+      <TestBuildCommand>$(TestBuildCommand) /t:Test</TestBuildCommand>
+      <TestBuildCommand  Condition="'$(TestPackages)' != ''">$(TestBuildCommand) /p:TestPackages=$(TestPackages)</TestBuildCommand>
+    </PropertyGroup>
+
+    <Message Importance="High" Text="*** Testing *** %(SupportedPackage.Identity)" />
+    <Message Importance="High" Text="$(TestBuildCommand) &quot;$(TestProject)&quot;" />
+    <Exec Command="$(TestBuildCommand) &quot;$(TestProject)&quot;" EnvironmentVariables="@(CliEnvironment)" StandardOutputImportance="High" />
   </Target>
 
-  <Target Name="Build" DependsOnTargets="BuildProjects" />
+  <Target Name="Build" DependsOnTargets="BuildProjects;ArchiveHelixItems" />
 
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), Directory.Build.targets))\Directory.Build.targets" />
 </Project>