From bd6c81a944f609b88908864827b7797c6b127c5b Mon Sep 17 00:00:00 2001
From: Davis Goodin <dagood@users.noreply.github.com>
Date: Fri, 20 Dec 2019 15:26:47 -0600
Subject: [PATCH] Publish the official build to blob storage (#1092)

* Publish installers and checksums

- Use the arcade publishing process to publish the installers and checksums
- Remove old blob publishing
- Calculate the product version using an always-suffixed version
- Change the blob path that we should upload to to be Runtime/<version> instead of assets/core-setup/.. to line up with the desired paths.

* Updates for dotnet/runtime

Use InstallerTasksAssemblyPath.

Fail fast on RuntimeNupkgFile item duplicates.

Find some more symbol packages. AllConfigurations symbol nupkgs weren't detected due to different artifact arrangement. There may be more missing, however I did this to do unblock some quick local validation so I'm including it.

Co-authored-by: Matt Mitchell <mmitche@microsoft.com>
---
 eng/pipelines/official/stages/publish.yml     |  2 +
 src/installer/Directory.Build.targets         |  1 -
 src/installer/publish/Directory.Build.targets | 42 +++++++++++++--
 src/installer/publish/prepare-artifacts.proj  | 37 +++++++++++--
 src/installer/publish/publish-blobs.proj      | 75 ---------------------------
 5 files changed, 71 insertions(+), 86 deletions(-)
 delete mode 100644 src/installer/publish/publish-blobs.proj

diff --git a/eng/pipelines/official/stages/publish.yml b/eng/pipelines/official/stages/publish.yml
index 654e429..6663be3 100644
--- a/eng/pipelines/official/stages/publish.yml
+++ b/eng/pipelines/official/stages/publish.yml
@@ -33,6 +33,8 @@ stages:
     # Allow symbol publish to emit expected warnings without failing the build. Include single
     # quotes inside the string so that it passes through to MSBuild without script interference.
     symbolPublishingAdditionalParameters: "'-warnAsError:$false'"
+    # Publish to blob storage.
+    publishInstallersAndChecksums: true
     # Enable SDL validation, passing through values from the 'core-setup-sdl-validation' group.
     SDLValidationParameters:
       enable: false # TODO: (Consolidation) Decide who owns SDL validation errors and enable. https://github.com/dotnet/runtime/issues/1027
diff --git a/src/installer/Directory.Build.targets b/src/installer/Directory.Build.targets
index e31f535..687aebc 100644
--- a/src/installer/Directory.Build.targets
+++ b/src/installer/Directory.Build.targets
@@ -90,7 +90,6 @@
       <HostPolicyVersion Condition="'$(UseShippedHostPolicyPackage)' != 'true'">$(ProductVersion)</HostPolicyVersion>
       <HostPolicyVersion Condition="'$(UseShippedHostPolicyPackage)' == 'true'">2.0.0</HostPolicyVersion>
 
-      <BinariesRelativePath>Runtime/$(SharedFrameworkNugetVersion)/</BinariesRelativePath>
       <InstallersRelativePath>Runtime/$(SharedFrameworkNugetVersion)/</InstallersRelativePath>
     </PropertyGroup>
 
diff --git a/src/installer/publish/Directory.Build.targets b/src/installer/publish/Directory.Build.targets
index de17992..b33d0f1 100644
--- a/src/installer/publish/Directory.Build.targets
+++ b/src/installer/publish/Directory.Build.targets
@@ -33,15 +33,31 @@
         are published from every job. RID-agnostic nupkgs are built with the same ID/version by
         every job, so one specific job's outputs must be picked to sign and publish.
       -->
+
+      <!-- RID-specific framework packs. -->
       <RuntimeNupkgFile
         Include="
-          $(DownloadDirectory)**\runtime.*.nupkg;
-          $(DownloadDirectory)**\*.Runtime.*.nupkg;
-          $(DownloadDirectory)**\*.App.Host.*.nupkg;
-          $(DownloadDirectory)**\VS.Redist.Common.*.nupkg"
+          $(DownloadDirectory)**\Microsoft.*.Runtime.*.nupkg;
+          $(DownloadDirectory)**\Microsoft.*.App.Host.*.nupkg"
+        Exclude="@(DownloadedSymbolNupkgFile)" />
+
+      <!-- VS insertion packages, carrying RID-specific installers. -->
+      <RuntimeNupkgFile
+        Include="$(DownloadDirectory)**\VS.Redist.Common.*.nupkg"
         Exclude="@(DownloadedSymbolNupkgFile)" />
 
       <!--
+        Runtime packages associated with some identity packages. Need to exclude "runtime.native.*"
+        because Libraries produces some "runtime.native.Foo" packages with
+        "runtime.<rid>.runtime.native.Foo" identity packages.
+      -->
+      <RuntimeNupkgFile
+        Include="$(DownloadDirectory)**\runtime.*.nupkg"
+        Exclude="
+          $(DownloadDirectory)**\runtime.native.*.nupkg;
+          @(DownloadedSymbolNupkgFile)" />
+
+      <!--
         Packages that aren't matched above as RID-specific are considered RID-agnostic. Also include
         the AllConfigurations packages from the Libraries build.
       -->
@@ -61,9 +77,15 @@
         reasons, such as the VS insertion packages that transport MSIs. Symbol package validation
         will check for symbol completeness with file-by-file granularity rather than looking for
         missing symbols.nupkg files: https://github.com/dotnet/arcade/issues/2499.
+
+        Handles several conventions:
+        * NonShipping packages have symbol nupkgs that are Shipping.
+        * Shipping packages have symbol packages in a "symbols" subdirectory.
       -->
       <PotentialSymbolNupkgToPublishFile
-        Include="@(NupkgToPublishFile->Replace('\NonShipping\', '\Shipping\')->Replace('.nupkg', '.symbols.nupkg'))" />
+        Include="
+          @(NupkgToPublishFile->Replace('\NonShipping\', '\Shipping\')->Replace('.nupkg', '.symbols.nupkg'));
+          @(NupkgToPublishFile->Replace('\Shipping\', '\Shipping\symbols\')->Replace('.nupkg', '.symbols.nupkg'))" />
 
       <SymbolNupkgToPublishFile
         Include="@(PotentialSymbolNupkgToPublishFile)"
@@ -75,6 +97,16 @@
     <Error
       Condition="'@(SymbolNupkgToPublishFile)' == ''"
       Text="No symbol packages found." />
+
+    <!--
+      Duplicate RuntimeNupkgFile items mean artifact upload will fail, but only after another hour
+      of signing. Detect this early. It's possible to automatically "fix" this with Distinct(),
+      however the patterns should be fairly specific: this is likely a build infra mistake that
+      should be corrected.
+    -->
+    <Error
+      Text="Duplicate RuntimeNupkgFile entries for: %(RuntimeNupkgFile.Identity)"
+      Condition="@(RuntimeNupkgFile->Count()) &gt; 1" />
   </Target>
 
   <Import Project="..\Directory.Build.targets" />
diff --git a/src/installer/publish/prepare-artifacts.proj b/src/installer/publish/prepare-artifacts.proj
index 69e4aa8..582fffa 100644
--- a/src/installer/publish/prepare-artifacts.proj
+++ b/src/installer/publish/prepare-artifacts.proj
@@ -1,6 +1,24 @@
 <Project>
   <Import Project="Sdk.props" Sdk="Microsoft.NET.Sdk" />
 
+  <UsingTask TaskName="GenerateChecksums" AssemblyFile="$(InstallerTasksAssemblyPath)" />
+
+  <PropertyGroup>
+    <SuppressFinalPackageVersion>true</SuppressFinalPackageVersion>
+  </PropertyGroup>
+
+  <Target Name="CreateChecksums">
+    <ItemGroup>
+      <ArtifactsForGeneratingChecksums
+        Include="@(UploadToBlobStorageFile)"
+        DestinationPath="%(FullPath)$(ChecksumExtension)" />
+
+      <GeneratedChecksumFile Include="@(ArtifactsForGeneratingChecksums -> '%(DestinationPath)')" />
+    </ItemGroup>
+
+    <GenerateChecksums Items="@(ArtifactsForGeneratingChecksums)" />
+  </Target>
+
   <Target Name="SignPackages"
           Condition="'$(SkipSigning)' != 'true' and '$(SignType)' != 'public'">
     <Message Importance="High" Text="Signing final packages" />
@@ -13,12 +31,17 @@
   <Target Name="UploadPreparedArtifactsToPipeline"
           DependsOnTargets="
             FindDownloadedArtifacts;
-            SignPackages">
+            SignPackages;
+            CreateChecksums">
     <PropertyGroup>
       <PreparedFileUploadDir>$(ArtifactsObjDir)PreparedFileUpload\</PreparedFileUploadDir>
     </PropertyGroup>
 
-    <Copy SourceFiles="@(UploadToBlobStorageFile)" DestinationFolder="$(PreparedFileUploadDir)">
+    <ItemGroup>
+      <AllFilesToBlobStorage Include="@(UploadToBlobStorageFile);@(UploadToBlobStorageFile)" />
+    </ItemGroup>
+
+    <Copy SourceFiles="@(AllFilesToBlobStorage)" DestinationFolder="$(PreparedFileUploadDir)">
       <Output TaskParameter="CopiedFiles" ItemName="CopiedUploadToBlobStorageFile" />
     </Copy>
 
@@ -96,9 +119,13 @@
 
       <ItemsToPush
         Include="@(UploadToBlobStorageFile)"
-        Exclude="@(NupkgToPublishFile);@(SymbolNupkgToPublishFile)" />
-      <ItemsToPush>
-        <RelativeBlobPath>assets/core-setup/$(InstallersRelativePath)%(Filename)%(Extension)</RelativeBlobPath>
+        Exclude="@(NupkgToPublishFile);@(SymbolNupkgToPublishFile)">
+        <RelativeBlobPath>$(InstallersRelativePath)%(Filename)%(Extension)</RelativeBlobPath>
+      </ItemsToPush>
+
+      <ItemsToPush Include="@(GeneratedChecksumFile)">
+        <RelativeBlobPath>$(InstallersRelativePath)%(Filename)%(Extension)</RelativeBlobPath>
+        <Category>Checksum</Category>
       </ItemsToPush>
     </ItemGroup>
 
diff --git a/src/installer/publish/publish-blobs.proj b/src/installer/publish/publish-blobs.proj
deleted file mode 100644
index 4622456..0000000
--- a/src/installer/publish/publish-blobs.proj
+++ /dev/null
@@ -1,75 +0,0 @@
-<Project>
-  <Import Project="Sdk.props" Sdk="Microsoft.NET.Sdk" />
-
-  <UsingTask TaskName="GenerateChecksums" AssemblyFile="$(InstallerTasksAssemblyPath)" />
-  <UsingTask TaskName="UploadToAzure" AssemblyFile="$(InstallerTasksAssemblyPath)" />
-
-  <Target Name="CreateChecksums"
-          DependsOnTargets="FindDownloadedArtifacts">
-    <ItemGroup>
-      <ArtifactsForGeneratingChecksums
-        Include="@(UploadToBlobStorageFile)"
-        DestinationPath="%(FullPath)$(ChecksumExtension)" />
-      
-      <GeneratedChecksumFile Include="@(ArtifactsForGeneratingChecksums -> '%(DestinationPath)')" />
-    </ItemGroup>
-
-    <GenerateChecksums Items="@(ArtifactsForGeneratingChecksums)" />
-  </Target>
-
-  <Target Name="UploadToAzure"
-          DependsOnTargets="
-            FindDownloadedArtifacts;
-            GetProductVersions">
-    <Error Condition="'$(AzureAccessToken)' == ''" Text="Missing required property 'AzureAccessToken'" />
-    <Error Condition="'$(AzureAccountName)' == ''" Text="Missing required property 'AzureAccountName'" />
-    <Error Condition="'$(ContainerName)' == ''" Text="Missing required property 'ContainerName'" />
-
-    <ItemGroup>
-      <FileToPublish
-        Include="@(UploadToBlobStorageFile)" 
-        RelativeBlobPath="$(BinariesRelativePath)%(Filename)%(Extension)" />
-    </ItemGroup>
-
-    <Message Importance="High" Text="Uploading files to '$(AzureAccountName)/$(ContainerName)/$(BinariesRelativePath)'" />
-    <UploadToAzure AccountName="$(AzureAccountName)"
-                   AccountKey="$(AzureAccessToken)"
-                   ContainerName="$(ContainerName)"
-                   Items="@(FileToPublish)"
-                   Overwrite="false" />
-  </Target>
-
-  <Target Name="UploadChecksumsToAzure"
-          DependsOnTargets="
-            FindDownloadedArtifacts;
-            GetProductVersions;
-            CreateChecksums">
-    <Error Condition="'$(ChecksumAzureAccessToken)' == ''" Text="Missing required property 'ChecksumAzureAccessToken'" />
-    <Error Condition="'$(ChecksumAzureAccountName)' == ''" Text="Missing required property 'ChecksumAzureAccountName'" />
-    <Error Condition="'$(ChecksumContainerName)' == ''" Text="Missing required property 'ChecksumContainerName'" />
-
-    <ItemGroup>
-      <ChecksumToPublish
-        Include="@(GeneratedChecksumFile)"
-        RelativeBlobPath="$(BinariesRelativePath)%(Filename)%(Extension)" />
-    </ItemGroup>
-
-    <Message Importance="High" Text="Uploading checksums to '$(ChecksumAzureAccountName)/$(ChecksumContainerName)/$(BinariesRelativePath)'" />
-    <UploadToAzure AccountName="$(ChecksumAzureAccountName)"
-                   AccountKey="$(ChecksumAzureAccessToken)"
-                   ContainerName="$(ChecksumContainerName)"
-                   Items="@(ChecksumToPublish)"
-                   Overwrite="false" />
-  </Target>
-
-  <Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
-
-  <!-- Perform extra publish steps that Arcade doesn't support. -->
-  <Target Name="Build"
-          DependsOnTargets="
-            UploadToAzure;
-            UploadChecksumsToAzure">
-    <Message Importance="High" Text="Complete!" />
-  </Target>
-
-</Project>
-- 
2.7.4