Fix several issues in build and publishing (#889)

* Make packages explicitly Shipping/Non-Shipping

* Work around SDK bug that trips sourcelink validation

* Cleanup properies in publishing

 These properties used to be needed, but they have been removed in arcade.
 The new workflow is to use pipeline publishing which this repo uses.

* Make publishing explicit for test products that need it

This will save us from hardcoding paths by making it more explicit.
That way when we clean up and decouple builds we can have breaks when needed.
It also brings down artifact size from 318 MB to 206
for xcopy tests needed for helix.

* Correctly bubble up native build issues.

diff --git a/diagnostics.yml b/diagnostics.yml
index deb3d84a4b5a6f16705112376b92b6bef8821077..8fe76a879fcf45cf2e1d456092c782026cf9cab9 100644 (file)
--- a/diagnostics.yml
+++ b/diagnostics.yml
@@ -13,7 +13,21 @@ pr:
     - LICENSE.TXT
 
 variables:
-- template: /eng/common-variables.yml
+  - name: _TeamName
+    value: DotNetCore
+  - name: _InternalBuildArgs
+    value: ''
+
+  - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - name: _SignType
+      value: real
+    # DotNet-Diagnostics-SDL-Params provides Tsa* variables for SDL checks.
+    - group: DotNet-Diagnostics-SDL-Params
+    - name: _InternalBuildArgs
+      value: /p:DotNetSignType=$(_SignType)
+        /p:TeamName=$(_TeamName)
+        /p:DotNetPublishUsingPipelines=$(PublishPackages)
+        /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
 
 stages:
   - stage: build
@@ -248,7 +262,7 @@ stages:
         - template: /eng/common/templates/job/job.yml
           parameters:
             name: Sign_Package_Publish
-            displayName: Sign, Package, and Publish to BAR
+            displayName: Sign, Package, and Generate BAR Manifests
             dependsOn:
             - Windows
             - CentOS_7

diff --git a/eng/Build-Native.cmd b/eng/Build-Native.cmd
index 8ab3931944c4bc851b47d4052721552d7ed1237a..3389991565e236d2cf113ca0c587bba672336582 100644 (file)
--- a/eng/Build-Native.cmd
+++ b/eng/Build-Native.cmd
@@ -10,7 +10,7 @@ set __ThisScriptFull="%~f0"
 set __ThisScriptDir="%~dp0"
 
 call "%__ThisScriptDir%"\setup-vs-tools.cmd
-if NOT '%ERRORLEVEL%' == '0' exit /b 1
+if NOT '%ERRORLEVEL%' == '0' goto ExitWithError
 
 if defined VS160COMNTOOLS (
     set "__VSToolsRoot=%VS160COMNTOOLS%"
@@ -150,7 +150,7 @@ REM ============================================================================
 set __DotNetCli=%__ProjectDir%\.dotnet\dotnet.exe
 if not exist "%__DotNetCli%" (
     echo %__MsgPrefix%Assertion failed: dotnet cli not found at path "%__DotNetCli%"
-    exit /b 1
+    goto ExitWithError
 )
 
 REM =========================================================================================
@@ -180,7 +180,7 @@ if /i %__BuildCrossArch% EQU 1 (
     powershell -NoProfile -ExecutionPolicy ByPass -NoLogo -File "%__ProjectDir%\eng\common\msbuild.ps1" "%__ProjectDir%\eng\CreateVersionFile.csproj" /bl:!__GenerateVersionLog! /t:GenerateVersionFiles /restore /p:FileVersionFile=%__RootBinDir%\bin\FileVersion.txt /p:GenerateVersionHeader=true /p:NativeVersionHeaderFile=%__CrossCompIntermediatesDir%\_version.h %__CommonBuildArgs%
     if not !errorlevel! == 0 (
         echo Generate Version Header FAILED
-        exit /b 1
+        goto ExitWithError
     )
     if defined __SkipConfigure goto SkipConfigureCrossBuild
 
@@ -199,7 +199,7 @@ if /i %__BuildCrossArch% EQU 1 (
 :SkipConfigureCrossBuild
     if not exist "%__CrossCompIntermediatesDir%\install.vcxproj" (
         echo %__MsgPrefix%Error: failed to generate cross-arch components build project!
-        exit /b 1
+        goto ExitWithError
     )
     if defined __ConfigureOnly goto SkipCrossCompBuild
 
@@ -211,7 +211,7 @@ if /i %__BuildCrossArch% EQU 1 (
     if not !ERRORLEVEL! == 0 (
         echo %__MsgPrefix%Error: cross-arch components build failed. Refer to the build log files for details:
         echo     !__BuildLog!
-        exit /b 1
+        goto ExitWithError
     )
 
 :SkipCrossCompBuild
@@ -250,7 +250,7 @@ if %__Build% EQU 1 (
 
     if not defined VSINSTALLDIR (
         echo %__MsgPrefix%Error: VSINSTALLDIR variable not defined.
-        exit /b 1
+        goto ExitWithError
     )
 
     echo Generating Version Header
@@ -258,7 +258,7 @@ if %__Build% EQU 1 (
     powershell -NoProfile -ExecutionPolicy ByPass -NoLogo -File "%__ProjectDir%\eng\common\msbuild.ps1" "%__ProjectDir%\eng\CreateVersionFile.csproj" /bl:!__GenerateVersionLog! /t:GenerateVersionFiles /restore /p:FileVersionFile=%__RootBinDir%\bin\FileVersion.txt /p:GenerateVersionHeader=true /p:NativeVersionHeaderFile=%__IntermediatesDir%\_version.h %__CommonBuildArgs%
     if not !errorlevel! == 0 (
         echo Generate Version Header FAILED
-        exit /b 1
+        goto ExitWithError
     )
     if defined __SkipConfigure goto SkipConfigure
 
@@ -278,7 +278,7 @@ if %__Build% EQU 1 (
 
     if not exist "%__IntermediatesDir%\install.vcxproj" (
         echo %__MsgPrefix%Error: failed to generate native component build project!
-        exit /b 1
+        goto ExitWithError
     )
     set __BuildLog="%__LogDir%\Native.Build.binlog"
 
@@ -288,7 +288,7 @@ if %__Build% EQU 1 (
     if not !ERRORLEVEL! == 0 (
         echo %__MsgPrefix%Error: native component build failed. Refer to the build log files for details:
         echo     !__BuildLog!
-        exit /b 1
+        goto ExitWithError
     )
 
 :SkipNativeBuild
@@ -314,6 +314,18 @@ echo %__MsgPrefix%Product binaries are available at !__BinDir!
 
 exit /b 0
 
+REM =========================================================================================
+REM === These two routines are intended for the exit code to propagate to the parent process
+REM === Like MSBuild or Powershell. If we directly goto ExitWithError from within a if statement in
+REM === any of the routines, the exit code is not propagated due to quirks of nested conditonals
+REM === in delayed expansion scripts.
+REM =========================================================================================
+:ExitWithError
+exit /b 1
+
+:ExitWithCode
+exit /b !__exitCode!
+
 REM =========================================================================================
 REM ===
 REM === Helper routines
@@ -333,4 +345,4 @@ echo.-? -h -help --help: view this message.
 echo -architecture <x64|x86|arm|arm64>
 echo -configuration <debug|release>
 echo -verbosity <q[uiet]|m[inimal]|n[ormal]|d[etailed]|diag[nostic]>
-exit /b 1
+goto ExitWithError

diff --git a/eng/common-variables.yml b/eng/common-variables.yml
deleted file mode 100644 (file)
index c98bd86..0000000
--- a/eng/common-variables.yml
+++ /dev/null
@@ -1,32 +0,0 @@
-
-variables:
-  - name: _TeamName
-    value: DotNetCore
-  - name: _DotNetArtifactsCategory
-    value: .NETCore
-  - name: _InternalBuildArgs
-    value: ''
-
-  - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - name: _SignType
-      value: real
-    - name: _PublishBlobFeedUrl
-      value: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
-    # DotNet-Blob-Feed provides: dotnetfeed-storage-access-key-1
-    # DotNet-Diagnostics-SDL-Params provides Tsa* variables for SDL checks.
-    # DotNet-Symbol-Server-Pats provides: provides microsoft-symbol-server-pat and symweb-symbol-server-pat
-    # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT
-    - group: DotNet-Blob-Feed
-    - group: DotNet-Diagnostics-SDL-Params
-    - group: DotNet-Symbol-Server-Pats
-    - group: Publish-Build-Assets
-    - name: _InternalBuildArgs
-      value: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName)
-        /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory)
-        /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1)
-        /p:DotNetPublishBlobFeedUrl=$(_PublishBlobFeedUrl)
-        /p:DotNetPublishToBlobFeed=$(PublishPackages)
-        /p:DotNetPublishUsingPipelines=$(PublishPackages)
-        /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
-        /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
-        /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
\ No newline at end of file

diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets
index 1d48a1ff576818969ed700284a919119d5c3dc83..ffe4d76752e03c1afc9cd5f49fbbec3acfe8892d 100644 (file)
--- a/src/Directory.Build.targets
+++ b/src/Directory.Build.targets
@@ -1,10 +1,20 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project>
   <Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
+  
+  <!-- Work around https://github.com/dotnet/sourcelink/issues/572
+  Remove once we build using an SDK that contains https://github.com/dotnet/sdk/pull/10613 -->
+  <PropertyGroup>
+    <TargetFrameworkMonikerAssemblyAttributesPath>$([System.IO.Path]::Combine('$(IntermediateOutputPath)','$(TargetFrameworkMoniker).AssemblyAttributes$(DefaultLanguageSourceExtension)'))</TargetFrameworkMonikerAssemblyAttributesPath>
+  </PropertyGroup>
+  <ItemGroup>
+    <EmbeddedFiles Include="$(GeneratedAssemblyInfoFile)"/>
+  </ItemGroup>
 
-  <!-- Remove this once arcade version is bumped -->
+  <!-- We need this for the binplacing for testing assets.
+  This should be removed at some point as it's brittle (harcodes versions and creates native-managed coupling). -->
   <Target Name="_PublishPackageReferences"
           AfterTargets="PostBuildEvent"
-          Condition="$(IsPublishable) == 'true'"
+          Condition="$(NeedsPublishing) == 'true'"
           DependsOnTargets="$(_BeforePublishNoBuildTargets);$(_CorePublishTargets)" />
 </Project>

diff --git a/src/Microsoft.Diagnostics.DebugServices/Microsoft.Diagnostics.DebugServices.csproj b/src/Microsoft.Diagnostics.DebugServices/Microsoft.Diagnostics.DebugServices.csproj
index 49aedc5efb321d52930aaacddac45b6ae205af9b..1ae1f70ac33597883c1459cea8caa85bde4a05e0 100644 (file)
--- a/src/Microsoft.Diagnostics.DebugServices/Microsoft.Diagnostics.DebugServices.csproj
+++ b/src/Microsoft.Diagnostics.DebugServices/Microsoft.Diagnostics.DebugServices.csproj
@@ -10,7 +10,7 @@
     <PackageReleaseNotes>$(Description)</PackageReleaseNotes>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <IncludeSymbols>true</IncludeSymbols>
-    <IsShippingAssembly>true</IsShippingAssembly>
+    <IsShippingPackage>false</IsShippingPackage>
   </PropertyGroup>
   
   <ItemGroup>

diff --git a/src/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj b/src/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj
index 7e530dc181b9c7d334de4c0520f1ae5d10b6ceee..b9fb1b55e099fdbe640d9b8cf55e4b9e71f64e99 100644 (file)
--- a/src/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj
+++ b/src/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj
@@ -10,6 +10,6 @@
     <PackageReleaseNotes>$(Description)</PackageReleaseNotes>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <IncludeSymbols>true</IncludeSymbols>
-    <IsShippingAssembly>true</IsShippingAssembly>
+    <IsShipping>true</IsShipping>
   </PropertyGroup>
 </Project>

diff --git a/src/Microsoft.Diagnostics.Repl/Microsoft.Diagnostics.Repl.csproj b/src/Microsoft.Diagnostics.Repl/Microsoft.Diagnostics.Repl.csproj
index e89e6cc0180300e2329d923c9b4e1b07b5e72630..bed51100c8a0304fccc77070fca5f1b7e3acf215 100644 (file)
--- a/src/Microsoft.Diagnostics.Repl/Microsoft.Diagnostics.Repl.csproj
+++ b/src/Microsoft.Diagnostics.Repl/Microsoft.Diagnostics.Repl.csproj
@@ -10,7 +10,7 @@
     <PackageReleaseNotes>$(Description)</PackageReleaseNotes>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
     <IncludeSymbols>true</IncludeSymbols>
-    <IsShippingAssembly>true</IsShippingAssembly>
+    <IsShippingPackage>false</IsShippingPackage>
   </PropertyGroup>
   
   <ItemGroup>

diff --git a/src/Microsoft.Diagnostics.TestHelpers/Microsoft.Diagnostics.TestHelpers.csproj b/src/Microsoft.Diagnostics.TestHelpers/Microsoft.Diagnostics.TestHelpers.csproj
index bea81084285a6ee5dd428842a1512f249c004ae6..5d265b47fa51e7e34c0acd0fac419ebc20090aa8 100644 (file)
--- a/src/Microsoft.Diagnostics.TestHelpers/Microsoft.Diagnostics.TestHelpers.csproj
+++ b/src/Microsoft.Diagnostics.TestHelpers/Microsoft.Diagnostics.TestHelpers.csproj
@@ -9,7 +9,7 @@
     <PackageReleaseNotes>$(Description)</PackageReleaseNotes>
     <PackageTags>tests</PackageTags>
     <IncludeSymbols>true</IncludeSymbols>
-    <IsShippingAssembly>true</IsShippingAssembly>
+    <IsShipping>false</IsShipping>
   </PropertyGroup>
   
   <ItemGroup>

diff --git a/src/SOS/SOS.NETCore/SOS.NETCore.csproj b/src/SOS/SOS.NETCore/SOS.NETCore.csproj
index d4487ca8a5582f5f87644f84a9db016961f78161..90f326bd72ce9b157ddc7bc99fb216e65e7ab166 100644 (file)
--- a/src/SOS/SOS.NETCore/SOS.NETCore.csproj
+++ b/src/SOS/SOS.NETCore/SOS.NETCore.csproj
@@ -3,10 +3,11 @@
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
     <AssemblyName>SOS.NETCore</AssemblyName>
+    <NeedsPublishing>true</NeedsPublishing>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <NoWarn>;1591;1701</NoWarn>
     <Description>.NET Core SOS</Description>
-    <IsShippingAssembly>true</IsShippingAssembly>
+    <IsShippingPackage>false</IsShippingPackage>
   </PropertyGroup>
   
   <ItemGroup>

diff --git a/src/SOS/SOS.Package/SOS.Package.csproj b/src/SOS/SOS.Package/SOS.Package.csproj
index d429cbbbe728f027ae294116ddf18df758c05dc7..cf28fc298a1d0d80dccd7de3e084bf43bef5f32a 100644 (file)
--- a/src/SOS/SOS.Package/SOS.Package.csproj
+++ b/src/SOS/SOS.Package/SOS.Package.csproj
@@ -12,6 +12,7 @@
     <SOSPackagePathPrefix>tools</SOSPackagePathPrefix>
     <GalleryManifestName>$(ArtifactsPackagesDir)\GalleryManifest.xml</GalleryManifestName>
     <BeforePack>GenerateGalleryZip;GenerateSymbolsZip</BeforePack>
+    <IsShippingPackage>false</IsShippingPackage>
   </PropertyGroup>
 
   <ItemGroup>

diff --git a/src/SOS/SOS.UnitTests/ConfigFiles/Windows/Debugger.Tests.Config.txt b/src/SOS/SOS.UnitTests/ConfigFiles/Windows/Debugger.Tests.Config.txt
index 3ee2b5ededcf4d8ddf2f7b631e5c22326ff5cc2a..e708ee7f3ed34ca13b523980b23e931f0ac55bca 100644 (file)
--- a/src/SOS/SOS.UnitTests/ConfigFiles/Windows/Debugger.Tests.Config.txt
+++ b/src/SOS/SOS.UnitTests/ConfigFiles/Windows/Debugger.Tests.Config.txt
@@ -118,7 +118,7 @@
             <Option>
               <TestName>SOS.DualRuntimes</TestName>
               <!-- The assembly path, class and function name of the desktop test code to load/run -->
-              <DesktopTestParameters>$(RootBinDir)\bin\SymbolTestDll\$(TargetConfiguration)\$(DesktopFramework)\publish\SymbolTestDll.dll SymbolTestDll.TestClass ThrowException</DesktopTestParameters>
+              <DesktopTestParameters>$(RootBinDir)\bin\SymbolTestDll\$(TargetConfiguration)\$(DesktopFramework)\SymbolTestDll.dll SymbolTestDll.TestClass ThrowException</DesktopTestParameters>
             </Option>
           </Options>
           <Options>

diff --git a/src/SOS/SOS.UnitTests/SOS.UnitTests.csproj b/src/SOS/SOS.UnitTests/SOS.UnitTests.csproj
index 5443f467eb494c6e3d8aeead01c88fb31cf84161..1474a15f872cd563cd9f88a0b6fbf9b2e1bd7a4b 100644 (file)
--- a/src/SOS/SOS.UnitTests/SOS.UnitTests.csproj
+++ b/src/SOS/SOS.UnitTests/SOS.UnitTests.csproj
@@ -8,6 +8,7 @@
     <IsPackable>false</IsPackable>
     <NoWarn>;1591;1701</NoWarn>
     <DefineConstants>$(DefineConstants);CORE_CLR</DefineConstants>
+    <NeedsPublishing>true</NeedsPublishing>
   </PropertyGroup>
   
   <ItemGroup>

diff --git a/src/Tools/Directory.Build.props b/src/Tools/Directory.Build.props
index 6ca593a9f29cfce482876a69e968b0e89d2e185c..0ddd89abc28b51aa3f5dd754b5edefcd186cf26b 100644 (file)
--- a/src/Tools/Directory.Build.props
+++ b/src/Tools/Directory.Build.props
@@ -9,5 +9,6 @@
     <PackAsToolShimRuntimeIdentifiers>win-x64;win-x86;win-arm;osx-x64</PackAsToolShimRuntimeIdentifiers>
     <PackagedShimOutputRootDirectory>$(OutputPath)</PackagedShimOutputRootDirectory>
     <GenerateDocumentationFile>false</GenerateDocumentationFile>
+    <IsShipping>true</IsShipping>
   </PropertyGroup>
 </Project>

diff --git a/src/Tools/dotnet-dump/dotnet-dump.csproj b/src/Tools/dotnet-dump/dotnet-dump.csproj
index 831bca3e4adfc0135427f01338fd82f85538ae69..d8e22ed8cba00cbffa01610f6779e1bd3448273a 100644 (file)
--- a/src/Tools/dotnet-dump/dotnet-dump.csproj
+++ b/src/Tools/dotnet-dump/dotnet-dump.csproj
@@ -8,6 +8,7 @@
     <RootNamespace>Microsoft.Diagnostic.Tools.Dump</RootNamespace>
     <Description>Diagnostic dump collect and analyze tool</Description>
     <PackageTags>Diagnostic</PackageTags>
+    <NeedsPublishing>true</NeedsPublishing>
     <PackageReleaseNotes>$(Description)</PackageReleaseNotes>
     <SOSPackagePathPrefix>tools/$(TargetFramework)/any</SOSPackagePathPrefix>
   </PropertyGroup>