From: Mike McLaughlin Date: Tue, 9 Oct 2018 03:46:24 +0000 (-0700) Subject: Convert to Arcade SDK from the RoslynTools.RepoToolSet (#76) X-Git-Tag: submit/tizen/20190813.035844~76 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=031d06705078249202878fa47663b61d585b8c54;p=platform%2Fcore%2Fdotnet%2Fdiagnostics.git Convert to Arcade SDK from the RoslynTools.RepoToolSet (#76) Convert to Arcade SDK from RoslynTools.RepoToolSet. Upgrade to arcade SDK 1.0.0-beta.18501.3+25f5624a Switched to dotnet cli 2.1.401 There are still some zip signing and version file issues that will be addressed in future PRs. Fix the version file build and remove the build tools dependency --- diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index 0c9a0f858..10c8b1185 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -1,7 +1,7 @@ phases: - template: /eng/build.yml parameters: - phaseName: Windows + name: Windows agentOs: Windows_NT queue: name: Hosted VS2017 @@ -18,7 +18,7 @@ phases: - template: /eng/build.yml parameters: - phaseName: CentOS_7 + name: CentOS_7 agentOs: Linux dockerImage: microsoft/dotnet-buildtools-prereqs:centos-7-c103199-20180628120549 queue: @@ -36,7 +36,7 @@ phases: - template: /eng/build.yml parameters: - phaseName: MacOS + name: MacOS agentOs: Darwin queue: name: Hosted macOS Preview diff --git a/.vsts-dotnet.yml b/.vsts-dotnet.yml index 4bd79a890..2f8a07386 100644 --- a/.vsts-dotnet.yml +++ b/.vsts-dotnet.yml @@ -1,7 +1,7 @@ phases: - template: /eng/build.yml parameters: - phaseName: Windows + name: Windows agentOs: Windows_NT queue: name: DotNetCore-Build @@ -19,11 +19,11 @@ phases: Build_Release_x86: _BuildConfig: Release _BuildArch: x86 - _PublishArtifacts: bin/Windows_NT.x86 + _PublishArtifacts: bin/Windows_NT.x86.Release - template: /eng/build.yml parameters: - phaseName: CentOS_7 + name: CentOS_7 agentOs: Linux dockerImage: microsoft/dotnet-buildtools-prereqs:centos-7-c103199-20180628120549 queue: @@ -35,15 +35,15 @@ phases: Build_Debug: _BuildConfig: Debug _BuildArch: x64 - _PublishArtifacts: bin/Linux.x64 + _PublishArtifacts: bin/Linux.x64.Debug Build_Release: _BuildConfig: Release _BuildArch: x64 - _PublishArtifacts: bin/Linux.x64 + _PublishArtifacts: bin/Linux.x64.Release - template: /eng/build.yml parameters: - phaseName: MacOS + name: MacOS agentOs: Darwin queue: name: Hosted macOS Preview @@ -57,11 +57,11 @@ phases: Build_Release: _BuildConfig: Release _BuildArch: x64 - _PublishArtifacts: bin/OSX.x64 + _PublishArtifacts: bin/OSX.x64.Release - template: /eng/build.yml parameters: - phaseName: Debian_Stretch + name: Debian_Stretch agentOs: Linux dockerImage: microsoft/dotnet-buildtools-prereqs:debian-stretch-c103199-20180628122423 dependsOn: CentOS_7 @@ -77,7 +77,7 @@ phases: - template: /eng/build.yml parameters: - phaseName: Fedora_27 + name: Fedora_27 agentOs: Linux dockerImage: microsoft/dotnet-buildtools-prereqs:fedora-27-c103199-20180628122443 dependsOn: CentOS_7 @@ -93,7 +93,7 @@ phases: - template: /eng/build.yml parameters: - phaseName: Fedora_28 + name: Fedora_28 agentOs: Linux dockerImage: microsoft/dotnet-buildtools-prereqs:fedora-28-c103199-20180628122443 dependsOn: CentOS_7 @@ -109,7 +109,7 @@ phases: - template: /eng/build.yml parameters: - phaseName: OpenSuse_42_1 + name: OpenSuse_42_1 agentOs: Linux dockerImage: microsoft/dotnet-buildtools-prereqs:opensuse-42.1-c103199-20180628122439 dependsOn: CentOS_7 @@ -125,7 +125,7 @@ phases: - template: /eng/build.yml parameters: - phaseName: OpenSuse_42_3 + name: OpenSuse_42_3 agentOs: Linux dockerImage: microsoft/dotnet-buildtools-prereqs:opensuse-42.3-c103199-20180628122439 dependsOn: CentOS_7 @@ -141,7 +141,7 @@ phases: - template: /eng/build.yml parameters: - phaseName: Ubuntu_14_04 + name: Ubuntu_14_04 agentOs: Linux dockerImage: microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-c103199-20180628134413 dependsOn: CentOS_7 @@ -157,7 +157,7 @@ phases: - template: /eng/build.yml parameters: - phaseName: Ubuntu_16_04 + name: Ubuntu_16_04 agentOs: Linux dockerImage: microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-c103199-20180628134544 dependsOn: CentOS_7 @@ -173,7 +173,7 @@ phases: - template: /eng/build.yml parameters: - phaseName: Ubuntu_17_10 + name: Ubuntu_17_10 agentOs: Linux dockerImage: microsoft/dotnet-buildtools-prereqs:ubuntu-17.10-c103199-20180628134544 dependsOn: CentOS_7 @@ -189,7 +189,7 @@ phases: - template: /eng/build.yml parameters: - phaseName: Ubuntu_18_04 + name: Ubuntu_18_04 agentOs: Linux dockerImage: microsoft/dotnet-buildtools-prereqs:ubuntu-18.04-c103199-20180628134610 dependsOn: CentOS_7 @@ -213,23 +213,28 @@ phases: name: DotNetCore-Build demands: - agent.os -equals Windows_NT + variables: + _PublishBlobFeedUrl: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + _SOSNETCorePath: $(Build.SourcesDirectory)/artifacts/bin/SOS.NETCore/Release/netcoreapp1.0 + _TeamName: DotNetCore + steps: - task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@1 displayName: Install MicroBuild plugin inputs: - signType: real - zipSources: false + signType: $(SignType) esrpSigning: true + zipSources: false env: - TeamName: DotNetCore + TeamName: $(_TeamName) continueOnError: false condition: succeeded() - task: AzureKeyVault@1 inputs: - azureSubscription: 'DotNet-Engineering-Services_KeyVault' - KeyVaultName: EngKeyVault - SecretsFilter: 'dotnetfeed-storage-access-key-1' + azureSubscription: 'DncEng-VSTS' + keyVaultName: EngKeyVault + secretsFilter: 'dotnetfeed-storage-access-key-1,microsoft-symbol-server-pat,symweb-symbol-server-pat' condition: succeeded() # Windows x64 download, copy and archive. Everything under "bin" is published for the Windows x64 build. @@ -245,7 +250,24 @@ phases: displayName: Copy Windows Artifacts inputs: sourceFolder: '$(System.ArtifactsDirectory)/Windows_Build_Release' - targetFolder: '$(Build.SourcesDirectory)/artifacts/Release/bin' + targetFolder: '$(Build.SourcesDirectory)/artifacts/bin' + condition: succeeded() + + - task: CopyFiles@2 + displayName: Copy Windows x64 Artifacts + inputs: + sourceFolder: '$(Build.SourcesDirectory)/artifacts/bin/Windows_NT.x64.Release' + contents: 'sos.dll' + targetFolder: '$(Build.SourcesDirectory)/artifacts/Windows_NT.x64.Release' + condition: succeeded() + + - task: CopyFiles@2 + displayName: Copy Windows x64 Artifacts + inputs: + sourceFolder: '$(_SOSNETCorePath)' + contents: 'SOS.NETCore.dll' + overWrite: true + targetFolder: '$(Build.SourcesDirectory)/artifacts/Windows_NT.x64.Release' condition: succeeded() - task: ArchiveFiles@2 @@ -253,8 +275,8 @@ phases: inputs: archiveType: zip includeRootFolder: false - rootFolderOrFile: '$(Build.SourcesDirectory)/artifacts/Release/bin/Windows_NT.x64' - archiveFile: '$(Build.SourcesDirectory)/artifacts/Release/sos-win-x64.zip' + rootFolderOrFile: '$(Build.SourcesDirectory)/artifacts/Windows_NT.x64.Release' + archiveFile: '$(Build.SourcesDirectory)/artifacts/packages/Release/sos-win-x64.zip' condition: succeeded() # Windows x86 download, copy and archive. @@ -270,16 +292,17 @@ phases: displayName: Copy Windows x86 Artifacts inputs: sourceFolder: '$(System.ArtifactsDirectory)/Windows_Build_Release_x86' - targetFolder: '$(Build.SourcesDirectory)/artifacts/Release/bin/Windows_NT.x86' + contents: 'sos.dll' + targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/Windows_NT.x86.Release' condition: succeeded() - task: CopyFiles@2 displayName: Copy Windows x86 Artifacts inputs: - sourceFolder: '$(Build.SourcesDirectory)/artifacts/Release/bin/SOS.NETCore/netcoreapp1.0' + sourceFolder: '$(_SOSNETCorePath)' contents: 'SOS.NETCore.dll' overWrite: true - targetFolder: '$(Build.SourcesDirectory)/artifacts/Release/bin/Windows_NT.x86' + targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/Windows_NT.x86.Release' condition: succeeded() - task: ArchiveFiles@2 @@ -287,8 +310,8 @@ phases: inputs: archiveType: zip includeRootFolder: false - rootFolderOrFile: '$(Build.SourcesDirectory)/artifacts/Release/bin/Windows_NT.x86' - archiveFile: '$(Build.SourcesDirectory)/artifacts/Release/sos-win-x86.zip' + rootFolderOrFile: '$(Build.SourcesDirectory)/artifacts/bin/Windows_NT.x86.Release' + archiveFile: '$(Build.SourcesDirectory)/artifacts/packages/Release/sos-win-x86.zip' condition: succeeded() # Copy and archive the files for windbg extension gallery. @@ -296,35 +319,35 @@ phases: - task: CopyFiles@2 displayName: Copy Extension Gallery Files inputs: - sourceFolder: '$(Build.SourcesDirectory)/artifacts/Release/bin/Windows_NT.x64' + sourceFolder: '$(Build.SourcesDirectory)/artifacts/bin/Windows_NT.x64.Release' contents: 'sos.dll' - targetFolder: '$(Build.SourcesDirectory)/artifacts/Release/gallery/x64' + targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/gallery/x64' condition: succeeded() - task: CopyFiles@2 displayName: Copy Extension Gallery Files inputs: - sourceFolder: '$(Build.SourcesDirectory)/artifacts/Release/bin/SOS.NETCore/netcoreapp1.0' + sourceFolder: '$(_SOSNETCorePath)' contents: 'SOS.NETCore.dll' overWrite: true - targetFolder: '$(Build.SourcesDirectory)/artifacts/Release/gallery/x64' + targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/gallery/x64' condition: succeeded() - task: CopyFiles@2 displayName: Copy Extension Gallery Files inputs: - sourceFolder: '$(Build.SourcesDirectory)/artifacts/Release/bin/Windows_NT.x86' + sourceFolder: '$(Build.SourcesDirectory)/artifacts/bin/Windows_NT.x86.Release' contents: 'sos.dll' - targetFolder: '$(Build.SourcesDirectory)/artifacts/Release/gallery/x86' + targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/gallery/x86' condition: succeeded() - task: CopyFiles@2 displayName: Copy Extension Gallery Files inputs: - sourceFolder: '$(Build.SourcesDirectory)/artifacts/Release/bin/SOS.NETCore/netcoreapp1.0' + sourceFolder: '$(_SOSNETCorePath)' contents: 'SOS.NETCore.dll' overWrite: true - targetFolder: '$(Build.SourcesDirectory)/artifacts/Release/gallery/x86' + targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/gallery/x86' condition: succeeded() - task: CopyFiles@2 @@ -332,7 +355,7 @@ phases: inputs: sourceFolder: '$(Build.SourcesDirectory)/eng' contents: 'GalleryManifest.xml' - targetFolder: '$(Build.SourcesDirectory)/artifacts/Release/gallery' + targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/gallery' condition: succeeded() - task: ArchiveFiles@2 @@ -340,8 +363,8 @@ phases: inputs: archiveType: zip includeRootFolder: false - rootFolderOrFile: '$(Build.SourcesDirectory)/artifacts/Release/gallery' - archiveFile: '$(Build.SourcesDirectory)/artifacts/Release/sos-gallery.zip' + rootFolderOrFile: '$(Build.SourcesDirectory)/artifacts/bin/gallery' + archiveFile: '$(Build.SourcesDirectory)/artifacts/packages/Release/sos-gallery.zip' condition: succeeded() # Linux download, copy and archive @@ -357,26 +380,26 @@ phases: displayName: Copy Linux Artifacts inputs: sourceFolder: '$(System.ArtifactsDirectory)/CentOS_7_Build_Release' - targetFolder: '$(Build.SourcesDirectory)/artifacts/Release/bin/Linux.x64' + contents: ?(libsos.so|libsosplugin.so|sosdocsunix.txt) + targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/Linux.x64.Release' condition: succeeded() - task: CopyFiles@2 displayName: Copy Linux Artifacts inputs: - sourceFolder: '$(Build.SourcesDirectory)/artifacts/Release/bin/SOS.NETCore/netcoreapp1.0' + sourceFolder: '$(_SOSNETCorePath)' contents: 'SOS.NETCore.dll' overWrite: true - targetFolder: '$(Build.SourcesDirectory)/artifacts/Release/bin/Linux.x64' + targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/Linux.x64.Release' condition: succeeded() - task: ArchiveFiles@2 displayName: Archive Linux Artifacts inputs: - archiveType: tar + archiveType: zip includeRootFolder: false - tarCompression: gz - rootFolderOrFile: '$(Build.SourcesDirectory)/artifacts/Release/bin/Linux.x64' - archiveFile: '$(Build.SourcesDirectory)/artifacts/Release/sos-linux-x64.tar.gz' + rootFolderOrFile: '$(Build.SourcesDirectory)/artifacts/bin/Linux.x64.Release' + archiveFile: '$(Build.SourcesDirectory)/artifacts/packages/Release/sos-linux-x64.zip' condition: succeeded() # MacOS download, copy and archive @@ -392,26 +415,26 @@ phases: displayName: Copy MacOS Artifacts inputs: sourceFolder: '$(System.ArtifactsDirectory)/MacOS_Build_Release' - targetFolder: '$(Build.SourcesDirectory)/artifacts/Release/bin/OSX.x64' + contents: ?(libsos.dylib|libsosplugin.dylib|sosdocsunix.txt) + targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/OSX.x64.Release' condition: succeeded() - task: CopyFiles@2 displayName: Copy MacOS Artifacts inputs: - sourceFolder: '$(Build.SourcesDirectory)/artifacts/Release/bin/SOS.NETCore/netcoreapp1.0' + sourceFolder: '$(_SOSNETCorePath)' contents: 'SOS.NETCore.dll' overWrite: true - targetFolder: '$(Build.SourcesDirectory)/artifacts/Release/bin/OSX.x64' + targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/OSX.x64.Release' condition: succeeded() - task: ArchiveFiles@2 displayName: Archive MacOS Artifacts inputs: - archiveType: tar + archiveType: zip includeRootFolder: false - tarCompression: gz - rootFolderOrFile: '$(Build.SourcesDirectory)/artifacts/Release/bin/OSX.x64' - archiveFile: '$(Build.SourcesDirectory)/artifacts/Release/sos-osx-x64.tar.gz' + rootFolderOrFile: '$(Build.SourcesDirectory)/artifacts/bin/OSX.x64.Release' + archiveFile: '$(Build.SourcesDirectory)/artifacts/packages/Release/sos-osx-x64.zip' condition: succeeded() # Create nuget package and sign binaries @@ -419,33 +442,52 @@ phases: - script: $(Build.SourcesDirectory)\eng\cipack.cmd -configuration Release -prepareMachine - /p:SignType=$(SignType) - /p:PB_PublishBlobFeedKey=$(dotnetfeed-storage-access-key-1) - /p:PB_PublishBlobFeedUrl=https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json - /p:PB_PublishType=$(PublishType) + /p:TeamName=$(_TeamName) + /p:DotNetSignType=$(SignType) + /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1) + /p:DotNetPublishBlobFeedUrl=$(_PublishBlobFeedUrl) + /p:DotNetPublishToBlobFeed=false + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:OfficialBuildId=$(BUILD.BUILDNUMBER) displayName: Sign / Package - env: - OfficialBuildId: $(BUILD.BUILDNUMBER) continueOnError: true condition: succeeded() # Optionally drop the artifacts on a share - task: PublishBuildArtifacts@1 - displayName: Drop Artifacts + displayName: Drop Binaries inputs: publishLocation: FilePath - pathtoPublish: '$(Build.SourcesDirectory)\artifacts\Release' - artifactName: '$(Build.BuildNumber)' - targetPath: '$(DropRoot)\DotNetCore\$(Build.DefinitionName)' + pathtoPublish: '$(Build.SourcesDirectory)\artifacts\bin' + artifactName: bin + targetPath: '$(DropRoot)\DotNetCore\$(Build.DefinitionName)\$(Build.BuildNumber)' condition: and(succeeded(), eq(variables['DropArtifacts'], 'true')) - # Optionally publish the packages to a myget feed + - task: PublishBuildArtifacts@1 + displayName: Drop Packages + inputs: + publishLocation: FilePath + pathtoPublish: '$(Build.SourcesDirectory)\artifacts\packages' + artifactName: packages + targetPath: '$(DropRoot)\DotNetCore\$(Build.DefinitionName)\$(Build.BuildNumber)' + condition: and(succeeded(), eq(variables['DropArtifacts'], 'true')) + + # Optionally publish the packages to a blob feed - task: NuGetCommand@2 displayName: Publish TestHelpers to MyGet dotnet-buildtools feed inputs: command: custom - arguments: 'push $(Build.SourcesDirectory)\artifacts\Release\packages\Shipping\Microsoft.Diagnostic.TestHelpers.*.nupkg -ApiKey $(dotnet-myget-org-api-key) -Source $(MyGetBuildToolsFeed)' + arguments: 'push $(Build.SourcesDirectory)\artifacts\packages\Release\NonShipping\Microsoft.Diagnostic.TestHelpers.*.nupkg -ApiKey $(dotnetfeed-storage-access-key-1) -Source $(_PublishBlobFeedUrl)' condition: and(succeeded(), eq(variables['PushPackages'], 'true')) + - task: PublishBuildArtifacts@1 + displayName: Publish Logs to VSTS + inputs: + pathtoPublish: '$(Build.SourcesDirectory)/artifacts/log' + publishLocation: Container + artifactName: Logs_Packaging_Signing + continueOnError: true + condition: always() diff --git a/Directory.Build.props b/Directory.Build.props index 920036cb9..fcb0f4b2a 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,12 +1,7 @@ - - $(RepositoryUrl) - http://go.microsoft.com/fwlink/?LinkId=529443 - http://go.microsoft.com/fwlink/?LinkID=288859 - true - - + + false false false diff --git a/NuGet.Config b/NuGet.Config index 1b0fc7e28..04240c399 100644 --- a/NuGet.Config +++ b/NuGet.Config @@ -1,8 +1,12 @@ - + + + + - + + diff --git a/diagnostics.sln b/diagnostics.sln index a821688fd..486b96bf6 100644 --- a/diagnostics.sln +++ b/diagnostics.sln @@ -13,7 +13,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SOS", "SOS", "{41638A4C-0DA EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SOS.UnitTests", "src\SOS\SOS.UnitTests\SOS.UnitTests.csproj", "{1532DB3C-7DCD-45C6-B697-62B8378A16A2}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BuildTools", "eng\BuildTools.csproj", "{32F02701-A0CD-48DD-BC34-536251B90461}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CreateVersionFile", "eng\CreateVersionFile.csproj", "{365381BB-2A89-4F52-BC66-1BB4A9D514BB}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -73,18 +73,18 @@ Global {1532DB3C-7DCD-45C6-B697-62B8378A16A2}.Release|x64.Build.0 = Release|Any CPU {1532DB3C-7DCD-45C6-B697-62B8378A16A2}.Release|x86.ActiveCfg = Release|Any CPU {1532DB3C-7DCD-45C6-B697-62B8378A16A2}.Release|x86.Build.0 = Release|Any CPU - {32F02701-A0CD-48DD-BC34-536251B90461}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {32F02701-A0CD-48DD-BC34-536251B90461}.Debug|Any CPU.Build.0 = Debug|Any CPU - {32F02701-A0CD-48DD-BC34-536251B90461}.Debug|x64.ActiveCfg = Debug|Any CPU - {32F02701-A0CD-48DD-BC34-536251B90461}.Debug|x64.Build.0 = Debug|Any CPU - {32F02701-A0CD-48DD-BC34-536251B90461}.Debug|x86.ActiveCfg = Debug|Any CPU - {32F02701-A0CD-48DD-BC34-536251B90461}.Debug|x86.Build.0 = Debug|Any CPU - {32F02701-A0CD-48DD-BC34-536251B90461}.Release|Any CPU.ActiveCfg = Release|Any CPU - {32F02701-A0CD-48DD-BC34-536251B90461}.Release|Any CPU.Build.0 = Release|Any CPU - {32F02701-A0CD-48DD-BC34-536251B90461}.Release|x64.ActiveCfg = Release|Any CPU - {32F02701-A0CD-48DD-BC34-536251B90461}.Release|x64.Build.0 = Release|Any CPU - {32F02701-A0CD-48DD-BC34-536251B90461}.Release|x86.ActiveCfg = Release|Any CPU - {32F02701-A0CD-48DD-BC34-536251B90461}.Release|x86.Build.0 = Release|Any CPU + {365381BB-2A89-4F52-BC66-1BB4A9D514BB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {365381BB-2A89-4F52-BC66-1BB4A9D514BB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {365381BB-2A89-4F52-BC66-1BB4A9D514BB}.Debug|x64.ActiveCfg = Debug|Any CPU + {365381BB-2A89-4F52-BC66-1BB4A9D514BB}.Debug|x64.Build.0 = Debug|Any CPU + {365381BB-2A89-4F52-BC66-1BB4A9D514BB}.Debug|x86.ActiveCfg = Debug|Any CPU + {365381BB-2A89-4F52-BC66-1BB4A9D514BB}.Debug|x86.Build.0 = Debug|Any CPU + {365381BB-2A89-4F52-BC66-1BB4A9D514BB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {365381BB-2A89-4F52-BC66-1BB4A9D514BB}.Release|Any CPU.Build.0 = Release|Any CPU + {365381BB-2A89-4F52-BC66-1BB4A9D514BB}.Release|x64.ActiveCfg = Release|Any CPU + {365381BB-2A89-4F52-BC66-1BB4A9D514BB}.Release|x64.Build.0 = Release|Any CPU + {365381BB-2A89-4F52-BC66-1BB4A9D514BB}.Release|x86.ActiveCfg = Release|Any CPU + {365381BB-2A89-4F52-BC66-1BB4A9D514BB}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/eng/Build-Native.cmd b/eng/Build-Native.cmd index 9e92c7dc5..74b48ed2a 100644 --- a/eng/Build-Native.cmd +++ b/eng/Build-Native.cmd @@ -35,8 +35,8 @@ set ghprbCommentBody= :: __SourceDir -- default: %__ProjectDir%\src\ :: __RootBinDir -- default: %__ProjectDir%\artifacts\ :: __IntermediatesDir -- default: %__RootBinDir%\obj\%__BuildOS%.%__BuildArch.%__BuildType%\ -:: __BinDir -- default: %__RootBinDir%\%__BuildType%\bin\%__BuildOS%.%__BuildArch\ -:: __LogDir -- default: %__RootBinDir%\%__BuildType%\log\%__BuildOS%.%__BuildArch\ +:: __BinDir -- default: %__RootBinDir%\bin\%__BuildOS%.%__BuildArch.%__BuildType%\ +:: __LogDir -- default: %__RootBinDir%\log\%__BuildOS%.%__BuildArch.%__BuildType%\ :: :: Thus, these variables are not simply internal to this script! @@ -80,7 +80,7 @@ if /i "%1" == "-configuration" (set __BuildType=%2&set processedArgs=!proc if /i "%1" == "-architecture" (set __BuildArch=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) if /i "%1" == "-verbosity" (set __Verbosity=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop) :: These options are passed on to the common build script when testing -if /i "%1" == "-ci" (set __TestArgs=!__TestArgs! %1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "-ci" (set __CI=1&set __TestArgs=!__TestArgs! %1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "-solution" (set __TestArgs=!__TestArgs! %1 %2&set processedArgs=!processedArgs! %1&shift&shift&goto Arg_Loop) :: These options are ignored for a native build if /i "%1" == "-build" (set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) @@ -91,7 +91,7 @@ if /i "%1" == "-pack" (set processedArgs=!processedArgs! %1&shift if /i "%1" == "-publish" (set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "-preparemachine" (set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) -if [!processedArgs!]==[] ( +if [!processedArgs!] == [] ( set __UnprocessedBuildArgs=%__args% ) else ( set __UnprocessedBuildArgs=%__args% @@ -104,26 +104,35 @@ if [!processedArgs!]==[] ( :: Determine if this is a cross-arch build -if /i "%__BuildArch%"=="arm64" ( +if /i "%__BuildArch%" == "arm64" ( set __DoCrossArchBuild=1 set __CrossArch=x86 - ) +) -if /i "%__BuildArch%"=="arm" ( +if /i "%__BuildArch%" == "arm" ( set __DoCrossArchBuild=1 set __CrossArch=x64 +) + +if /i "%__BuildType%" == "debug" set __BuildType=Debug +if /i "%__BuildType%" == "release" set __BuildType=Release + +if "%NUGET_PACKAGES%" == "" ( + if %__CI% EQU 1 ( + set "NUGET_PACKAGES=%__ProjectDir%\.packages" + ) else ( + set "NUGET_PACKAGES=%UserProfile%\.nuget\packages" ) +) -if /i "%__BuildType%"=="debug" set __BuildType=Debug -if /i "%__BuildType%"=="release" set __BuildType=Release +echo %NUGET_PACKAGES% :: Set the remaining variables based upon the determined build configuration set "__RootBinDir=%__ProjectDir%\artifacts" -set "__ConfigBinDir=%__RootBinDir%\%__BuildType%" -set "__BinDir=%__ConfigBinDir%\bin\%__BuildOS%.%__BuildArch%" -set "__LogDir=%__ConfigBinDir%\log\%__BuildOS%.%__BuildArch%" +set "__BinDir=%__RootBinDir%\bin\%__BuildOS%.%__BuildArch%.%__BuildType%" +set "__LogDir=%__RootBinDir%\log\%__BuildOS%.%__BuildArch%.%__BuildType%" set "__IntermediatesDir=%__RootBinDir%\obj\%__BuildOS%.%__BuildArch%.%__BuildType%" -set "__PackagesBinDir=%__ConfigBinDir%\packages" +set "__PackagesBinDir=%__RootBinDir%\packages\%__BuildType%\Shipping" set "__CrossComponentBinDir=%__BinDir%" set "__CrossCompIntermediatesDir=%__IntermediatesDir%\crossgen" @@ -156,17 +165,19 @@ REM ============================================================================ @if defined _echo @echo on :: Parse the optdata package versions out of msbuild so that we can pass them on to CMake -set DotNetCli=%__ProjectDir%\.dotnet\dotnet.exe -if not exist "%DotNetCli%" ( - echo %__MsgPrefix%Assertion failed: dotnet cli not found at path "%DotNetCli%" +set __DotNetCli=%__ProjectDir%\.dotnet\dotnet.exe +if not exist "%__DotNetCli%" ( + echo %__MsgPrefix%Assertion failed: dotnet cli not found at path "%__DotNetCli%" exit /b 1 ) -set MSBuildPath=%__ProjectDir%\.dotnet\sdk\2.1.300\MSBuild.dll -if not exist "%MSBuildPath%" ( - echo %__MsgPrefix%Assertion failed: dotnet cli sdk not found at path "%MSBuildPath%" +set __MSBuildPath=%__ProjectDir%\.dotnet\sdk\2.1.401\MSBuild.dll +if not exist "%__MSBuildPath%" ( + echo %__MsgPrefix%Assertion failed: dotnet cli sdk not found at path "%__MSBuildPath%" exit /b 1 ) +set __DotNetRuntimeVersion=2.1.3 + REM ========================================================================================= REM === REM === Build the native code @@ -218,7 +229,7 @@ if %__Build% EQU 1 ( :GenVSSolution echo Generating Version Header set __GenerateVersionLog="%__LogDir%\GenerateVersion.binlog" - %DotNetCli% %MSBuildPath% %__ProjectDir%\eng\CreateVersionFile.csproj /v:!__Verbosity! /bl:!__GenerateVersionLog! /t:GenerateVersionFiles /p:GenerateVersionHeader=true /p:NativeVersionHeaderFile=%__IntermediatesDir%\_version.h /p:Configuration=%__BuildType% /p:Platform=%__BuildArch% %__UnprocessedBuildArgs% + %__DotNetCli% %__MSBuildPath% %__ProjectDir%\eng\CreateVersionFile.csproj /v:!__Verbosity! /bl:!__GenerateVersionLog! /t:GenerateVersionFiles /p:GenerateVersionHeader=true /p:NativeVersionHeaderFile=%__IntermediatesDir%\_version.h /p:Configuration=%__BuildType% /p:Platform=%__BuildArch% %__UnprocessedBuildArgs% if not !errorlevel! == 0 ( echo Generate Version Header FAILED exit /b 1 @@ -228,9 +239,9 @@ if %__Build% EQU 1 ( echo %__MsgPrefix%Regenerating the Visual Studio solution - set "__ManagedBinaryDir=%__RootBinDir%\%__BuildType%\bin" + set "__ManagedBinaryDir=%__RootBinDir%\bin" set "__ManagedBinaryDir=!__ManagedBinaryDir:\=/!" - set __ExtraCmakeArgs=!___SDKVersion! "-DCLR_MANAGED_BINARY_DIR=!__ManagedBinaryDir!" + set __ExtraCmakeArgs=!___SDKVersion! "-DCLR_MANAGED_BINARY_DIR=!__ManagedBinaryDir!" "-DCLR_BUILD_TYPE=!__BuildType!" pushd "%__IntermediatesDir%" call "%__ProjectDir%\eng\gen-buildsys-win.bat" "%__ProjectDir%" %__VSVersion% %__BuildArch% !__ExtraCmakeArgs! @@ -246,7 +257,7 @@ if %__Build% EQU 1 ( ) set __BuildLog="%__LogDir%\Native.Build.binlog" - :: For some currently unknown reason, "%DotNetCli% msbuild" fails because VCTargetsPath isn't defined. + :: For some currently unknown reason, "%__DotNetCli% msbuild" fails because VCTargetsPath isn't defined. msbuild.exe %__IntermediatesDir%\install.vcxproj /v:!__Verbosity! /bl:!__BuildLog! /p:Configuration=%__BuildType% /p:Platform=%__BuildArch% %__UnprocessedBuildArgs% if not !ERRORLEVEL! == 0 ( @@ -288,7 +299,7 @@ if /i "%__DoCrossArchBuild%"=="1" ( set "__ManagedBinaryDir=%__RootBinDir%\%__BuildType%\bin" set "__ManagedBinaryDir=!__ManagedBinaryDir:\=/!" - set __ExtraCmakeArgs="-DCLR_MANAGED_BINARY_DIR=!__ManagedBinaryDir!" "-DCLR_CROSS_COMPONENTS_BUILD=1" "-DCLR_CMAKE_TARGET_ARCH=%__BuildArch%" "-DCMAKE_SYSTEM_VERSION=10.0" + set __ExtraCmakeArgs="-DCLR_MANAGED_BINARY_DIR=!__ManagedBinaryDir!" "-DCLR_BUILD_TYPE=!__BuildType!" "-DCLR_CROSS_COMPONENTS_BUILD=1" "-DCLR_CMAKE_TARGET_ARCH=%__BuildArch%" "-DCMAKE_SYSTEM_VERSION=10.0" pushd "%__CrossCompIntermediatesDir%" call "%__ProjectDir%\eng\gen-buildsys-win.bat" "%__ProjectDir%" %__VSVersion% %__CrossArch% !__ExtraCmakeArgs! @@ -304,7 +315,7 @@ if /i "%__DoCrossArchBuild%"=="1" ( set __BuildLog="%__LogDir%\Cross.Build.binlog" - :: For some currently unknown reason, "%DotNetCli% msbuild" fails because VCTargetsPath isn't defined. + :: For some currently unknown reason, "%__DotNetCli% msbuild" fails because VCTargetsPath isn't defined. msbuild.exe %__CrossCompIntermediatesDir%\install.vcxproj /v:!__Verbosity! /bl:!__BuildLog! /p:Configuration=%__BuildType% /p:Platform=%__CrossArch% %__UnprocessedBuildArgs% if not !ERRORLEVEL! == 0 ( @@ -330,7 +341,7 @@ echo %__MsgPrefix%Product binaries are available at !__BinDir! :: Test components if %__Test% EQU 1 ( :: Install the other versions of .NET Core runtime we are going to test on - powershell -ExecutionPolicy ByPass -NoProfile -command "& """%__ProjectDir%\eng\install-test-runtimes.ps1""" -DotNetDir %__ProjectDir%\.dotnet -TempDir %__IntermediatesDir% -BuildArch %__BuildArch%" %__DailyTest% + powershell -ExecutionPolicy ByPass -NoProfile -command "& """%__ProjectDir%\eng\install-test-runtimes.ps1""" -DotNetDir %__ProjectDir%\.dotnet -RuntimeVersion21 %__DotNetRuntimeVersion% -TempDir %__IntermediatesDir% -BuildArch %__BuildArch%" %__DailyTest% :: Run the xunit tests powershell -ExecutionPolicy ByPass -NoProfile -command "& """%__ProjectDir%\eng\common\Build.ps1""" -test -configuration %__BuildType% -verbosity %__Verbosity% %__TestArgs%" diff --git a/eng/BuildTools.csproj b/eng/BuildTools.csproj deleted file mode 100644 index 01dcd1301..000000000 --- a/eng/BuildTools.csproj +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - netcoreapp2.1 - BuildTools - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/eng/CreateVersionFile.csproj b/eng/CreateVersionFile.csproj index 534a557d0..42143fdf2 100644 --- a/eng/CreateVersionFile.csproj +++ b/eng/CreateVersionFile.csproj @@ -1,60 +1,88 @@ - + + + - $(BUILD_BUILDNUMBER) + netcoreapp2.0 - - + - - $(BuildVersionFilePath)BuildVersion2-$(TodayTimeStamp).props - true - + + + + + + + + + + + + + - + + - + + + - + - + + + - - + + + + + + + - $(MajorVersion).$(MinorVersion).$(_BuildNumberFiveDigitDateStamp).$(_BuildNumberBuildOfTheDayPadded) - $(_BuildNumberFiveDigitDateStamp) - $(_BuildNumberBuildOfTheDayPadded) - - - - - - - $(FileVersion) - $(BuildNumberMajor) - $(BuildNumberMinor) - - -]]> - + $(USERNAME) - + + + + + + + $(COMPUTERNAME) + - + + + - + + $(BuiltByString) %40BuiltBy: $(VersionUserName)-$(VersionHostName) + + + + + + + + + + + + + + + + + + + + + + diff --git a/eng/SignToolData.json b/eng/SignToolData.json deleted file mode 100644 index 1592fa20d..000000000 --- a/eng/SignToolData.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "sign": [ - { - "certificate": "MicrosoftSHA2", - "strongName": "MsSharedLib72", - "values": [ - "bin/Microsoft.Diagnostic.TestHelpers/netcoreapp2.0/Microsoft.Diagnostic.TestHelpers.dll", - "bin/SOS.NETCore/netcoreapp1.0/SOS.NETCore.dll", - ] - }, - { - "certificate": "MicrosoftSHA2", - "values": [ - "bin/Windows_NT.x64/sos.dll", - "bin/Windows_NT.x86/sos.dll" - ] - }, - { - "certificate": "NuGet", - "strongName": null, - "values": [ - "packages/Shipping/*.nupkg" - ] - }, - { - "certificate": null, - "strongName": null, - "values": [ - "*.zip" - ] - } - ], - "exclude": [ - "Microsoft.FileFormats.dll", - "Microsoft.SymbolStore.dll", - "System.Collections.Immutable.dll", - "System.Net.Http.dll", - "System.Reflection.Metadata.dll" - ] -} diff --git a/eng/Signing.props b/eng/Signing.props new file mode 100644 index 000000000..cc1142955 --- /dev/null +++ b/eng/Signing.props @@ -0,0 +1,9 @@ + + + $(MSBuildAllProjects);$(MSBuildThisFileFullPath) + + + + + + diff --git a/eng/Versions.props b/eng/Versions.props index c8e1daacb..587a7c52f 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -6,20 +6,24 @@ 1.0.0 beta - + + + false + true + + 2.2.0-preview1-03108-01 - 1.0.0-beta2-63127-01 2.8.0-beta6-62830-08 - true - false 1.5.0 + 4.3.0 $(RestoreSources); + https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json; https://dotnet.myget.org/F/dotnet-core/api/v3/index.json; https://dotnet.myget.org/F/dotnet-buildtools/api/v3/index.json diff --git a/eng/build-native.sh b/eng/build-native.sh index 345260381..1918c23a6 100755 --- a/eng/build-native.sh +++ b/eng/build-native.sh @@ -30,6 +30,7 @@ __Build=false __Test=false __DailyTest= __CI=false +__Verbosity=minimal __TestArgs= __UnprocessedBuildArgs= @@ -178,6 +179,7 @@ while :; do ;; --verbosity) + __Verbosity=$2 __TestArgs="$__TestArgs $1 $2" shift ;; @@ -227,11 +229,6 @@ while :; do __ClangMinorVersion=0 ;; - --verbosity) - __TestArgs="$__TestArgs --verbosity $2" - shift - ;; - # Ignored for a native build --build|--rebuild|--sign|--restore|--pack|--publish|--preparemachine) ;; @@ -281,15 +278,15 @@ fi echo $PYTHON __RootBinDir=$__ProjectRoot/artifacts -__ConfigBinDir=$__RootBinDir/$__BuildType -__BinDir=$__ConfigBinDir/bin/$__BuildOS.$__BuildArch -__LogDir=$__ConfigBinDir/log/$__BuildOS.$__BuildArch +__BinDir=$__RootBinDir/bin/$__BuildOS.$__BuildArch.$__BuildType +__LogDir=$__RootBinDir/log/$__BuildOS.$__BuildArch.$__BuildType __IntermediatesDir=$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType -__ResultsDir=$__ConfigBinDir/TestResults -__PackagesBinDir=$__ConfigBinDir/packages -__ExtraCmakeArgs=-DCLR_MANAGED_BINARY_DIR=$__ConfigBinDir/bin +__ResultsDir=$__RootBinDir/TestResults/$__BuildType +__PackagesBinDir=$__RootBinDir/packages/$__BuildType/Shipping +__ExtraCmakeArgs="-DCLR_MANAGED_BINARY_DIR=$__RootBinDir/bin -DCLR_BUILD_TYPE=$__BuildType" __DotNetCli=$__ProjectRoot/.dotnet/dotnet -__MSBuildPath=$__ProjectRoot/.dotnet/sdk/2.1.300/MSBuild.dll +__MSBuildPath=$__ProjectRoot/.dotnet/sdk/2.1.401/MSBuild.dll +__DotNetRuntimeVersion=2.1.3 if [ ! -e $__DotNetCli ]; then echo "dotnet cli not installed $__DotNetCli" @@ -461,7 +458,7 @@ if [ $__Build == true ]; then if [[ $__CI == true ]]; then echo "Generating Version Source File" __GenerateVersionLog="$__LogDir/GenerateVersion.binlog" - $__DotNetCli $__MSBuildPath $__ProjectRoot/eng/CreateVersionFile.csproj /noconlog /bl:$__GenerateVersionLog /t:GenerateVersionFiles /p:GenerateVersionSourceFile=true /p:NativeVersionSourceFile="$__IntermediatesDir/version.cpp" /p:Configuration="$__BuildType" /p:Platform="$__BuildArch" $__UnprocessedBuildArgs + $__DotNetCli $__MSBuildPath $__ProjectRoot/eng/CreateVersionFile.csproj /v:$__Verbosity /bl:$__GenerateVersionLog /t:GenerateVersionFiles /p:GenerateVersionSourceFile=true /p:NativeVersionSourceFile="$__IntermediatesDir/version.cpp" /p:Configuration="$__BuildType" /p:Platform="$__BuildArch" $__UnprocessedBuildArgs if [ $? != 0 ]; then echo "Generating Version Source File FAILED" exit 1 @@ -477,7 +474,7 @@ fi # Run SOS/lldbplugin tests if [ $__Test == true ]; then # Install the other versions of .NET Core runtime we are going to test on - "$__ProjectRoot/eng/install-test-runtimes.sh" --dotnet-directory "$__ProjectRoot/.dotnet" --temp-directory "$__IntermediatesDir" --architecture "$__BuildArch" $__DailyTest + "$__ProjectRoot/eng/install-test-runtimes.sh" --dotnet-directory "$__ProjectRoot/.dotnet" --runtime-version-21 "$__DotNetRuntimeVersion" --temp-directory "$__IntermediatesDir" --architecture "$__BuildArch" $__DailyTest if [ "$LLDB_PATH" = "" ]; then export LLDB_PATH="$(which lldb-3.9.1 2> /dev/null)" @@ -514,7 +511,7 @@ if [ $__Test == true ]; then fi # Run lldb python tests - "$__ProjectRoot/src/SOS/lldbplugin.tests/testsos.sh" "$__ProjectRoot" "$__Plugin" "$__ConfigBinDir/bin" "$__ResultsDir" + "$__ProjectRoot/src/SOS/lldbplugin.tests/testsos.sh" "$__ProjectRoot" "$__Plugin" "$__DotNetRuntimeVersion" "$__RootBinDir/bin/TestDebuggee/$__BuildType/netcoreapp2.0/TestDebuggee.dll" "$__ResultsDir" if [ $? != 0 ]; then exit 1 fi diff --git a/eng/build.yml b/eng/build.yml index 9cbc2bfb0..fa582628e 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -9,34 +9,51 @@ # variables YAML object - https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-schema.md#phase parameters: - phaseName: '' + name: '' agentOs: Windows_NT dockerImage: '' testOnly: false dependsOn: '' + # send telemetry enableTelemetry: true + # install Microbuild plugin if not a public build + enableMicrobuild: true + # queue YAML object - https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-schema.md#queue queue: {} + # variables YAML object - https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-schema.md#phase variables: {} +# Common conditionals: There are a number of common conditionals that are useful. Generally these are used to decide what resources can be accessed, +# or what logic should be applied based on the context the build is being run in. +# - eq/ne(variables['Agent.Os'], 'Windows_NT') - Running/not running on a windows machine +# - eq/ne(variables['System.TeamProject'], 'public') - Running/not running on the dotnet public VSTS project +# - and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest') - Not running in public and not a pull request. +# - or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest') - Running in public or a pull request. + phases: - template: /eng/common/templates/phases/base.yml parameters: - name: ${{ parameters.phaseName }} + enableTelemetry: ${{ parameters.enableTelemetry }} + + enableMicrobuild: ${{ parameters.enableMicrobuild }} + + name: ${{ parameters.name }} agentOs: ${{ parameters.agentOs }} dependsOn: ${{ parameters.dependsOn }} - enableTelemetry: ${{ parameters.enableTelemetry }} queue: ${{ parameters.queue }} variables: ${{ insert }}: ${{ parameters.variables }} _PortableLinuxBuild: CentOS_7_Build_$(_BuildConfig) _DockerImageName: ${{ parameters.dockerImage }} - _PhaseName : ${{ parameters.phaseName }} + _PhaseName : ${{ parameters.name }} _HelixType: build/product _HelixBuildConfig: $(_BuildConfig) - ${{ if ne(variables['System.TeamProject'], 'public') }}: + # Only enable publishing in non-public, non PR scenarios. + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _HelixSource: official/dotnet/arcade/$(Build.SourceBranch) - ${{ if eq(variables['System.TeamProject'], 'public') }}: + # else + ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: _HelixSource: pr/dotnet/arcade/$(Build.SourceBranch) ${{ if eq(parameters.testOnly, 'true') }}: _LinuxScript: $(Build.SourcesDirectory)/eng/citest.sh @@ -51,6 +68,13 @@ phases: _DailyTest: '' steps: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: AzureKeyVault@1 + inputs: + azureSubscription: 'DncEng-VSTS' + KeyVaultName: EngKeyVault + SecretsFilter: 'dotnetfeed-storage-access-key-1,microsoft-symbol-server-pat,symweb-symbol-server-pat' + # Internal only build steps - template: /eng/common/templates/steps/run-on-windows.yml parameters: @@ -60,9 +84,8 @@ phases: -configuration $(_BuildConfig) -architecture $(_BuildArch) -prepareMachine + /p:OfficialBuildId=$(BUILD.BUILDNUMBER) displayName: Build / Test - env: - OfficialBuildId: $(BUILD.BUILDNUMBER) condition: succeeded() - ${{ if eq(parameters.agentOs, 'Linux') }}: @@ -78,7 +101,7 @@ phases: displayName: Place Linux-x64 Artifacts inputs: sourceFolder: '$(System.ArtifactsDirectory)/$(_PortableLinuxBuild)' - targetFolder: '$(Build.SourcesDirectory)/artifacts/$(_BuildConfig)/bin/Linux.x64' + targetFolder: '$(Build.SourcesDirectory)/artifacts/bin/Linux.x64.$(_BuildConfig)' condition: succeeded() - script: $(Build.SourcesDirectory)/eng/docker-build.sh @@ -89,10 +112,9 @@ phases: --configuration $(_BuildConfig) --architecture $(_BuildArch) --prepareMachine + /p:OfficialBuildId=$(BUILD.BUILDNUMBER) /p:BUILD_BUILDNUMBER=$(BUILD.BUILDNUMBER) displayName: Docker Build / Test - env: - OfficialBuildId: $(BUILD.BUILDNUMBER) condition: succeeded() - ${{ if eq(parameters.agentOs, 'Darwin') }}: @@ -100,9 +122,8 @@ phases: --configuration $(_BuildConfig) --architecture $(_BuildArch) --prepareMachine + /p:OfficialBuildId=$(BUILD.BUILDNUMBER) displayName: Build / Test - env: - OfficialBuildId: $(BUILD.BUILDNUMBER) condition: succeeded() # Internal only build steps (until publishing artifacts in public CI is supported) @@ -110,22 +131,15 @@ phases: - task: PublishBuildArtifacts@1 displayName: Upload Artifacts inputs: - pathtoPublish: '$(Build.SourcesDirectory)/artifacts/$(_BuildConfig)/$(_PublishArtifacts)' + pathtoPublish: '$(Build.SourcesDirectory)/artifacts/$(_PublishArtifacts)' artifactName: $(_PhaseName)_$(Agent.JobName) condition: and(succeeded(), ne(variables['_PublishArtifacts'], '')) - task: CopyFiles@2 displayName: Gather Build Logs inputs: - sourceFolder: '$(Build.SourcesDirectory)/artifacts/$(_BuildConfig)/log' - targetFolder: '$(Build.StagingDirectory)/BuildLogs' - continueOnError: true - condition: always() - - - task: CopyFiles@2 - displayName: Gather Test Logs - inputs: - sourceFolder: '$(Build.SourcesDirectory)/artifacts/$(_BuildConfig)/TestResults' + sourceFolder: '$(Build.SourcesDirectory)/artifacts' + contents: '?(log|TestResults)/**' targetFolder: '$(Build.StagingDirectory)/BuildLogs' continueOnError: true condition: always() @@ -133,7 +147,8 @@ phases: - task: PublishBuildArtifacts@1 displayName: Publish Logs to VSTS inputs: - pathtoPublish: '$(Build.StagingDirectory)/BuildLogs' - artifactName: Logs_$(_PhaseName)_$(Agent.JobName) + PathtoPublish: '$(Build.StagingDirectory)/BuildLogs' + PublishLocation: Container + ArtifactName: Logs_$(_PhaseName)_$(Agent.JobName) continueOnError: true condition: always() diff --git a/eng/cipack.cmd b/eng/cipack.cmd index b26c4a421..a06ea3ce4 100644 --- a/eng/cipack.cmd +++ b/eng/cipack.cmd @@ -1,3 +1,3 @@ @echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0common\Build.ps1""" -restore -sign -pack %*" +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0common\Build.ps1""" -restore -sign -pack -ci %*" exit /b %ErrorLevel% diff --git a/eng/common/PublishBuildAssets.cmd b/eng/common/PublishBuildAssets.cmd new file mode 100644 index 000000000..4f9d54d02 --- /dev/null +++ b/eng/common/PublishBuildAssets.cmd @@ -0,0 +1,3 @@ +@echo off +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -publishBuildAssets %*" +exit /b %ErrorLevel% diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index c6bebef2f..928902094 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -15,6 +15,7 @@ Param( [switch] $sign, [switch] $pack, [switch] $publish, + [switch] $publishBuildAssets, [switch] $ci, [switch] $prepareMachine, [switch] $help, @@ -45,6 +46,7 @@ function Print-Usage() { Write-Host " -performanceTest Run all performance tests in the solution" Write-Host " -sign Sign build outputs" Write-Host " -publish Publish artifacts (e.g. symbols)" + Write-Host " -publishBuildAssets Push assets to BAR" Write-Host "" Write-Host "Advanced settings:" @@ -81,14 +83,14 @@ function InitializeDotNetCli { # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version, # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues. - if (($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$($GlobalJson.sdk.version)"))) { + if (($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$($GlobalJson.tools.dotnet)"))) { $dotnetRoot = $env:DOTNET_INSTALL_DIR } else { $dotnetRoot = Join-Path $RepoRoot ".dotnet" $env:DOTNET_INSTALL_DIR = $dotnetRoot if ($restore) { - InstallDotNetSdk $dotnetRoot $GlobalJson.sdk.version + InstallDotNetSdk $dotnetRoot $GlobalJson.tools.dotnet } } @@ -132,7 +134,7 @@ function InitializeVisualStudioBuild { } function LocateVisualStudio { - $vswhereVersion = $GlobalJson.vswhere.version + $vswhereVersion = $GlobalJson.tools.vswhere $toolsRoot = Join-Path $RepoRoot ".tools" $vsWhereDir = Join-Path $toolsRoot "vswhere\$vswhereVersion" $vsWhereExe = Join-Path $vsWhereDir "vswhere.exe" @@ -154,7 +156,9 @@ function LocateVisualStudio { } function GetBuildCommand() { - if ((Get-Member -InputObject $GlobalJson -Name "sdk") -ne $null) { + $tools = $GlobalJson.tools + + if ((Get-Member -InputObject $tools -Name "dotnet") -ne $null) { $dotnetRoot = InitializeDotNetCli # by default build with dotnet cli: @@ -162,7 +166,7 @@ function GetBuildCommand() { $buildArgs = "msbuild" } - if ((Get-Member -InputObject $GlobalJson -Name "vswhere") -ne $null) { + if ((Get-Member -InputObject $tools -Name "vswhere") -ne $null) { $vsInstallDir = InitializeVisualStudioBuild # Presence of vswhere.version indicates the repo needs to build using VS msbuild: @@ -171,7 +175,7 @@ function GetBuildCommand() { } if ($buildDriver -eq $null) { - Write-Host "/global.json must either specify 'sdk.version' or 'vswhere.version'." -ForegroundColor Red + Write-Host "/global.json must either specify 'tools.dotnet' or 'tools.vswhere'." -ForegroundColor Red exit 1 } @@ -183,7 +187,7 @@ function GetBuildCommand() { } function InitializeToolset([string] $buildDriver, [string]$buildArgs) { - $toolsetVersion = $GlobalJson.'msbuild-sdks'.'RoslynTools.RepoToolset' + $toolsetVersion = $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk' $toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt" if (Test-Path $toolsetLocationFile) { @@ -201,7 +205,7 @@ function InitializeToolset([string] $buildDriver, [string]$buildArgs) { $proj = Join-Path $ToolsetDir "restore.proj" - '' | Set-Content $proj + '' | Set-Content $proj & $buildDriver $buildArgs $proj /t:__WriteToolsetLocation /m /nologo /clp:None /warnaserror /bl:$ToolsetRestoreLog /v:$verbosity /p:__ToolsetLocationOutputFile=$toolsetLocationFile if ($lastExitCode -ne 0) { @@ -249,6 +253,8 @@ function Build([string] $buildDriver, [string]$buildArgs) { /p:PerformanceTest=$performanceTest ` /p:Sign=$sign ` /p:Publish=$publish ` + /p:PublishBuildAssets=$publishBuildAssets ` + /p:ContinuousIntegrationBuild=$ci ` /p:CIBuild=$ci ` $properties @@ -270,10 +276,10 @@ try { $EngRoot = Join-Path $PSScriptRoot ".." $ArtifactsDir = Join-Path $RepoRoot "artifacts" $ToolsetDir = Join-Path $ArtifactsDir "toolset" - $LogDir = Join-Path (Join-Path $ArtifactsDir $configuration) "log" + $LogDir = Join-Path (Join-Path $ArtifactsDir "log") $configuration $BuildLog = Join-Path $LogDir "Build.binlog" $ToolsetRestoreLog = Join-Path $LogDir "ToolsetRestore.binlog" - $TempDir = Join-Path (Join-Path $ArtifactsDir $configuration) "tmp" + $TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration $GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json if ($projects -eq "") { diff --git a/eng/common/build.sh b/eng/common/build.sh index 22a3b69a4..8872980e1 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -128,22 +128,18 @@ eng_root="$repo_root/eng" artifacts_dir="$repo_root/artifacts" artifacts_configuration_dir="$artifacts_dir/$configuration" toolset_dir="$artifacts_dir/toolset" -log_dir="$artifacts_configuration_dir/log" +log_dir="$artifacts_dir/log/$configuration" build_log="$log_dir/Build.binlog" toolset_restore_log="$log_dir/ToolsetRestore.binlog" -temp_dir="$artifacts_configuration_dir/tmp" +temp_dir="$artifacts_dir/tmp/$configuration" global_json_file="$repo_root/global.json" build_driver="" toolset_build_proj="" -# ReadJson [filename] [json key] -# Result: Sets 'readjsonvalue' to the value of the provided json key -# Note: this method may return unexpected results if there are duplicate -# keys in the json -function ReadJson { - local file=$1 - local key=$2 +# ReadVersionFromJson [json key] +function ReadGlobalVersion { + local key=$1 local unamestr="$(uname)" local sedextended='-r' @@ -151,11 +147,14 @@ function ReadJson { sedextended='-E' fi; - readjsonvalue="$(grep -m 1 "\"$key\"" $file | sed $sedextended 's/^ *//;s/.*: *"//;s/",?//')" - if [[ ! "$readjsonvalue" ]]; then - echo "Error: Cannot find \"$key\" in $file" >&2; + local version="$(grep -m 1 "\"$key\"" $global_json_file | sed $sedextended 's/^ *//;s/.*: *"//;s/",?//')" + if [[ ! "$version" ]]; then + echo "Error: Cannot find \"$key\" in $global_json_file" >&2; ExitWithExitCode 1 fi; + + # return value + echo "$version" } function InitializeDotNetCli { @@ -170,8 +169,8 @@ function InitializeDotNetCli { export DOTNET_INSTALL_DIR="$DotNetCoreSdkDir" fi - ReadJson "$global_json_file" "version" - local dotnet_sdk_version="$readjsonvalue" + + local dotnet_sdk_version=`ReadGlobalVersion "dotnet"` local dotnet_root="" # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version, @@ -226,8 +225,7 @@ function GetDotNetInstallScript { } function InitializeToolset { - ReadJson $global_json_file "RoslynTools.RepoToolset" - local toolset_version=$readjsonvalue + local toolset_version=`ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk"` local toolset_location_file="$toolset_dir/$toolset_version.txt" if [[ -a "$toolset_location_file" ]]; then @@ -245,7 +243,7 @@ function InitializeToolset { local proj="$toolset_dir/restore.proj" - echo '' > $proj + echo '' > $proj "$build_driver" msbuild $proj /t:__WriteToolsetLocation /m /nologo /clp:None /warnaserror /bl:$toolset_restore_log /v:$verbosity /p:__ToolsetLocationOutputFile=$toolset_location_file local lastexitcode=$? @@ -288,6 +286,7 @@ function Build { /p:PerformanceTest=$performance_test \ /p:Sign=$sign \ /p:Publish=$publish \ + /p:ContinuousIntegrationBuild=$ci \ /p:CIBuild=$ci \ $properties local lastexitcode=$? diff --git a/eng/common/helixpublish.proj b/eng/common/helixpublish.proj new file mode 100644 index 000000000..4496c23f8 --- /dev/null +++ b/eng/common/helixpublish.proj @@ -0,0 +1,9 @@ + + + + + $(WorkItemDirectory) + $(WorkItemCommand) + + + diff --git a/eng/common/init-tools-native.cmd b/eng/common/init-tools-native.cmd new file mode 100644 index 000000000..438cd548c --- /dev/null +++ b/eng/common/init-tools-native.cmd @@ -0,0 +1,3 @@ +@echo off +powershell -NoProfile -NoLogo -ExecutionPolicy ByPass -command "& """%~dp0init-tools-native.ps1""" %*" +exit /b %ErrorLevel% \ No newline at end of file diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 new file mode 100644 index 000000000..45f4082bb --- /dev/null +++ b/eng/common/init-tools-native.ps1 @@ -0,0 +1,129 @@ +<# +.SYNOPSIS +Entry point script for installing native tools + +.DESCRIPTION +Reads $RepoRoot\global.json file to determine native assets to install +and executes installers for those tools + +.PARAMETER BaseUri +Base file directory or Url from which to acquire tool archives + +.PARAMETER InstallDirectory +Directory to install native toolset. This is a command-line override for the default +Install directory precedence order: +- InstallDirectory command-line override +- NETCOREENG_INSTALL_DIRECTORY environment variable +- (default) %USERPROFILE%/.netcoreeng/native + +.PARAMETER Clean +Switch specifying to not install anything, but cleanup native asset folders + +.PARAMETER Force +Clean and then install tools + +.PARAMETER DownloadRetries +Total number of retry attempts + +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds + +.PARAMETER GlobalJsonFile +File path to global.json file + +.NOTES +#> +[CmdletBinding(PositionalBinding=$false)] +Param ( + [string] $BaseUri = "https://netcorenativeassets.blob.core.windows.net/resource-packages/external", + [string] $InstallDirectory, + [switch] $Clean = $False, + [switch] $Force = $False, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30, + [string] $GlobalJsonFile = "$PSScriptRoot\..\..\global.json" +) + +Set-StrictMode -version 2.0 +$ErrorActionPreference="Stop" + +Import-Module -Name (Join-Path $PSScriptRoot "native\CommonLibrary.psm1") + +try { + # Define verbose switch if undefined + $Verbose = $VerbosePreference -Eq "Continue" + + $EngCommonBaseDir = Join-Path $PSScriptRoot "native\" + $NativeBaseDir = $InstallDirectory + if (!$NativeBaseDir) { + $NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory + } + $Env:CommonLibrary_NativeInstallDir = $NativeBaseDir + $InstallBin = Join-Path $NativeBaseDir "bin" + + # Process tools list + Write-Host "Processing $GlobalJsonFile" + If (-Not (Test-Path $GlobalJsonFile)) { + Write-Host "Unable to find '$GlobalJsonFile'" + exit 0 + } + $NativeTools = Get-Content($GlobalJsonFile) -Raw | + ConvertFrom-Json | + Select-Object -Expand "native-tools" -ErrorAction SilentlyContinue + if ($NativeTools) { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $InstallerFilename = "install-$ToolName.ps1" + $LocalInstallerCommand = Join-Path $EngCommonBaseDir $InstallerFilename + $LocalInstallerCommand += " -InstallPath $InstallBin" + $LocalInstallerCommand += " -BaseUri $BaseUri" + $LocalInstallerCommand += " -CommonLibraryDirectory $EngCommonBaseDir" + $LocalInstallerCommand += " -Version $ToolVersion" + + if ($Verbose) { + $LocalInstallerCommand += " -Verbose" + } + if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { + if($Force) { + $LocalInstallerCommand += " -Force" + } + } + if ($Clean) { + $LocalInstallerCommand += " -Clean" + } + + Write-Verbose "Installing $ToolName version $ToolVersion" + Write-Verbose "Executing '$LocalInstallerCommand'" + Invoke-Expression "$LocalInstallerCommand" + if ($LASTEXITCODE -Ne "0") { + Write-Error "Execution failed" + exit 1 + } + } + } + else { + Write-Host "No native tools defined in global.json" + exit 0 + } + + if ($Clean) { + exit 0 + } + if (Test-Path $InstallBin) { + Write-Host "Native tools are available from" (Convert-Path -Path $InstallBin) + if ($env:BUILD_BUILDNUMBER) { + Write-Host "##vso[task.prependpath]" (Convert-Path -Path $InstallBin) + } + } + else { + Write-Error "Native tools install directory does not exist, installation failed" + exit 1 + } + exit 0 +} +catch { + Write-Host $_ + Write-Host $_.Exception + exit 1 +} diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh new file mode 100644 index 000000000..54b70f678 --- /dev/null +++ b/eng/common/init-tools-native.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +base_uri='https://netcorenativeassets.blob.core.windows.net/resource-packages/external' +install_directory='' +clean=false +force=false +download_retries=5 +retry_wait_time_seconds=30 +global_json_file="${scriptroot}/../../global.json" +declare -A native_assets + +. $scriptroot/native/common-library.sh + +while (($# > 0)); do + lowerI="$(echo $1 | awk '{print tolower($0)}')" + case $lowerI in + --baseuri) + base_uri=$2 + shift 2 + ;; + --installdirectory) + install_directory=$2 + shift 2 + ;; + --clean) + clean=true + shift 1 + ;; + --force) + force=true + shift 1 + ;; + --downloadretries) + download_retries=$2 + shift 2 + ;; + --retrywaittimeseconds) + retry_wait_time_seconds=$2 + shift 2 + ;; + --help) + echo "Common settings:" + echo " --installdirectory Directory to install native toolset." + echo " This is a command-line override for the default" + echo " Install directory precedence order:" + echo " - InstallDirectory command-line override" + echo " - NETCOREENG_INSTALL_DIRECTORY environment variable" + echo " - (default) %USERPROFILE%/.netcoreeng/native" + echo "" + echo " --clean Switch specifying not to install anything, but cleanup native asset folders" + echo " --force Clean and then install tools" + echo " --help Print help and exit" + echo "" + echo "Advanced settings:" + echo " --baseuri Base URI for where to download native tools from" + echo " --downloadretries Number of times a download should be attempted" + echo " --retrywaittimeseconds Wait time between download attempts" + echo "" + exit 0 + ;; + esac +done + +function ReadGlobalJsonNativeTools { + # Get the native-tools section from the global.json. + local native_tools_section=$(cat $global_json_file | awk '/"native-tools"/,/}/') + # Only extract the contents of the object. + local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}') + native_tools_list=${native_tools_list//[\" ]/} + native_tools_list=${native_tools_list//,/$'\n'} + + local old_IFS=$IFS + while read -r line; do + # Lines are of the form: 'tool:version' + IFS=: + while read -r key value; do + native_assets[$key]=$value + done <<< "$line" + done <<< "$native_tools_list" + IFS=$old_IFS + + return 0; +} + +native_base_dir=$install_directory +if [[ -z $install_directory ]]; then + native_base_dir=$(GetNativeInstallDirectory) +fi + +install_bin="${native_base_dir}/bin" + +ReadGlobalJsonNativeTools + +if [[ ${#native_assets[@]} -eq 0 ]]; then + echo "No native tools defined in global.json" + exit 0; +else + native_installer_dir="$scriptroot/native" + for tool in "${!native_assets[@]}" + do + tool_version=${native_assets[$tool]} + installer_name="install-$tool.sh" + installer_command="$native_installer_dir/$installer_name" + installer_command+=" --baseuri $base_uri" + installer_command+=" --installpath $install_bin" + installer_command+=" --version $tool_version" + + if [[ $force = true ]]; then + installer_command+=" --force" + fi + + if [[ $clean = true ]]; then + installer_command+=" --clean" + fi + + echo "Installing $tool version $tool_version" + echo "Executing '$installer_command'" + $installer_command + + if [[ $? != 0 ]]; then + echo "Execution Failed" >&2 + exit 1 + fi + done +fi + +if [[ ! -z $clean ]]; then + exit 0 +fi + +if [[ -d $install_bin ]]; then + echo "Native tools are available from $install_bin" + if [[ !-z BUILD_BUILDNUMBER ]]; then + echo "##vso[task.prependpath]$install_bin" + fi +else + echo "Native tools install directory does not exist, installation failed" >&2 + exit 1 +fi + +exit 0 + diff --git a/eng/common/is-vsts.ps1 b/eng/common/is-vsts.ps1 deleted file mode 100644 index f88472955..000000000 --- a/eng/common/is-vsts.ps1 +++ /dev/null @@ -1,6 +0,0 @@ -if ($env:Build_BuildNumber) { - return $true -} -else { - return $false -} diff --git a/eng/common/is-vsts.sh b/eng/common/is-vsts.sh deleted file mode 100644 index f7086fae0..000000000 --- a/eng/common/is-vsts.sh +++ /dev/null @@ -1,5 +0,0 @@ -if [ ! -z $BUILD_BUILDNUMBER ]; then - exit 0 -else - exit 1 -fi diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 new file mode 100644 index 000000000..18fd261f1 --- /dev/null +++ b/eng/common/native/CommonLibrary.psm1 @@ -0,0 +1,351 @@ +<# +.SYNOPSIS +Helper module to install an archive to a directory + +.DESCRIPTION +Helper module to download and extract an archive to a specified directory + +.PARAMETER Uri +Uri of artifact to download + +.PARAMETER InstallDirectory +Directory to extract artifact contents to + +.PARAMETER Force +Force download / extraction if file or contents already exist. Default = False + +.PARAMETER DownloadRetries +Total number of retry attempts. Default = 5 + +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds. Default = 30 + +.NOTES +Returns False if download or extraction fail, True otherwise +#> +function DownloadAndExtract { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $Uri, + [Parameter(Mandatory=$True)] + [string] $InstallDirectory, + [switch] $Force = $False, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30 + ) + # Define verbose switch if undefined + $Verbose = $VerbosePreference -Eq "Continue" + + $TempToolPath = CommonLibrary\Get-TempPathFilename -Path $Uri + + # Download native tool + $DownloadStatus = CommonLibrary\Get-File -Uri $Uri ` + -Path $TempToolPath ` + -DownloadRetries $DownloadRetries ` + -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` + -Force:$Force ` + -Verbose:$Verbose + + if ($DownloadStatus -Eq $False) { + Write-Error "Download failed" + return $False + } + + # Extract native tool + $UnzipStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath ` + -OutputDirectory $InstallDirectory ` + -Force:$Force ` + -Verbose:$Verbose + + if ($UnzipStatus -Eq $False) { + Write-Error "Unzip failed" + return $False + } + return $True +} + +<# +.SYNOPSIS +Download a file, retry on failure + +.DESCRIPTION +Download specified file and retry if attempt fails + +.PARAMETER Uri +Uri of file to download. If Uri is a local path, the file will be copied instead of downloaded + +.PARAMETER Path +Path to download or copy uri file to + +.PARAMETER Force +Overwrite existing file if present. Default = False + +.PARAMETER DownloadRetries +Total number of retry attempts. Default = 5 + +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds Default = 30 + +#> +function Get-File { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $Uri, + [Parameter(Mandatory=$True)] + [string] $Path, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30, + [switch] $Force = $False + ) + $Attempt = 0 + + if ($Force) { + if (Test-Path $Path) { + Remove-Item $Path -Force + } + } + if (Test-Path $Path) { + Write-Host "File '$Path' already exists, skipping download" + return $True + } + + $DownloadDirectory = Split-Path -ErrorAction Ignore -Path "$Path" -Parent + if (-Not (Test-Path $DownloadDirectory)) { + New-Item -path $DownloadDirectory -force -itemType "Directory" | Out-Null + } + + if (Test-Path -IsValid -Path $Uri) { + Write-Verbose "'$Uri' is a file path, copying file to '$Path'" + Copy-Item -Path $Uri -Destination $Path + return $? + } + else { + Write-Verbose "Downloading $Uri" + while($Attempt -Lt $DownloadRetries) + { + try { + Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $Path + Write-Verbose "Downloaded to '$Path'" + return $True + } + catch { + $Attempt++ + if ($Attempt -Lt $DownloadRetries) { + $AttemptsLeft = $DownloadRetries - $Attempt + Write-Warning "Download failed, $AttemptsLeft attempts remaining, will retry in $RetryWaitTimeInSeconds seconds" + Start-Sleep -Seconds $RetryWaitTimeInSeconds + } + else { + Write-Error $_ + Write-Error $_.Exception + } + } + } + } + + return $False +} + +<# +.SYNOPSIS +Generate a shim for a native tool + +.DESCRIPTION +Creates a wrapper script (shim) that passes arguments forward to native tool assembly + +.PARAMETER ShimPath +Path to shim file + +.PARAMETER ToolFilePath +Path to file that shim forwards to + +.PARAMETER Force +Replace shim if already present. Default = False + +.NOTES +Returns $True if generating shim succeeds, $False otherwise +#> +function New-ScriptShim { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $ShimPath, + [Parameter(Mandatory=$True)] + [string] $ToolFilePath, + [switch] $Force + ) + try { + Write-Verbose "Generating '$ShimPath' shim" + + if ((Test-Path $ShimPath) -And (-Not $Force)) { + Write-Error "$ShimPath already exists" + return $False + } + + if (-Not (Test-Path $ToolFilePath)){ + Write-Error "Specified tool file path '$ToolFilePath' does not exist" + return $False + } + + $ShimContents = "@echo off`n" + $ShimContents += "setlocal enableextensions enabledelayedexpansion`n" + $ShimContents += "set SHIMARGS=`n" + $ShimContents += "for %%x in (%*) do (set SHIMARGS=!SHIMARGS! `"%%~x`")`n" + $ShimContents += "`"$ToolFilePath`" %SHIMARGS%`n" + $ShimContents += "endlocal" + + # Write shim file + $ShimContents | Out-File $ShimPath -Encoding "ASCII" + + if (-Not $?) { + Write-Error "Failed to generate shim" + return $False + } + return $True + } + catch { + Write-Host $_ + Write-Host $_.Exception + return $False + } +} + +<# +.SYNOPSIS +Returns the machine architecture of the host machine + +.NOTES +Returns 'x64' on 64 bit machines + Returns 'x86' on 32 bit machines +#> +function Get-MachineArchitecture { + $ProcessorArchitecture = $Env:PROCESSOR_ARCHITECTURE + $ProcessorArchitectureW6432 = $Env:PROCESSOR_ARCHITEW6432 + if($ProcessorArchitecture -Eq "X86") + { + if(($ProcessorArchitectureW6432 -Eq "") -Or + ($ProcessorArchitectureW6432 -Eq "X86")) { + return "x86" + } + $ProcessorArchitecture = $ProcessorArchitectureW6432 + } + if (($ProcessorArchitecture -Eq "AMD64") -Or + ($ProcessorArchitecture -Eq "IA64") -Or + ($ProcessorArchitecture -Eq "ARM64")) { + return "x64" + } + return "x86" +} + +<# +.SYNOPSIS +Get the name of a temporary folder under the native install directory +#> +function Get-TempDirectory { + return Join-Path (Get-NativeInstallDirectory) "temp/" +} + +function Get-TempPathFilename { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $Path + ) + $TempDir = CommonLibrary\Get-TempDirectory + $TempFilename = Split-Path $Path -leaf + $TempPath = Join-Path $TempDir $TempFilename + return $TempPath +} + +<# +.SYNOPSIS +Returns the base directory to use for native tool installation + +.NOTES +Returns the value of the NETCOREENG_INSTALL_DIRECTORY if that environment variable +is set, or otherwise returns an install directory under the %USERPROFILE% +#> +function Get-NativeInstallDirectory { + $InstallDir = $Env:NETCOREENG_INSTALL_DIRECTORY + if (!$InstallDir) { + $InstallDir = Join-Path $Env:USERPROFILE ".netcoreeng/native/" + } + return $InstallDir +} + +<# +.SYNOPSIS +Unzip an archive + +.DESCRIPTION +Powershell module to unzip an archive to a specified directory + +.PARAMETER ZipPath (Required) +Path to archive to unzip + +.PARAMETER OutputDirectory (Required) +Output directory for archive contents + +.PARAMETER Force +Overwrite output directory contents if they already exist + +.NOTES +- Returns True and does not perform an extraction if output directory already exists but Overwrite is not True. +- Returns True if unzip operation is successful +- Returns False if Overwrite is True and it is unable to remove contents of OutputDirectory +- Returns False if unable to extract zip archive +#> +function Expand-Zip { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $ZipPath, + [Parameter(Mandatory=$True)] + [string] $OutputDirectory, + [switch] $Force + ) + + Write-Verbose "Extracting '$ZipPath' to '$OutputDirectory'" + try { + if ((Test-Path $OutputDirectory) -And (-Not $Force)) { + Write-Host "Directory '$OutputDirectory' already exists, skipping extract" + return $True + } + if (Test-Path $OutputDirectory) { + Write-Verbose "'Force' is 'True', but '$OutputDirectory' exists, removing directory" + Remove-Item $OutputDirectory -Force -Recurse + if ($? -Eq $False) { + Write-Error "Unable to remove '$OutputDirectory'" + return $False + } + } + if (-Not (Test-Path $OutputDirectory)) { + New-Item -path $OutputDirectory -Force -itemType "Directory" | Out-Null + } + + Add-Type -assembly "system.io.compression.filesystem" + [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$OutputDirectory") + if ($? -Eq $False) { + Write-Error "Unable to extract '$ZipPath'" + return $False + } + } + catch { + Write-Host $_ + Write-Host $_.Exception + + return $False + } + return $True +} + +export-modulemember -function DownloadAndExtract +export-modulemember -function Expand-Zip +export-modulemember -function Get-File +export-modulemember -function Get-MachineArchitecture +export-modulemember -function Get-NativeInstallDirectory +export-modulemember -function Get-TempDirectory +export-modulemember -function Get-TempPathFilename +export-modulemember -function New-ScriptShim diff --git a/eng/common/native/common-library.sh b/eng/common/native/common-library.sh new file mode 100644 index 000000000..271bddfac --- /dev/null +++ b/eng/common/native/common-library.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash + +function GetNativeInstallDirectory { + local install_dir + + if [[ -z $NETCOREENG_INSTALL_DIRECTORY ]]; then + install_dir=$HOME/.netcoreeng/native/ + else + install_dir=$NETCOREENG_INSTALL_DIRECTORY + fi + + echo $install_dir + return 0 +} + +function GetTempDirectory { + + echo $(GetNativeInstallDirectory)temp/ + return 0 +} + +function ExpandZip { + local zip_path=$1 + local output_directory=$2 + local force=${3:-false} + + echo "Extracting $zip_path to $output_directory" + if [[ -d $output_directory ]] && [[ $force = false ]]; then + echo "Directory '$output_directory' already exists, skipping extract" + return 0 + fi + + if [[ -d $output_directory ]]; then + echo "'Force flag enabled, but '$output_directory' exists. Removing directory" + rm -rf $output_directory + if [[ $? != 0 ]]; then + echo Unable to remove '$output_directory'>&2 + return 1 + fi + fi + + echo "Creating directory: '$output_directory'" + mkdir -p $output_directory + + echo "Extracting archive" + tar -xf $zip_path -C $output_directory + if [[ $? != 0 ]]; then + echo "Unable to extract '$zip_path'" >&2 + return 1 + fi + + return 0 +} + +function GetCurrentOS { + local unameOut="$(uname -s)" + case $unameOut in + Linux*) echo "Linux";; + Darwin*) echo "MacOS";; + esac + return 0 +} + +function GetFile { + local uri=$1 + local path=$2 + local force=${3:-false} + local download_retries=${4:-5} + local retry_wait_time_seconds=${5:-30} + + if [[ -f $path ]]; then + if [[ $force = false ]]; then + echo "File '$path' already exists. Skipping download" + return 0 + else + rm -rf $path + fi + fi + + if [[ -f $uri ]]; then + echo "'$uri' is a file path, copying file to '$path'" + cp $uri $path + return $? + fi + + echo "Downloading $uri" + # Use curl if available, otherwise use wget + if command -v curl > /dev/null; then + curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail + else + wget -q -O "$path" "$uri" --tries="$download_retries" + fi + + return $? +} + +function GetTempPathFileName { + local path=$1 + + local temp_dir=$(GetTempDirectory) + local temp_file_name=$(basename $path) + echo $temp_dir$temp_file_name + return 0 +} + +function DownloadAndExtract { + local uri=$1 + local installDir=$2 + local force=${3:-false} + local download_retries=${4:-5} + local retry_wait_time_seconds=${5:-30} + + local temp_tool_path=$(GetTempPathFileName $uri) + + echo "downloading to: $temp_tool_path" + + # Download file + GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds + if [[ $? != 0 ]]; then + echo "Failed to download '$uri' to '$temp_tool_path'." >&2 + return 1 + fi + + # Extract File + echo "extracting from $temp_tool_path to $installDir" + ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds + if [[ $? != 0 ]]; then + echo "Failed to extract '$temp_tool_path' to '$installDir'." >&2 + return 1 + fi + + return 0 +} + +function NewScriptShim { + local shimpath=$1 + local tool_file_path=$2 + local force=${3:-false} + + echo "Generating '$shimpath' shim" + if [[ -f $shimpath ]]; then + if [[ $force = false ]]; then + echo "File '$shimpath' already exists." >&2 + return 1 + else + rm -rf $shimpath + fi + fi + + if [[ ! -f $tool_file_path ]]; then + echo "Specified tool file path:'$tool_file_path' does not exist" >&2 + return 1 + fi + + local shim_contents=$'#!/usr/bin/env bash\n' + shim_contents+="SHIMARGS="$'$1\n' + shim_contents+="$tool_file_path"$' $SHIMARGS\n' + + # Write shim file + echo "$shim_contents" > $shimpath + + chmod +x $shimpath + + echo "Finished generating shim '$shimpath'" + + return $? +} + diff --git a/eng/common/native/install-cmake.ps1 b/eng/common/native/install-cmake.ps1 new file mode 100644 index 000000000..40fc9ac55 --- /dev/null +++ b/eng/common/native/install-cmake.ps1 @@ -0,0 +1,119 @@ +<# +.SYNOPSIS +Install cmake native tool + +.DESCRIPTION +Install cmake native tool from Azure blob storage + +.PARAMETER InstallPath +Base directory to install native tool to + +.PARAMETER BaseUri +Base file directory or Url from which to acquire tool archives + +.PARAMETER CommonLibraryDirectory +Path to folder containing common library modules + +.PARAMETER Force +Force install of tools even if they previously exist + +.PARAMETER Clean +Don't install the tool, just clean up the current install of the tool + +.PARAMETER DownloadRetries +Total number of retry attempts + +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds + +.NOTES +Returns 0 if install succeeds, 1 otherwise +#> +[CmdletBinding(PositionalBinding=$false)] +Param ( + [Parameter(Mandatory=$True)] + [string] $InstallPath, + [Parameter(Mandatory=$True)] + [string] $BaseUri, + [Parameter(Mandatory=$True)] + [string] $Version, + [string] $CommonLibraryDirectory = $PSScriptRoot, + [switch] $Force = $False, + [switch] $Clean = $False, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30 +) + +# Import common library modules +Import-Module -Name (Join-Path $CommonLibraryDirectory "CommonLibrary.psm1") + +try { + # Define verbose switch if undefined + $Verbose = $VerbosePreference -Eq "Continue" + + $ToolName = "cmake" + + $Arch = CommonLibrary\Get-MachineArchitecture + $ToolOs = "win64" + if($Arch -Eq "x32") { + $ToolOs = "win32" + } + $ToolNameMoniker = "$ToolName-$Version-$ToolOs-$Arch" + $ToolInstallDirectory = Join-Path $InstallPath "$ToolName\$Version\" + $ToolFilePath = Join-Path $ToolInstallDirectory "$ToolNameMoniker\bin\$ToolName.exe" + $ShimPath = Join-Path $InstallPath "$ToolName.cmd" + $Uri = "$BaseUri/windows/$Toolname/$ToolNameMoniker.zip" + + if ($Clean) { + Write-Host "Cleaning $ToolInstallDirectory" + if (Test-Path $ToolInstallDirectory) { + Remove-Item $ToolInstallDirectory -Force -Recurse + } + Write-Host "Cleaning $ShimPath" + if (Test-Path $ShimPath) { + Remove-Item $ShimPath -Force + } + $ToolTempPath = CommonLibrary\Get-TempPathFilename -Path $Uri + Write-Host "Cleaning $ToolTempPath" + if (Test-Path $ToolTempPath) { + Remove-Item $ToolTempPath -Force + } + exit 0 + } + + # Install tool + if ((Test-Path $ToolFilePath) -And (-Not $Force)) { + Write-Verbose "$ToolName ($Version) already exists, skipping install" + } + else { + $InstallStatus = CommonLibrary\DownloadAndExtract -Uri $Uri ` + -InstallDirectory $ToolInstallDirectory ` + -Force:$Force ` + -DownloadRetries $DownloadRetries ` + -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` + -Verbose:$Verbose + + if ($InstallStatus -Eq $False) { + Write-Error "Installation failed" + exit 1 + } + } + # Generate shim + # Always rewrite shims so that we are referencing the expected version + $GenerateShimStatus = CommonLibrary\New-ScriptShim -ShimPath $ShimPath ` + -ToolFilePath $ToolFilePath ` + -Force ` + -Verbose:$Verbose + + if ($GenerateShimStatus -Eq $False) { + Write-Error "Generate shim failed" + return 1 + } + + exit 0 +} +catch { + Write-Host $_ + Write-Host $_.Exception + exit 1 +} \ No newline at end of file diff --git a/eng/common/native/install-cmake.sh b/eng/common/native/install-cmake.sh new file mode 100644 index 000000000..293af6017 --- /dev/null +++ b/eng/common/native/install-cmake.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. $scriptroot/common-library.sh + +base_uri= +install_path= +version= +clean=false +force=false +download_retries=5 +retry_wait_time_seconds=30 + +while (($# > 0)); do + lowerI="$(echo $1 | awk '{print tolower($0)}')" + case $lowerI in + --baseuri) + base_uri=$2 + shift 2 + ;; + --installpath) + install_path=$2 + shift 2 + ;; + --version) + version=$2 + shift 2 + ;; + --clean) + clean=true + shift 1 + ;; + --force) + force=true + shift 1 + ;; + --downloadretries) + download_retries=$2 + shift 2 + ;; + --retrywaittimeseconds) + retry_wait_time_seconds=$2 + shift 2 + ;; + --help) + echo "Common settings:" + echo " --baseuri Base file directory or Url wrom which to acquire tool archives" + echo " --installpath Base directory to install native tool to" + echo " --clean Don't install the tool, just clean up the current install of the tool" + echo " --force Force install of tools even if they previously exist" + echo " --help Print help and exit" + echo "" + echo "Advanced settings:" + echo " --downloadretries Total number of retry attempts" + echo " --retrywaittimeseconds Wait time between retry attempts in seconds" + echo "" + exit 0 + ;; + esac +done + +tool_name="cmake" +tool_os=$(GetCurrentOS) +tool_folder=$(echo $tool_os | awk '{print tolower($0)}') +tool_arch="x86_64" +tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" +tool_install_directory="$install_path/$tool_name/$version" +tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name" +shim_path="$install_path/$tool_name.sh" +uri="${base_uri}/$tool_folder/cmake/$tool_name_moniker.tar.gz" + +# Clean up tool and installers +if [[ $clean = true ]]; then + echo "Cleaning $tool_install_directory" + if [[ -d $tool_install_directory ]]; then + rm -rf $tool_install_directory + fi + + echo "Cleaning $shim_path" + if [[ -f $shim_path ]]; then + rm -rf $shim_path + fi + + tool_temp_path=$(GetTempPathFileName $uri) + echo "Cleaning $tool_temp_path" + if [[ -f $tool_temp_path ]]; then + rm -rf $tool_temp_path + fi + + exit 0 +fi + +# Install tool +if [[ -f $tool_file_path ]] && [[ $force = false ]]; then + echo "$tool_name ($version) already exists, skipping install" + exit 0 +fi + +DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds + +if [[ $? != 0 ]]; then + echo "Installation failed" >&2 + exit 1 +fi + +# Generate Shim +# Always rewrite shims so that we are referencing the expected version +NewScriptShim $shim_path $tool_file_path true + +if [[ $? != 0 ]]; then + echo "Shim generation failed" >&2 + exit 1 +fi + +exit 0 \ No newline at end of file diff --git a/eng/common/templates/phases/base.yml b/eng/common/templates/phases/base.yml index 2071c4abc..0ca095688 100644 --- a/eng/common/templates/phases/base.yml +++ b/eng/common/templates/phases/base.yml @@ -29,6 +29,15 @@ parameters: # _HelixType - Example: official/dotnet/arcade/$(Build.SourceBranch) enableTelemetry: false + # Optional: Enable installing Microbuild plugin + # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix + # _TeamName - the name of your team + # _SignType - 'test' or 'real' + enableMicrobuild: false + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + phases: - phase: ${{ parameters.name }} @@ -52,6 +61,21 @@ phases: helixSource: $(_HelixSource) helixType: $(_HelixType) + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + # Internal only resource, and Microbuild signing shouldn't be applied to PRs. + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildSigningPlugin@1 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + + env: + TeamName: $(_TeamName) + continueOnError: false + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + # Run provided build steps - ${{ parameters.steps }} @@ -69,3 +93,20 @@ phases: parameters: helixSource: $(_HelixSource) helixType: $(_HelixType) + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: CopyFiles@2 + displayName: Gather Asset Manifests + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' + TargetFolder: '$(Build.StagingDirectory)/AssetManifests' + continueOnError: false + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + - task: PublishBuildArtifacts@1 + displayName: Push Asset Manifests + inputs: + PathtoPublish: '$(Build.StagingDirectory)/AssetManifests' + PublishLocation: Container + ArtifactName: AssetManifests + continueOnError: false + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) \ No newline at end of file diff --git a/eng/common/templates/phases/publish-build-assets.yml b/eng/common/templates/phases/publish-build-assets.yml new file mode 100644 index 000000000..00072fe0d --- /dev/null +++ b/eng/common/templates/phases/publish-build-assets.yml @@ -0,0 +1,27 @@ +parameters: + dependsOn: '' + queue: {} +phases: + - phase: Asset_Registry_Publish + displayName: Publish to Build Asset Registry + dependsOn: ${{ parameters.dependsOn }} + queue: ${{ parameters.queue }} + steps: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download artifact + inputs: + artifactName: AssetManifests + downloadPath: '$(Build.StagingDirectory)/Download' + condition: succeeded() + - task: AzureKeyVault@1 + inputs: + azureSubscription: 'DotNet-Engineering-Services_KeyVault' + KeyVaultName: EngKeyVault + SecretsFilter: 'MaestroAccessToken' + condition: succeeded() + - script: eng\common\publishbuildassets.cmd + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + displayName: Publish Build Assets \ No newline at end of file diff --git a/eng/common/templates/steps/helix-publish.yml b/eng/common/templates/steps/helix-publish.yml new file mode 100644 index 000000000..6e89d07e4 --- /dev/null +++ b/eng/common/templates/steps/helix-publish.yml @@ -0,0 +1,33 @@ +parameters: + HelixSource: 'pr/dotnet-github-anon-kaonashi-bot' + HelixType: ̓'tests/default' + HelixBuild: $(Build.BuildNumber) + HelixTargetQueues: '' + HelixAccessToken: '' + HelixPreCommands: '' + HelixPostCommands: '' + WorkItemDirectory: '' + WorkItemCommand: '' + EnableXUnitReporter: false + WaitForWorkItemCompletion: true + +steps: + - task: DotNetCoreCLI@2 + inputs: + command: custom + projects: eng/common/helixpublish.proj + custom: msbuild + arguments: '/t:test /p:Language=msbuild' + displayName: Send tests job to Helix + env: + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} diff --git a/eng/install-test-runtimes.ps1 b/eng/install-test-runtimes.ps1 index 04d84aed0..20e7a33c5 100644 --- a/eng/install-test-runtimes.ps1 +++ b/eng/install-test-runtimes.ps1 @@ -1,6 +1,7 @@ [cmdletbinding()] param( [string]$DotNetDir, + [string]$RuntimeVersion21, [string]$TempDir, [string]$BuildArch, [switch]$DailyTest, @@ -13,7 +14,6 @@ $ErrorActionPreference="Stop" $RuntimeVersion11="1.1.9" $RuntimeVersion20="2.0.9" -$RuntimeVersion21="2.1.0" $DailyTestText="true" # Install the other versions of .NET Core runtime we are going to test. 1.1.x, 2.0.x, 2.1.x (installed with the CLI) diff --git a/eng/install-test-runtimes.sh b/eng/install-test-runtimes.sh index f4fc0ce7e..2edf1b0f3 100755 --- a/eng/install-test-runtimes.sh +++ b/eng/install-test-runtimes.sh @@ -9,6 +9,10 @@ daily_test=0 branch="master" uncached_feed="https://dotnetcli.blob.core.windows.net/dotnet" +runtime_version_11="1.1.9" +runtime_version_20="2.0.9" +runtime_version_21= + while [ $# -ne 0 ]; do name=$1 case $name in @@ -16,6 +20,10 @@ while [ $# -ne 0 ]; do shift dotnet_dir=$1 ;; + --runtime-version-21) + shift + runtime_version_21=$1 + ;; --temp-directory) shift temp_dir=$1 @@ -38,9 +46,6 @@ while [ $# -ne 0 ]; do shift done -runtime_version_11="1.1.9" -runtime_version_20="2.0.9" -runtime_version_21="2.1.0" daily_test_text="true" # Install the other versions of .NET Core runtime we are going to test. 1.1.x, 2.0.x, 2.1.x (installed with the CLI) diff --git a/global.json b/global.json index aa15e36a5..4c81226a1 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { - "sdk": { - "version": "2.1.300" + "tools": { + "dotnet": "2.1.401" }, "msbuild-sdks": { - "RoslynTools.RepoToolset": "1.0.0-beta2-63206-01" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.18501.3" } } diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 76b56816c..24999e271 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -1,9 +1,19 @@ + + false + + + Latest + + false + true diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets new file mode 100644 index 000000000..177d64c47 --- /dev/null +++ b/src/Directory.Build.targets @@ -0,0 +1,4 @@ + + + + diff --git a/src/Microsoft.Diagnostic.TestHelpers/Microsoft.Diagnostic.TestHelpers.csproj b/src/Microsoft.Diagnostic.TestHelpers/Microsoft.Diagnostic.TestHelpers.csproj index 431e0d0ab..847fd8fff 100644 --- a/src/Microsoft.Diagnostic.TestHelpers/Microsoft.Diagnostic.TestHelpers.csproj +++ b/src/Microsoft.Diagnostic.TestHelpers/Microsoft.Diagnostic.TestHelpers.csproj @@ -1,5 +1,5 @@  - + netcoreapp2.0 true @@ -12,7 +12,6 @@ - diff --git a/src/Microsoft.Diagnostic.TestHelpers/TestConfiguration.cs b/src/Microsoft.Diagnostic.TestHelpers/TestConfiguration.cs index 09a743007..37aa51ab4 100644 --- a/src/Microsoft.Diagnostic.TestHelpers/TestConfiguration.cs +++ b/src/Microsoft.Diagnostic.TestHelpers/TestConfiguration.cs @@ -43,10 +43,10 @@ namespace Microsoft.Diagnostic.TestHelpers string nugetPackages = Environment.GetEnvironmentVariable("NUGET_PACKAGES"); if (nugetPackages == null) { - // If not already set, the RoslynTools RepoToolSet scripts/build system sets - // NUGET_PACKAGES to the UserProfile or HOME nuget cache directories if building - // locally (for speed) or to the repo root/.packages in CI builds (to isolate - // global machine dependences). + // If not already set, the arcade SDK scripts/build system sets NUGET_PACKAGES + // to the UserProfile or HOME nuget cache directories if building locally (for + // speed) or to the repo root/.packages in CI builds (to isolate global machine + // dependences). // // This emulates that logic so the VS Test Explorer can still run the tests for // config files that don't set the NugetPackagesCacheDir value (like the SOS unit diff --git a/src/SOS/SOS.NETCore/CMakeLists.txt b/src/SOS/SOS.NETCore/CMakeLists.txt index 28ff7ea70..735d541f3 100644 --- a/src/SOS/SOS.NETCore/CMakeLists.txt +++ b/src/SOS/SOS.NETCore/CMakeLists.txt @@ -1,7 +1,7 @@ project(SOS.NETCore) if(NOT ${CLR_MANAGED_BINARY_DIR} STREQUAL "") - set(MANAGED_BINDIR ${CLR_MANAGED_BINARY_DIR}/SOS.NETCore/netcoreapp1.0) + set(MANAGED_BINDIR ${CLR_MANAGED_BINARY_DIR}/SOS.NETCore/${CLR_BUILD_TYPE}/netcoreapp1.0) install(FILES ${MANAGED_BINDIR}/SOS.NETCore.dll DESTINATION .) install(FILES ${MANAGED_BINDIR}/SOS.NETCore.pdb DESTINATION .) diff --git a/src/SOS/SOS.NETCore/SOS.NETCore.csproj b/src/SOS/SOS.NETCore/SOS.NETCore.csproj index 93ed689de..87a7e0936 100644 --- a/src/SOS/SOS.NETCore/SOS.NETCore.csproj +++ b/src/SOS/SOS.NETCore/SOS.NETCore.csproj @@ -1,5 +1,5 @@  - + netcoreapp1.0 SOS.NETCore @@ -11,6 +11,7 @@ $(Description) SOS false + $(ArtifactsBinDir)\SOS.NETCore\$(Configuration)\netcoreapp1.0\SOS.NETCore.dll @@ -18,54 +19,54 @@ - <_PackageFiles Include="$(ArtifactsBinDir)\SOS.NETCore\netcoreapp1.0\SOS.NETCore.dll"> + <_PackageFiles Include="$(SOSNETCorePath)"> None tools/win-x64 - <_PackageFiles Include="$(ArtifactsBinDir)\Windows_NT.x64\sos.dll"> + <_PackageFiles Include="$(ArtifactsBinDir)\Windows_NT.x64.$(Configuration)\sos.dll"> None tools/win-x64 - <_PackageFiles Include="$(ArtifactsBinDir)\SOS.NETCore\netcoreapp1.0\SOS.NETCore.dll"> + <_PackageFiles Include="$(SOSNETCorePath)"> None tools/win-x86 - <_PackageFiles Include="$(ArtifactsBinDir)\Windows_NT.x86\sos.dll"> + <_PackageFiles Include="$(ArtifactsBinDir)\Windows_NT.x86.$(Configuration)\sos.dll"> None tools/win-x86 - <_PackageFiles Include="$(ArtifactsBinDir)\SOS.NETCore\netcoreapp1.0\SOS.NETCore.dll"> + <_PackageFiles Include="$(SOSNETCorePath)"> None tools/linux-x64 - <_PackageFiles Include="$(ArtifactsBinDir)\Linux.x64\libsosplugin.so"> + <_PackageFiles Include="$(ArtifactsBinDir)\Linux.x64.$(Configuration)\libsosplugin.so"> None tools/linux-x64 - <_PackageFiles Include="$(ArtifactsBinDir)\Linux.x64\libsos.so"> + <_PackageFiles Include="$(ArtifactsBinDir)\Linux.x64.$(Configuration)\libsos.so"> None tools/linux-x64 - <_PackageFiles Include="$(ArtifactsBinDir)\Linux.x64\sosdocsunix.txt"> + <_PackageFiles Include="$(ArtifactsBinDir)\Linux.x64.$(Configuration)\sosdocsunix.txt"> None tools/linux-x64 - <_PackageFiles Include="$(ArtifactsBinDir)\SOS.NETCore\netcoreapp1.0\SOS.NETCore.dll"> + <_PackageFiles Include="$(SOSNETCorePath)"> None tools/osx-x64 - <_PackageFiles Include="$(ArtifactsBinDir)\OSX.x64\libsosplugin.dylib"> + <_PackageFiles Include="$(ArtifactsBinDir)\OSX.x64.$(Configuration)\libsosplugin.dylib"> None tools/osx-x64 - <_PackageFiles Include="$(ArtifactsBinDir)\OSX.x64\libsos.dylib"> + <_PackageFiles Include="$(ArtifactsBinDir)\OSX.x64.$(Configuration)\libsos.dylib"> None tools/osx-x64 - <_PackageFiles Include="$(ArtifactsBinDir)\OSX.x64\sosdocsunix.txt"> + <_PackageFiles Include="$(ArtifactsBinDir)\OSX.x64.$(Configuration)\sosdocsunix.txt"> None tools/osx-x64 diff --git a/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt b/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt index 80ac42af4..e0590daf5 100644 --- a/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt +++ b/src/SOS/SOS.UnitTests/ConfigFiles/Unix/Debugger.Tests.Config.txt @@ -15,9 +15,9 @@ ProjectK $(RepoRootDir)/src/SOS/SOS.UnitTests/Scripts $(RepoRootDir)/artifacts - $(RootBinDir)/$(TargetConfiguration)/bin/$(OS).$(TargetArchitecture) - $(RootBinDir)/$(TargetConfiguration)/TestResults/sos.unittests_$(Timestamp) - $(RootBinDir)/$(TargetConfiguration)/tmp/dumps + $(RootBinDir)/bin/$(OS).$(TargetArchitecture).$(TargetConfiguration) + $(RootBinDir)/TestResults/$(TargetConfiguration)/sos.unittests_$(Timestamp) + $(RootBinDir)/tmp/$(TargetConfiguration)\dumps $(RepoRootDir)/src/SOS/SOS.UnitTests/Debuggees $(RootBinDir)/Debuggees diff --git a/src/SOS/SOS.UnitTests/ConfigFiles/Windows/Debugger.Tests.Config.txt b/src/SOS/SOS.UnitTests/ConfigFiles/Windows/Debugger.Tests.Config.txt index c73a0dabf..981337a62 100644 --- a/src/SOS/SOS.UnitTests/ConfigFiles/Windows/Debugger.Tests.Config.txt +++ b/src/SOS/SOS.UnitTests/ConfigFiles/Windows/Debugger.Tests.Config.txt @@ -14,9 +14,9 @@ $(RepoRootDir)\src\SOS\SOS.UnitTests\Scripts $(RepoRootDir)\artifacts - $(RootBinDir)\$(TargetConfiguration)\bin\Windows_NT.$(TargetArchitecture) - $(RootBinDir)\$(TargetConfiguration)\TestResults\sos.unittests_$(Timestamp) - $(RootBinDir)\$(TargetConfiguration)\tmp\dumps + $(RootBinDir)\bin\Windows_NT.$(TargetArchitecture).$(TargetConfiguration) + $(RootBinDir)\TestResults\$(TargetConfiguration)\sos.unittests_$(Timestamp) + $(RootBinDir)\tmp\$(TargetConfiguration)\dumps $(NuGetPackageCacheDir)\cdb-sos\1.1.0\runtimes\win-$(TargetArchitecture)\native\cdb.exe $(InstallDir)\runcommand.dll diff --git a/src/SOS/SOS.UnitTests/SOS.UnitTests.csproj b/src/SOS/SOS.UnitTests/SOS.UnitTests.csproj index f914a8b3d..d4c9a8c5a 100644 --- a/src/SOS/SOS.UnitTests/SOS.UnitTests.csproj +++ b/src/SOS/SOS.UnitTests/SOS.UnitTests.csproj @@ -1,5 +1,5 @@  - + netcoreapp2.0 false @@ -30,6 +30,10 @@ + + + + diff --git a/src/SOS/Strike/Native.rc b/src/SOS/Strike/Native.rc index 179ddfd24..c1ffa2b9a 100644 --- a/src/SOS/Strike/Native.rc +++ b/src/SOS/Strike/Native.rc @@ -2,7 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -#define FX_VER_FILEDESCRIPTION_STR "Microsoft NTSD extension for .NET Runtime\0" +#define FX_VER_FILEDESCRIPTION_STR "Microsoft debugger extension for .NET Core\0" #include #include diff --git a/src/SOS/Strike/sosdocs.txt b/src/SOS/Strike/sosdocs.txt index 344574955..abd432e49 100644 --- a/src/SOS/Strike/sosdocs.txt +++ b/src/SOS/Strike/sosdocs.txt @@ -61,8 +61,8 @@ Examining the GC history Other ----------------------------- ----------------------------- HistInit SetHostRuntime (sethostruntime) HistRoot FAQ -HistObj soshelp -HistObjFind +HistObj SOSFlush +HistObjFind Help (soshelp) HistClear \\ diff --git a/src/SOS/Strike/sosdocsunix.txt b/src/SOS/Strike/sosdocsunix.txt index c2df0d141..3c6d512f5 100644 --- a/src/SOS/Strike/sosdocsunix.txt +++ b/src/SOS/Strike/sosdocsunix.txt @@ -54,8 +54,8 @@ Examining the GC history Other ----------------------------- ----------------------------- HistInit (histinit) SetHostRuntime (sethostruntime) HistRoot (histroot) FAQ -HistObj (histobj) Help (soshelp) -HistObjFind (histobjfind) +HistObj (histobj) SOSFlush +HistObjFind (histobjfind) Help (soshelp) HistClear (histclear) \\ @@ -202,12 +202,12 @@ The arguments in detail: \\ COMMAND: dumpasync. -!DumpAsync [-mt ] - [-type ] - [-waiting] - [-roots]] +DumpAsync [-mt ] + [-type ] + [-waiting] + [-roots] -!DumpAsync traverses the garbage collected heap, looking for objects representing +DumpAsync traverses the garbage collected heap, looking for objects representing async state machines as created when an async method's state is transferred to the heap. This command recognizes async state machines defined as "async void", "async Task", "async Task", "async ValueTask", and "async ValueTask". diff --git a/src/SOS/Strike/strike.cpp b/src/SOS/Strike/strike.cpp index b9eeba2d4..a426f0c88 100644 --- a/src/SOS/Strike/strike.cpp +++ b/src/SOS/Strike/strike.cpp @@ -13097,6 +13097,8 @@ DECLARE_API( VMMap ) return Status; } // DECLARE_API( vmmap ) +#endif // FEATURE_PAL + DECLARE_API( SOSFlush ) { INIT_API(); @@ -13106,6 +13108,8 @@ DECLARE_API( SOSFlush ) return Status; } // DECLARE_API( SOSFlush ) +#ifndef FEATURE_PAL + DECLARE_API( VMStat ) { INIT_API(); diff --git a/src/SOS/lldbplugin.tests/TestDebuggee/TestDebuggee.csproj b/src/SOS/lldbplugin.tests/TestDebuggee/TestDebuggee.csproj index a6585e5c2..f779b8afd 100644 --- a/src/SOS/lldbplugin.tests/TestDebuggee/TestDebuggee.csproj +++ b/src/SOS/lldbplugin.tests/TestDebuggee/TestDebuggee.csproj @@ -1,5 +1,5 @@  - + Exe netcoreapp2.0 diff --git a/src/SOS/lldbplugin.tests/testsos.sh b/src/SOS/lldbplugin.tests/testsos.sh index 355c2b67e..9aa83c1c7 100755 --- a/src/SOS/lldbplugin.tests/testsos.sh +++ b/src/SOS/lldbplugin.tests/testsos.sh @@ -1,16 +1,16 @@ #!/usr/bin/env bash -__ProjectRoot="$1" -__Plugin="$2" -__ManagedBinDir="$3" -__ResultsDir="$4" +project_root="$1" +plugin="$2" +host_version="$3" +test_program="$4" +results_dir="$5" -if [[ "$__ProjectRoot" = "" || "$__Plugin" = "" || "$__ManagedBinDir" = "" || "$__ResultsDir" = "" ]]; then +if [[ "$project_root" = "" || "$plugin" = "" || "$test_program" = "" || "$results_dir" = "" ]]; then echo "Project root, plugin or log directory required" exit 1 fi -__Host="$__ProjectRoot/.dotnet/dotnet --fx-version 2.1.0" -__TestProgram=$__ManagedBinDir/TestDebuggee/netcoreapp2.0/TestDebuggee.dll +host="$project_root/.dotnet/dotnet --fx-version $host_version" # Turn on stress logging so the dumplog and histinit commands pass export COMPlus_LogFacility=0xffffffbf @@ -23,10 +23,10 @@ if [[ ! -x "$LLDB_PATH" ]]; then exit 1 fi -__LogFileDir=$__ResultsDir/lldbplugin.tests_$(date +%Y_%m_%d_%H_%M_%S) -mkdir -p $__LogFileDir +log_dir=$results_dir/lldbplugin.tests_$(date +%Y_%m_%d_%H_%M_%S) +mkdir -p $log_dir -cd $__ProjectRoot/src/SOS/lldbplugin.tests/ +cd $project_root/src/SOS/lldbplugin.tests/ rm -f StressLog.txt -python $__ProjectRoot/src/SOS/lldbplugin.tests/test_libsosplugin.py --lldb $LLDB_PATH --host "$__Host" --plugin $__Plugin --logfiledir $__LogFileDir --assembly $__TestProgram +python $project_root/src/SOS/lldbplugin.tests/test_libsosplugin.py --lldb $LLDB_PATH --host "$host" --plugin $plugin --logfiledir $log_dir --assembly $test_program diff --git a/src/pal/prebuilt/inc/asm_version.h b/src/pal/prebuilt/inc/asm_version.h deleted file mode 100644 index 44e09c0e8..000000000 --- a/src/pal/prebuilt/inc/asm_version.h +++ /dev/null @@ -1,22 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. -#if defined(SILVERLIGHT) -#if defined(FEATURE_CORESYSTEM) -#define asm_rmj 4 -#define asm_rmm 0 -#define asm_rup 0 -#define asm_rpt 0 -#else -#define asm_rmj 5 -#define asm_rmm 0 -#define asm_rup 5 -#define asm_rpt 0 -#endif -#else -#define asm_rmj 4 -#define asm_rmm 0 -#define asm_rup 0 -#define asm_rpt 0 -#endif - diff --git a/src/pal/prebuilt/inc/fxver.rc b/src/pal/prebuilt/inc/fxver.rc index a34104130..755501c38 100644 --- a/src/pal/prebuilt/inc/fxver.rc +++ b/src/pal/prebuilt/inc/fxver.rc @@ -57,7 +57,7 @@ BEGIN VALUE "LegalCopyright", VER_LEGALCOPYRIGHT_STR VALUE "OriginalFilename",VER_ORIGINALFILENAME_STR VALUE "ProductName", VER_PRODUCTNAME_STR - VALUE "ProductVersion", VER_FILEVERSION_STR + VALUE "ProductVersion", VER_PRODUCTVERSION_STR #ifdef VER_OLESELFREGISTER VALUE "OleSelfRegister", "\0" #endif @@ -82,7 +82,7 @@ BEGIN VALUE "LegalCopyright", VER_LEGALCOPYRIGHT_STR VALUE "OriginalFilename",VER_ORIGINALFILENAME_STR VALUE "ProductName", VER_PRODUCTNAME_STR - VALUE "ProductVersion", VER_FILEVERSION_STR + VALUE "ProductVersion", VER_PRODUCTVERSION_STR #ifdef VER_OLESELFREGISTER VALUE "OleSelfRegister", "\0" #endif diff --git a/src/pal/prebuilt/inc/ndpversion.h b/src/pal/prebuilt/inc/ndpversion.h deleted file mode 100644 index ba6d01826..000000000 --- a/src/pal/prebuilt/inc/ndpversion.h +++ /dev/null @@ -1,4 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. -#include diff --git a/src/pal/prebuilt/inc/ndpversion_generated.h b/src/pal/prebuilt/inc/ndpversion_generated.h deleted file mode 100644 index 563e31697..000000000 --- a/src/pal/prebuilt/inc/ndpversion_generated.h +++ /dev/null @@ -1,15 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -#if 0 -/**** Generated Based on d:\ProjectK\src\InternalApis\Version\buildnumber.settings.targets -One can't put comments in this file (without the #if) -because this header is preprocessed in non-C++ context (xml, perl, etc.). *****/ -#endif -#define NDPVersionNumberMajor 4 -#define NDPVersionNumberMinor 0 -#define NDPVersionNumberMajor_A "4" -#define NDPVersionNumberMinor_A "00" -#define NDPVersionNumbers_A "4.00" -#include "buildnumber.h" diff --git a/src/pal/prebuilt/inc/product_version.h b/src/pal/prebuilt/inc/product_version.h index a21ee6874..374cbab24 100644 --- a/src/pal/prebuilt/inc/product_version.h +++ b/src/pal/prebuilt/inc/product_version.h @@ -7,13 +7,6 @@ // See the LICENSE file in the project root for more information. #endif -#include -#ifdef USE_CLR20_VERSION -#include -#else -#include -#endif - #ifdef CLR_MAJOR_VERSION #undef CLR_MAJOR_VERSION #endif diff --git a/src/pal/prebuilt/inc/version.h b/src/pal/prebuilt/inc/version.h deleted file mode 100644 index a6de92ca9..000000000 --- a/src/pal/prebuilt/inc/version.h +++ /dev/null @@ -1,17 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -#include - -#define rmj NDPVersionNumberMajor -#define rmm NDPVersionNumberMinor -#define rup NDPBuildNumberMajor -#define rpt NDPBuildNumberMinor - -#define fvn NDPFileVersionMinor -#define fvb NDPFileVersionBuild -#define fvr NDPFileVersionRevision - -#define szVerName "" -#define szVerUser ""