From 3f9ba8e95567517b9ddf77b8d55eda547473c744 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Tue, 17 Sep 2019 10:00:14 -0500 Subject: [PATCH] Add internal build support (dotnet/core-setup#8279) Add NuGetAuthenticate task. Create proxy build scripts that include pre-build auth plugin install. Add workaround to avoid running test restore against authenticated feed. Pass the NuGet auth env vars set by the AzDO build step into the Docker container. Add configurability for internal publish. Commit migrated from https://github.com/dotnet/core-setup/commit/a896b3ee381b6d58830613def6f118e16a8a6dc7 --- eng/install-nuget-credprovider-then-build.sh | 24 ++++++ eng/install-nuget-credprovider-then-msbuild.sh | 24 ++++++ eng/install-nuget-credprovider.sh | 35 ++++++++ eng/jobs/bash-build.yml | 21 ++++- eng/jobs/osx-build.yml | 4 + eng/jobs/prepare-signed-artifacts.yml | 3 + eng/jobs/run-publish-project.yml | 92 +++++++++++++++------- eng/jobs/steps/build-linux-package.yml | 2 +- eng/jobs/windows-build.yml | 2 + eng/pipelines/installer/azure-pipelines.yml | 22 ++++++ src/installer/test/Directory.Build.props | 1 + .../test/PrepareTestAssets/PrepareTestAssets.proj | 39 +++++++++ 12 files changed, 238 insertions(+), 31 deletions(-) create mode 100644 eng/install-nuget-credprovider-then-build.sh create mode 100644 eng/install-nuget-credprovider-then-msbuild.sh create mode 100644 eng/install-nuget-credprovider.sh diff --git a/eng/install-nuget-credprovider-then-build.sh b/eng/install-nuget-credprovider-then-build.sh new file mode 100644 index 0000000..5f86dd5 --- /dev/null +++ b/eng/install-nuget-credprovider-then-build.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -e + +# Installs the NuGet Credential Provider, then calls ../build.sh with all arguments. This creates a +# build context that can restore from authenticated sources. This script is intended for use by the +# official Microsoft build inside a Docker container. + +source="${BASH_SOURCE[0]}" + +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done + +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. "$scriptroot/install-nuget-credprovider.sh" + +"$scriptroot/../build.sh" "$@" diff --git a/eng/install-nuget-credprovider-then-msbuild.sh b/eng/install-nuget-credprovider-then-msbuild.sh new file mode 100644 index 0000000..4781ba1 --- /dev/null +++ b/eng/install-nuget-credprovider-then-msbuild.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -e + +# Installs the NuGet Credential Provider, then calls common/msbuild.sh with all arguments. This +# creates a build context that can restore from authenticated sources. This script is intended for +# use by the official Microsoft build inside a Docker container. + +source="${BASH_SOURCE[0]}" + +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done + +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. "$scriptroot/install-nuget-credprovider.sh" + +"$scriptroot/common/msbuild.sh" "$@" diff --git a/eng/install-nuget-credprovider.sh b/eng/install-nuget-credprovider.sh new file mode 100644 index 0000000..897214f --- /dev/null +++ b/eng/install-nuget-credprovider.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -e + +# This script installs the NuGet Credential Provider. + +# Install curl if necessary. Dependency exists inside downloaded script. +if command -v curl > /dev/null; then + echo "curl found." +else + echo "curl not found, trying to install..." + ( + set +e + set -x + apt update && apt install -y curl + apk update && apk upgrade && apk add curl + exit 0 + ) +fi + +# Install. Ported from https://gist.github.com/shubham90/ad85f2546a72caa20d57bce03ec3890f +install_credprovider() { + # Download the provider and install. + cred_provider_url='https://raw.githubusercontent.com/Microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh' + curl "$cred_provider_url" -s -S -L | bash + + # Environment variable to enable session token cache. More on this here: https://github.com/Microsoft/artifacts-credprovider#help + export NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED=true +} + +install_credprovider + +# Additional setup to try to avoid flakiness: https://github.com/dotnet/arcade/issues/3932 +export DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER=0 +export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20 +export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20 diff --git a/eng/jobs/bash-build.yml b/eng/jobs/bash-build.yml index 9c372d3..b6a6aa6 100644 --- a/eng/jobs/bash-build.yml +++ b/eng/jobs/bash-build.yml @@ -31,18 +31,32 @@ jobs: queue: buildpool.ubuntu.1604.amd64 strategy: ${{ parameters.strategy }} variables: + # Preserve the NuGet authentication env vars into the Docker container. + # The 'NuGetAuthenticate' build step may have set these. + PreserveNuGetAuthDockerArgs: >- + -e VSS_NUGET_URI_PREFIXES + -e VSS_NUGET_ACCESSTOKEN ${{ if ne(parameters.name, 'FreeBSD_x64')}}: RunArguments: >- docker run --privileged --rm -v "$(Build.SourcesDirectory):/root/coresetup" -w="/root/coresetup" + $(PreserveNuGetAuthDockerArgs) ${{ parameters.additionalRunArgs }} ${{ parameters.dockerImage }} ${{ if eq(parameters.name, 'FreeBSD_x64')}}: RunArguments: export DotNetBootstrapCliTarPath=/dotnet-sdk-freebsd-x64.tar && + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + BuildScript: ./eng/install-nuget-credprovider-then-build.sh + MSBuildScript: /root/coresetup/eng/install-nuget-credprovider-then-msbuild.sh + + ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}: + BuildScript: ./build.sh + MSBuildScript: /root/coresetup/eng/common/msbuild.sh + CommonMSBuildArgs: >- /p:Configuration=$(_BuildConfig) /p:OfficialBuildId=$(OfficialBuildId) @@ -73,15 +87,18 @@ jobs: /p:SharedFrameworkPublishDir=/root/sharedFrameworkPublish/ /p:InstallerSourceOSPlatformConfig=linux-x64.$(_BuildConfig) - MSBuildScript: /root/coresetup/eng/common/msbuild.sh DockerRunMSBuild: >- docker run -v $(Build.SourcesDirectory):/root/coresetup -v $(Build.StagingDirectory)/sharedFrameworkPublish/:/root/sharedFrameworkPublish/ -w=/root/coresetup + $(PreserveNuGetAuthDockerArgs) steps: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: NuGetAuthenticate@0 + # Builds don't set user ID, so files might be owned by root and unable to be cleaned up by AzDO. # Clean up the build dirs ourselves in another Docker container to avoid failures. # Using hosted agents is tracked by https://github.com/dotnet/core-setup/issues/4997 @@ -101,7 +118,7 @@ jobs: - script: | set -x df -h - $(RunArguments) ./build.sh $(BuildArguments) + $(RunArguments) $(BuildScript) $(BuildArguments) displayName: Build # Only for glibc leg, here we produce RPMs and Debs diff --git a/eng/jobs/osx-build.yml b/eng/jobs/osx-build.yml index cfe06a0..28479a9 100644 --- a/eng/jobs/osx-build.yml +++ b/eng/jobs/osx-build.yml @@ -21,6 +21,10 @@ jobs: /p:Configuration=$(_BuildConfig) /p:PortableBuild=true steps: + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: NuGetAuthenticate@0 + - script: >- $(Build.SourcesDirectory)/build.sh --ci --test /p:OfficialBuildId=$(OfficialBuildId) diff --git a/eng/jobs/prepare-signed-artifacts.yml b/eng/jobs/prepare-signed-artifacts.yml index 5caa613..ed807af 100644 --- a/eng/jobs/prepare-signed-artifacts.yml +++ b/eng/jobs/prepare-signed-artifacts.yml @@ -16,6 +16,9 @@ jobs: steps: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: NuGetAuthenticate@0 + - task: MicroBuildSigningPlugin@2 displayName: Install MicroBuild plugin for Signing inputs: diff --git a/eng/jobs/run-publish-project.yml b/eng/jobs/run-publish-project.yml index 734cd31..448c702 100644 --- a/eng/jobs/run-publish-project.yml +++ b/eng/jobs/run-publish-project.yml @@ -25,23 +25,76 @@ jobs: # Only get the secret variable groups if the def has the official name. Reduce dev build def risk. - ${{ if eq(variables['Build.DefinitionName'], 'dotnet-core-setup') }}: # Used for publishing individual leg assets to azure blob storage - - group: DotNet-DotNetCli-Storage + - ${{ if eq(parameters.dependency.channel.public, 'true') }}: + - group: DotNet-DotNetCli-Storage + - ${{ if ne(parameters.dependency.channel.public, 'true') }}: + - group: DotNet-MSRC-Storage # Used for dotnet/versions update - group: DotNet-Versions-Publish - # Blob storage publish (installers and checksums) + - name: _DefaultContainerName + value: dotnet + - name: _DefaultChecksumsContainerName + value: dotnet + + - ${{ if eq(parameters.dependency.channel.public, 'true') }}: + - name: _DefaultAzureAccountName + value: dotnetcli + - name: _DefaultAzureAccessToken + value: $(dotnetcli-storage-key) + - name: _DefaultChecksumAzureAccountName + value: dotnetclichecksums + - name: _DefaultChecksumAzureAccessToken + value: $(dotnetclichecksums-storage-key) + # dotnet/versions update + - name: _GitHubUser + value: $[ coalesce(variables.GitHubUser, 'dotnet-build-bot') ] + - name: _GitHubEmail + value: $[ coalesce(variables.GitHubEmail, 'dotnet-build-bot@microsoft.com') ] + - name: _GitHubAuthToken + value: $[ coalesce(variables.GitHubAuthToken, '$(AccessToken-dotnet-build-bot-public-repo)') ] + - name: _VersionsRepoOwner + value: $[ coalesce(variables.VersionsRepoOwner, 'dotnet') ] + - name: _VersionsRepo + value: $[ coalesce(variables.VersionsRepo, 'versions') ] + - name: _DotNetVersionsArgs + value: >- + /p:GitHubUser=$(_GitHubUser) + /p:GitHubEmail=$(_GitHubEmail) + /p:GitHubAuthToken=$(_GitHubAuthToken) + /p:VersionsRepoOwner=$(_VersionsRepoOwner) + /p:VersionsRepo=$(_VersionsRepo) + /p:VersionsRepoPath=build-info/dotnet/core-setup/$(FullBranchName) + + - ${{ if ne(parameters.dependency.channel.public, 'true') }}: + - name: _DefaultAzureAccountName + value: dotnetclimsrc + - name: _DefaultAzureAccessToken + value: $(dotnetclimsrc-access-key) + - name: _DefaultChecksumAzureAccountName + value: dotnetclimsrc + - name: _DefaultChecksumsContainerName + value: dotnet-checksums + - name: _DefaultChecksumAzureAccessToken + value: $(dotnetclimsrc-access-key) + # dotnet/versions update (disabled) + - name: _DotNetVersionsArgs + value: '' + + # Blob storage publish (installers and checksums) - name: _AzureAccountName - value: $[ coalesce(variables.AzureAccountName, 'dotnetcli') ] + value: $[ coalesce(variables.AzureAccountName, '$(_DefaultAzureAccountName)') ] - name: _ContainerName - value: $[ coalesce(variables.ContainerName, 'dotnet') ] + value: $[ coalesce(variables.ContainerName, '$(_DefaultContainerName)') ] - name: _AzureAccessToken - value: $[ coalesce(variables.AzureAccessToken, '$(dotnetcli-storage-key)') ] + value: $[ coalesce(variables.AzureAccessToken, '$(_DefaultAzureAccessToken)') ] - name: _ChecksumAzureAccountName - value: $[ coalesce(variables.ChecksumAzureAccountName, 'dotnetclichecksums') ] + value: $[ coalesce(variables.ChecksumAzureAccountName, '$(_DefaultChecksumAzureAccountName)') ] - name: _ChecksumContainerName - value: $[ coalesce(variables.ChecksumContainerName, 'dotnet') ] + value: $[ coalesce(variables.ChecksumContainerName, '$(_DefaultChecksumsContainerName)') ] - name: _ChecksumAzureAccessToken - value: $[ coalesce(variables.ChecksumAzureAccessToken, '$(dotnetclichecksums-storage-key)') ] + value: $[ coalesce(variables.ChecksumAzureAccessToken, '$(_DefaultChecksumAzureAccessToken)') ] + - name: _CommonPublishArgs value: >- /p:AzureAccountName=$(_AzureAccountName) @@ -51,28 +104,11 @@ jobs: /p:ChecksumContainerName=$(_ChecksumContainerName) /p:ChecksumAzureAccessToken=$(_ChecksumAzureAccessToken) - # dotnet/versions update - - name: _GitHubUser - value: $[ coalesce(variables.GitHubUser, 'dotnet-build-bot') ] - - name: _GitHubEmail - value: $[ coalesce(variables.GitHubEmail, 'dotnet-build-bot@microsoft.com') ] - - name: _GitHubAuthToken - value: $[ coalesce(variables.GitHubAuthToken, '$(AccessToken-dotnet-build-bot-public-repo)') ] - - name: _VersionsRepoOwner - value: $[ coalesce(variables.VersionsRepoOwner, 'dotnet') ] - - name: _VersionsRepo - value: $[ coalesce(variables.VersionsRepo, 'versions') ] - - name: _DotNetVersionsArgs - value: >- - /p:GitHubUser=$(_GitHubUser) - /p:GitHubEmail=$(_GitHubEmail) - /p:GitHubAuthToken=$(_GitHubAuthToken) - /p:VersionsRepoOwner=$(_VersionsRepoOwner) - /p:VersionsRepo=$(_VersionsRepo) - /p:VersionsRepoPath=build-info/dotnet/core-setup/$(FullBranchName) - steps: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: NuGetAuthenticate@0 + - task: DownloadBuildArtifacts@0 displayName: Download Artifacts inputs: diff --git a/eng/jobs/steps/build-linux-package.yml b/eng/jobs/steps/build-linux-package.yml index 1d01738..5e5e562 100644 --- a/eng/jobs/steps/build-linux-package.yml +++ b/eng/jobs/steps/build-linux-package.yml @@ -21,7 +21,7 @@ steps: - script: | set -x df -h - $(DockerRunMSBuild) ${{ parameters.image }} ./build.sh \ + $(DockerRunMSBuild) ${{ parameters.image }} $(BuildScript) \ --ci \ /p:Subset=Installer \ /p:UsePrebuiltPortableBinariesForInstallers=true \ diff --git a/eng/jobs/windows-build.yml b/eng/jobs/windows-build.yml index 0ff3341..7fc1e25 100644 --- a/eng/jobs/windows-build.yml +++ b/eng/jobs/windows-build.yml @@ -41,6 +41,8 @@ jobs: steps: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: NuGetAuthenticate@0 + - task: MicroBuildSigningPlugin@2 displayName: Install MicroBuild plugin for Signing inputs: diff --git a/eng/pipelines/installer/azure-pipelines.yml b/eng/pipelines/installer/azure-pipelines.yml index c8d2187..2c511b5 100644 --- a/eng/pipelines/installer/azure-pipelines.yml +++ b/eng/pipelines/installer/azure-pipelines.yml @@ -183,28 +183,50 @@ stages: # bar: # storage: dependsOnPublishStages: + - dependsOn: NetCore_Dev30_Publish channel: name: .NET Core 3 Dev bar: PublicDevRelease_30_Channel_Id storage: release/3.0 + public: true + - dependsOn: NetCore_Release30_Publish + channel: + name: .NET Core 3 Release + bar: PublicRelease_30_Channel_Id + storage: release/3.0-preview9 + public: true + - dependsOn: NetCore_30_Internal_Servicing_Publish + channel: + name: .NET Core 3 Internal Servicing + bar: InternalServicing_30_Channel_Id + storage: internal/release/3.0 + public: false + - dependsOn: NetCore_Dev31_Publish channel: name: .NET Core 3.1 Dev bar: PublicDevRelease_31_Channel_Id storage: release/3.1 + public: true + - dependsOn: NetCore_Release31_Publish channel: name: .NET Core 3.1 Release bar: PublicRelease_31_Channel_Id storage: release/3.1-preview1 + public: true + - dependsOn: NetCore_Dev5_Publish channel: name: .NET Core 5 Dev bar: NetCore_5_Dev_Channel_Id storage: master + public: true + - dependsOn: PVR_Publish channel: name: .NET Tools - Validation bar: PublicValidationRelease_30_Channel_Id storage: dev/validation + public: true diff --git a/src/installer/test/Directory.Build.props b/src/installer/test/Directory.Build.props index 590382a..4b971c5 100644 --- a/src/installer/test/Directory.Build.props +++ b/src/installer/test/Directory.Build.props @@ -7,6 +7,7 @@ $(ObjDir)TestStabilizedPackages\ $(ObjDir)TestPackageCache\ $(ObjDir)TestNuGetConfig\NuGet.config + $(ObjDir)ExtraNupkgsForTestRestore\ $(TargetArchitecture) netcoreapp3.0 diff --git a/src/installer/test/PrepareTestAssets/PrepareTestAssets.proj b/src/installer/test/PrepareTestAssets/PrepareTestAssets.proj index 2faa657..c4aa79d 100644 --- a/src/installer/test/PrepareTestAssets/PrepareTestAssets.proj +++ b/src/installer/test/PrepareTestAssets/PrepareTestAssets.proj @@ -8,6 +8,7 @@ GetNETCoreAppRuntimePackVersion; CleanTestAssets; GenerateTestRestoreSourcesNuGetConfig; + CopyPotentiallyInternalPackagesForTestRestore; PrepareStabilizedLegacyPackages; RestoreTestAssetProjects" /> @@ -35,6 +36,7 @@ + @@ -48,6 +50,11 @@ + + @@ -58,6 +65,16 @@ $([System.IO.File]::ReadAllText('$(TemplateNuGetConfigFile)').Replace( '<!-- TEST_RESTORE_SOURCES_INSERTION_LINE -->', '$(RestoreTestSourceConfigLines)')) + + + $([System.Text.RegularExpressions.Regex]::Replace( + '$(TestRestoreNuGetConfigContent)', + '<add key=".+" value="https://pkgs.dev.azure.com/dnceng/.+" />', + '')) + + + + + + + + + + + -- 2.7.4