From: Juan Hoyos <19413848+hoyosjs@users.noreply.github.com> Date: Sun, 26 May 2024 00:26:01 +0000 (-0700) Subject: Use managed identity for blob upload of release assets (#4680) X-Git-Tag: accepted/tizen/unified/20241231.014852~40^2~54 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=dc2028e040991926361b5f598ea18c06975113a8;p=platform%2Fcore%2Fdotnet%2Fdiagnostics.git Use managed identity for blob upload of release assets (#4680) - Change release tool to use manage identity for uploads - Use WIF for authentication for release tool - Ensure log publishing doesn't collide on retry - Disable SBOM for logs and test binary assets - Fix token names and add appropriate groups --- diff --git a/diagnostics.yml b/diagnostics.yml index b10ce0795..c4ca6f91c 100644 --- a/diagnostics.yml +++ b/diagnostics.yml @@ -302,10 +302,12 @@ extends: displayName: 'Publish Bundled Tools' condition: succeeded() - output: pipelineArtifact - artifact: Logs_Packaging_Signing + artifact: Logs_Packaging_Signing_Attempt$(System.JobAttempt) path: '$(Build.SourcesDirectory)/artifacts/log' displayName: 'Publish Signing and Packaging Logs' condition: always() + continueOnError: true + sbomEnabled: false # we don't need SBOM for logs steps: - task: DownloadPipelineArtifact@2 displayName: 'Download release builds' diff --git a/eng/pipelines/build.yml b/eng/pipelines/build.yml index 42bb46bcd..a50d32530 100644 --- a/eng/pipelines/build.yml +++ b/eng/pipelines/build.yml @@ -260,6 +260,7 @@ jobs: inputs: targetPath: $(Build.ArtifactStagingDirectory)/artifacts_on_failure artifactName: Artifacts_On_Failure_$(_PhaseName)_$(System.JobAttempt) + sbomEnabled: false # we don't need SBOM for non-shipping diagnostics assets continueOnError: true condition: failed() @@ -278,6 +279,7 @@ jobs: inputs: targetPath: '$(Build.StagingDirectory)/BuildLogs' artifactName: Logs_$(_PhaseName)_$(System.JobAttempt) + sbomEnabled: false # we don't need SBOM for logs continueOnError: true condition: always() diff --git a/eng/pipelines/prepare-release.yml b/eng/pipelines/prepare-release.yml index 3a0d97b4c..41316eb40 100644 --- a/eng/pipelines/prepare-release.yml +++ b/eng/pipelines/prepare-release.yml @@ -3,7 +3,7 @@ stages: displayName: Release Preparation jobs: - job: PrepareReleaseJob - displayName: Prepare release with Darc + displayName: Prepare Release ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}: templateContext: outputs: @@ -15,14 +15,14 @@ stages: variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}: - group: DotNet-Diagnostics-Storage - - group: DotNet-DotNetStage-Storage + - group: DotNetBuilds storage account read tokens - group: Release-Pipeline steps: - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - script: '$(Build.Repository.LocalPath)\dotnet.cmd build $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release /bl' workingDirectory: '$(System.ArtifactsDirectory)' displayName: 'Build Manifest generation and asset publishing tool' - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}: + - ${{ elseif and(ne(variables['System.TeamProject'], 'public'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}: - task: UseDotNet@2 displayName: 'Use .NET Core runtime 6.x' inputs: @@ -37,13 +37,25 @@ stages: filePath: '$(Build.Repository.LocalPath)/eng/release/Scripts/AcquireBuild.ps1' arguments: >- -BarBuildId "$(BARBuildId)" - -AzdoToken "$(dn-bot-dotnet-all-scopes)" - -MaestroToken "$(MaestroAccessToken)" - -GitHubToken "$(BotAccount-dotnet-bot-repo-PAT)" - -DownloadTargetPath "$(System.ArtifactsDirectory)\ReleaseTarget" - -SasSuffixes "$(dotnetclichecksumsmsrc-dotnet-read-list-sas-token),$(dotnetclimsrc-read-sas-token)" -ReleaseVersion "$(Build.BuildNumber)" + -DownloadTargetPath "$(System.ArtifactsDirectory)\ReleaseTarget" + -AzdoToken "$(dn-bot-all-drop-rw-code-rw-release-all)" + -MaestroToken "$(MaestroAccessToken)" + -SasSuffixes "$(dotnetbuilds-internal-checksums-container-read-token),$(dotnetbuilds-internal-container-read-token)" workingDirectory: '$(Build.Repository.LocalPath)' + - task: AzureCLI@2 + displayName: 'Use WIF to obtain credentials for Azure CLI' + inputs: + azureSubscription: 'dotnetstage-diagnostics-tools-rw' + scriptType: pscore + scriptLocation: inlineScript + addSpnToEnvironment: true + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId" + - script: az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: 'Use az to authenticate using managed identity' - script: >- $(Build.Repository.LocalPath)\dotnet.cmd run --project $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release -- @@ -53,9 +65,8 @@ stages: --staging-directory "$(System.ArtifactsDirectory)\ReleaseStaging" --release-name "$(Build.BuildNumber)" --account-name "$(dotnet-diagnostics-storage-accountname)" - --account-key "$(dotnetstage-storage-key)" + --client-id $(ARM_CLIENT_ID) --container-name "$(dotnet-diagnostics-container-name)" - --sas-valid-days "$(dotnet-diagnostics-storage-retentiondays)" -v True workingDirectory: '$(Build.Repository.LocalPath)\' displayName: 'Manifest generation and asset publishing' diff --git a/eng/pipelines/publish-pipeline-artifact-shim.yml b/eng/pipelines/publish-pipeline-artifact-shim.yml index a54ad7b11..05b4987da 100644 --- a/eng/pipelines/publish-pipeline-artifact-shim.yml +++ b/eng/pipelines/publish-pipeline-artifact-shim.yml @@ -3,6 +3,7 @@ parameters: displayName: 'Publish Pipeline Artifact' condition: succeeded() continueOnError: true + enableSbom: true steps: - ${{ if ne(variables['System.TeamProject'], 'public') }}: @@ -11,6 +12,7 @@ steps: inputs: targetPath: ${{ parameters.inputs.targetPath }} artifactName: ${{ parameters.inputs.artifactName }} + enableSbom: ${{ parameters.enableSbom }} condition: ${{ parameters.condition }} displayName: ${{ parameters.displayName }} continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/release/DiagnosticsReleaseTool/Common/AzureBlobPublisher.cs b/eng/release/DiagnosticsReleaseTool/Common/AzureBlobPublisher.cs index c03283a51..aa146cafa 100644 --- a/eng/release/DiagnosticsReleaseTool/Common/AzureBlobPublisher.cs +++ b/eng/release/DiagnosticsReleaseTool/Common/AzureBlobPublisher.cs @@ -7,6 +7,8 @@ using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Azure.Core; +using Azure.Identity; using Azure.Storage; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; @@ -17,17 +19,14 @@ namespace ReleaseTool.Core { public class AzureBlobBublisher : IPublisher { - private const int ClockSkewSec = 15 * 60; private const int MaxRetries = 15; private const int MaxFullLoopRetries = 5; private readonly TimeSpan FullLoopRetryDelay = TimeSpan.FromSeconds(1); - private const string AccessPolicyDownloadId = "DownloadDrop"; private readonly string _accountName; - private readonly string _accountKey; + private readonly string _clientId; private readonly string _containerName; private readonly string _releaseName; - private readonly int _sasValidDays; private readonly ILogger _logger; private BlobContainerClient _client; @@ -40,12 +39,17 @@ namespace ReleaseTool.Core } } - private StorageSharedKeyCredential AccountCredential + private TokenCredential Credentials { get { - StorageSharedKeyCredential credential = new(_accountName, _accountKey); - return credential; + if (_clientId == null) + { + // Local development scenario. Use the default credential. + return new DefaultAzureCredential(); + } + + return new DefaultAzureCredential(new DefaultAzureCredentialOptions { ManagedIdentityClientId = _clientId }); } } @@ -68,13 +72,12 @@ namespace ReleaseTool.Core } } - public AzureBlobBublisher(string accountName, string accountKey, string containerName, string releaseName, int sasValidDays, ILogger logger) + public AzureBlobBublisher(string accountName, string clientId, string containerName, string releaseName, ILogger logger) { _accountName = accountName; - _accountKey = accountKey; + _clientId = clientId; _containerName = containerName; _releaseName = releaseName; - _sasValidDays = sasValidDays; _logger = logger; } @@ -107,20 +110,11 @@ namespace ReleaseTool.Core await blobClient.UploadAsync(srcStream, overwrite: true, ct); - BlobSasBuilder sasBuilder = new() - { - BlobContainerName = client.Name, - BlobName = blobClient.Name, - Identifier = AccessPolicyDownloadId, - Protocol = SasProtocol.Https - }; - Uri accessUri = blobClient.GenerateSasUri(sasBuilder); - using BlobDownloadStreamingResult blobStream = (await blobClient.DownloadStreamingAsync(cancellationToken: ct)).Value; srcStream.Position = 0; completed = await VerifyFileStreamsMatchAsync(srcStream, blobStream, ct); - result = accessUri; + result = blobClient.Uri; } catch (IOException ioEx) when (ioEx is not PathTooLongException) { @@ -155,7 +149,7 @@ namespace ReleaseTool.Core { if (_client == null) { - BlobServiceClient serviceClient = new(AccountBlobUri, AccountCredential, BlobOptions); + BlobServiceClient serviceClient = new(AccountBlobUri, Credentials, BlobOptions); _logger.LogInformation($"Attempting to connect to {serviceClient.Uri} to store blobs."); BlobContainerClient newClient; @@ -165,9 +159,9 @@ namespace ReleaseTool.Core try { newClient = serviceClient.GetBlobContainerClient(_containerName); - if (!(await newClient.ExistsAsync(ct)).Value) + if (!await newClient.ExistsAsync(ct)) { - newClient = (await serviceClient.CreateBlobContainerAsync(_containerName, PublicAccessType.None, metadata: null, ct)); + newClient = await serviceClient.CreateBlobContainerAsync(_containerName, PublicAccessType.None, metadata: null, ct); } } catch (Exception ex) @@ -176,31 +170,6 @@ namespace ReleaseTool.Core continue; } - try - { - DateTime baseTime = DateTime.UtcNow; - // Add the new (or update existing) "download" policy to the container - // This is used to mint the SAS tokens without an expiration policy - // Expiration can be added later by modifying this policy - BlobSignedIdentifier downloadPolicyIdentifier = new() - { - Id = AccessPolicyDownloadId, - AccessPolicy = new BlobAccessPolicy() - { - Permissions = "r", - PolicyStartsOn = new DateTimeOffset(baseTime.AddSeconds(-ClockSkewSec)), - PolicyExpiresOn = new DateTimeOffset(DateTime.UtcNow.AddDays(_sasValidDays).AddSeconds(ClockSkewSec)), - } - }; - _logger.LogInformation($"Writing download access policy: {AccessPolicyDownloadId} to {_containerName}."); - await newClient.SetAccessPolicyAsync(PublicAccessType.None, new BlobSignedIdentifier[] { downloadPolicyIdentifier }, cancellationToken: ct); - } - catch (Exception ex) - { - _logger.LogWarning(ex, $"Failed to write access policy for {_containerName}, retrying."); - continue; - } - _logger.LogInformation($"Container {_containerName} is ready."); _client = newClient; break; diff --git a/eng/release/DiagnosticsReleaseTool/Config.cs b/eng/release/DiagnosticsReleaseTool/Config.cs index 5c90e1c14..bb225340e 100644 --- a/eng/release/DiagnosticsReleaseTool/Config.cs +++ b/eng/release/DiagnosticsReleaseTool/Config.cs @@ -13,9 +13,8 @@ namespace DiagnosticsReleaseTool.Impl public DirectoryInfo StagingDirectory { get; } public string ReleaseName { get; } public string AccountName { get; } - public string AccountKey { get; } + public string ClientId { get; } public string ContainerName { get; } - public int SasValidDays { get; } public Config( FileInfo toolManifest, @@ -24,9 +23,8 @@ namespace DiagnosticsReleaseTool.Impl DirectoryInfo stagingDirectory, string releaseName, string accountName, - string accountKey, - string containerName, - int sasValidDays) + string clientId, + string containerName) { ToolManifest = toolManifest; ShouldVerifyManifest = verifyToolManifest; @@ -34,9 +32,8 @@ namespace DiagnosticsReleaseTool.Impl StagingDirectory = stagingDirectory; ReleaseName = releaseName; AccountName = accountName; - AccountKey = accountKey; + ClientId = clientId; ContainerName = containerName; - SasValidDays = sasValidDays; } } } diff --git a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseCommandLine.cs b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseCommandLine.cs index f4ff9c253..3a5b89589 100644 --- a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseCommandLine.cs +++ b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseCommandLine.cs @@ -38,19 +38,19 @@ namespace DiagnosticsReleaseTool.CommandLine ToolManifestVerificationOption(), DiagnosticLoggingOption(), // Outputs StagingPathOption(), - AzureStorageAccountNameOption(), AzureStorageAccountKeyOption(), AzureStorageContainerNameOption(), AzureStorageSasExpirationOption() + AzureStorageAccountNameOption(), AzureStorageAccountKeyOption(), AzureStorageContainerNameOption() }; private static Option DiagnosticLoggingOption() => new( - aliases: new[] { "-v", "--verbose" }, + aliases: ["-v", "--verbose"], description: "Enables diagnostic logging", getDefaultValue: () => false); private static Option ToolManifestPathOption() => new Option( - aliases: new[] { "--tool-manifest", "-t" }, + aliases: ["--tool-manifest", "-t"], description: "Full path to the manifest of tools and packages to publish.") { IsRequired = true @@ -64,7 +64,7 @@ namespace DiagnosticsReleaseTool.CommandLine private static Option InputDropPathOption() => new Option( - aliases: new[] { "-i", "--input-drop-path" }, + aliases: ["-i", "--input-drop-path"], description: "Path to drop generated by `darc gather-drop`") { IsRequired = true @@ -72,7 +72,7 @@ namespace DiagnosticsReleaseTool.CommandLine private static Option ReleaseNameOption() => new( - aliases: new[] { "-r", "--release-name" }, + aliases: ["-r", "--release-name"], description: "Name of this release.") { IsRequired = true, @@ -80,7 +80,7 @@ namespace DiagnosticsReleaseTool.CommandLine private static Option StagingPathOption() => new Option( - aliases: new[] { "--staging-directory", "-s" }, + aliases: ["--staging-directory", "-s"], description: "Full path to the staging path.", getDefaultValue: () => new DirectoryInfo( Path.Join(Path.GetTempPath(), Path.GetRandomFileName()))) @@ -88,7 +88,7 @@ namespace DiagnosticsReleaseTool.CommandLine private static Option AzureStorageAccountNameOption() => new( - aliases: new[] { "-n", "--account-name" }, + aliases: ["-n", "--account-name"], description: "Storage account name, must be in public azure cloud.") { IsRequired = true, @@ -96,24 +96,19 @@ namespace DiagnosticsReleaseTool.CommandLine private static Option AzureStorageAccountKeyOption() => new( - aliases: new[] { "-k", "--account-key" }, - description: "Storage account key, in base 64 format.") + aliases: ["-k", "--client-id"], + description: "Identity Client ID. If left blank, ambient identity will be used.", + getDefaultValue: () => null) { IsRequired = true, }; private static Option AzureStorageContainerNameOption() => new( - aliases: new[] { "-c", "--container-name" }, + aliases: ["-c", "--container-name"], description: "Storage account container name where the files will be uploaded.") { IsRequired = true, }; - - private static Option AzureStorageSasExpirationOption() => - new( - aliases: new[] { "--sas-valid-days" }, - description: "Number of days to allow access to the blobs via the provided SAS URIs.", - getDefaultValue: () => 1); } } diff --git a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseRunner.cs b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseRunner.cs index f8d94d09f..e83032b22 100644 --- a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseRunner.cs +++ b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseRunner.cs @@ -51,7 +51,7 @@ namespace DiagnosticsReleaseTool.Impl DirectoryInfo basePublishDirectory = darcLayoutHelper.GetShippingDirectoryForSingleProjectVariants(DiagnosticsRepoHelpers.ProductNames); string publishManifestPath = Path.Combine(releaseConfig.StagingDirectory.FullName, ManifestName); - IPublisher releasePublisher = new AzureBlobBublisher(releaseConfig.AccountName, releaseConfig.AccountKey, releaseConfig.ContainerName, releaseConfig.ReleaseName, releaseConfig.SasValidDays, logger); + IPublisher releasePublisher = new AzureBlobBublisher(releaseConfig.AccountName, releaseConfig.ClientId, releaseConfig.ContainerName, releaseConfig.ReleaseName, logger); IManifestGenerator manifestGenerator = new DiagnosticsManifestGenerator(releaseMetadata, releaseConfig.ToolManifest, logger); using Release diagnosticsRelease = new( diff --git a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseTool.csproj b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseTool.csproj index c64b60cfe..b67722c9d 100644 --- a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseTool.csproj +++ b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseTool.csproj @@ -21,7 +21,8 @@ - + + diff --git a/eng/release/DiagnosticsReleaseTool/DiagnosticsRepoHelpers.cs b/eng/release/DiagnosticsReleaseTool/DiagnosticsRepoHelpers.cs index 3c99df391..79a811b55 100644 --- a/eng/release/DiagnosticsReleaseTool/DiagnosticsRepoHelpers.cs +++ b/eng/release/DiagnosticsReleaseTool/DiagnosticsRepoHelpers.cs @@ -10,8 +10,8 @@ namespace DiagnosticsReleaseTool.Util { public static class DiagnosticsRepoHelpers { - public static readonly string[] ProductNames = new[] { "diagnostics", "dotnet-diagnostics" }; - public static readonly string[] RepositoryUrls = new[] { "https://github.com/dotnet/diagnostics", "https://dev.azure.com/dnceng/internal/_git/dotnet-diagnostics" }; + public static readonly string[] ProductNames = ["diagnostics", "dotnet-diagnostics"]; + public static readonly string[] RepositoryUrls = ["https://github.com/dotnet/diagnostics", "https://dev.azure.com/dnceng/internal/_git/dotnet-diagnostics"]; public static string BundleToolsPathInDrop => System.IO.Path.Combine("diagnostics", "bundledtools"); public const string BundledToolsPrefix = "diagnostic-tools-"; public const string BundledToolsCategory = "ToolBundleAssets"; diff --git a/eng/release/Scripts/AcquireBuild.ps1 b/eng/release/Scripts/AcquireBuild.ps1 index ecab93729..87154dab0 100644 --- a/eng/release/Scripts/AcquireBuild.ps1 +++ b/eng/release/Scripts/AcquireBuild.ps1 @@ -5,7 +5,6 @@ param( [Parameter(Mandatory=$true)][string] $SasSuffixes, [Parameter(Mandatory=$true)][string] $AzdoToken, [Parameter(Mandatory=$true)][string] $MaestroToken, - [Parameter(Mandatory=$true)][string] $GitHubToken, [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', [Parameter(Mandatory=$false)][string] $DarcVersion = $null, [switch] $help, @@ -19,7 +18,6 @@ function Write-Help() { Write-Host " -SasSuffixes Comma separated list of potential uri suffixes that can be used if anonymous access to a blob uri fails. Appended directly to the end of the URI. Use full SAS syntax with ?." Write-Host " -AzdoToken Azure DevOps token to use for builds queries" Write-Host " -MaestroToken Maestro token to use for querying BAR" - Write-Host " -GitHubToken GitHub token to use for querying repository information" Write-Host " -MaestroApiEndPoint BAR endpoint to use for build queries." Write-Host "" } @@ -55,7 +53,6 @@ try { --output-dir $DownloadTargetPath ` --overwrite ` --sas-suffixes $SasSuffixes ` - --github-pat $GitHubToken ` --azdev-pat $AzdoToken ` --bar-uri $MaestroApiEndPoint ` --password $MaestroToken `