displayName: 'Publish Bundled Tools'
condition: succeeded()
- output: pipelineArtifact
- artifact: Logs_Packaging_Signing
+ artifact: Logs_Packaging_Signing_Attempt$(System.JobAttempt)
path: '$(Build.SourcesDirectory)/artifacts/log'
displayName: 'Publish Signing and Packaging Logs'
condition: always()
+ continueOnError: true
+ sbomEnabled: false # we don't need SBOM for logs
steps:
- task: DownloadPipelineArtifact@2
displayName: 'Download release builds'
inputs:
targetPath: $(Build.ArtifactStagingDirectory)/artifacts_on_failure
artifactName: Artifacts_On_Failure_$(_PhaseName)_$(System.JobAttempt)
+ sbomEnabled: false # we don't need SBOM for non-shipping diagnostics assets
continueOnError: true
condition: failed()
inputs:
targetPath: '$(Build.StagingDirectory)/BuildLogs'
artifactName: Logs_$(_PhaseName)_$(System.JobAttempt)
+ sbomEnabled: false # we don't need SBOM for logs
continueOnError: true
condition: always()
displayName: Release Preparation
jobs:
- job: PrepareReleaseJob
- displayName: Prepare release with Darc
+ displayName: Prepare Release
${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}:
templateContext:
outputs:
variables:
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}:
- group: DotNet-Diagnostics-Storage
- - group: DotNet-DotNetStage-Storage
+ - group: DotNetBuilds storage account read tokens
- group: Release-Pipeline
steps:
- ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
- script: '$(Build.Repository.LocalPath)\dotnet.cmd build $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release /bl'
workingDirectory: '$(System.ArtifactsDirectory)'
displayName: 'Build Manifest generation and asset publishing tool'
- - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}:
+ - ${{ elseif and(ne(variables['System.TeamProject'], 'public'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}:
- task: UseDotNet@2
displayName: 'Use .NET Core runtime 6.x'
inputs:
filePath: '$(Build.Repository.LocalPath)/eng/release/Scripts/AcquireBuild.ps1'
arguments: >-
-BarBuildId "$(BARBuildId)"
- -AzdoToken "$(dn-bot-dotnet-all-scopes)"
- -MaestroToken "$(MaestroAccessToken)"
- -GitHubToken "$(BotAccount-dotnet-bot-repo-PAT)"
- -DownloadTargetPath "$(System.ArtifactsDirectory)\ReleaseTarget"
- -SasSuffixes "$(dotnetclichecksumsmsrc-dotnet-read-list-sas-token),$(dotnetclimsrc-read-sas-token)"
-ReleaseVersion "$(Build.BuildNumber)"
+ -DownloadTargetPath "$(System.ArtifactsDirectory)\ReleaseTarget"
+ -AzdoToken "$(dn-bot-all-drop-rw-code-rw-release-all)"
+ -MaestroToken "$(MaestroAccessToken)"
+ -SasSuffixes "$(dotnetbuilds-internal-checksums-container-read-token),$(dotnetbuilds-internal-container-read-token)"
workingDirectory: '$(Build.Repository.LocalPath)'
+ - task: AzureCLI@2
+ displayName: 'Use WIF to obtain credentials for Azure CLI'
+ inputs:
+ azureSubscription: 'dotnetstage-diagnostics-tools-rw'
+ scriptType: pscore
+ scriptLocation: inlineScript
+ addSpnToEnvironment: true
+ inlineScript: |
+ echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId"
+ echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken"
+ echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId"
+ - script: az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN)
+ displayName: 'Use az to authenticate using managed identity'
- script: >-
$(Build.Repository.LocalPath)\dotnet.cmd run --project $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release
--
--staging-directory "$(System.ArtifactsDirectory)\ReleaseStaging"
--release-name "$(Build.BuildNumber)"
--account-name "$(dotnet-diagnostics-storage-accountname)"
- --account-key "$(dotnetstage-storage-key)"
+ --client-id $(ARM_CLIENT_ID)
--container-name "$(dotnet-diagnostics-container-name)"
- --sas-valid-days "$(dotnet-diagnostics-storage-retentiondays)"
-v True
workingDirectory: '$(Build.Repository.LocalPath)\'
displayName: 'Manifest generation and asset publishing'
displayName: 'Publish Pipeline Artifact'
condition: succeeded()
continueOnError: true
+ enableSbom: true
steps:
- ${{ if ne(variables['System.TeamProject'], 'public') }}:
inputs:
targetPath: ${{ parameters.inputs.targetPath }}
artifactName: ${{ parameters.inputs.artifactName }}
+ enableSbom: ${{ parameters.enableSbom }}
condition: ${{ parameters.condition }}
displayName: ${{ parameters.displayName }}
continueOnError: ${{ parameters.continueOnError }}
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
+using Azure.Core;
+using Azure.Identity;
using Azure.Storage;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
{
public class AzureBlobBublisher : IPublisher
{
- private const int ClockSkewSec = 15 * 60;
private const int MaxRetries = 15;
private const int MaxFullLoopRetries = 5;
private readonly TimeSpan FullLoopRetryDelay = TimeSpan.FromSeconds(1);
- private const string AccessPolicyDownloadId = "DownloadDrop";
private readonly string _accountName;
- private readonly string _accountKey;
+ private readonly string _clientId;
private readonly string _containerName;
private readonly string _releaseName;
- private readonly int _sasValidDays;
private readonly ILogger _logger;
private BlobContainerClient _client;
}
}
- private StorageSharedKeyCredential AccountCredential
+ private TokenCredential Credentials
{
get
{
- StorageSharedKeyCredential credential = new(_accountName, _accountKey);
- return credential;
+ if (_clientId == null)
+ {
+ // Local development scenario. Use the default credential.
+ return new DefaultAzureCredential();
+ }
+
+ return new DefaultAzureCredential(new DefaultAzureCredentialOptions { ManagedIdentityClientId = _clientId });
}
}
}
}
- public AzureBlobBublisher(string accountName, string accountKey, string containerName, string releaseName, int sasValidDays, ILogger logger)
+ public AzureBlobBublisher(string accountName, string clientId, string containerName, string releaseName, ILogger logger)
{
_accountName = accountName;
- _accountKey = accountKey;
+ _clientId = clientId;
_containerName = containerName;
_releaseName = releaseName;
- _sasValidDays = sasValidDays;
_logger = logger;
}
await blobClient.UploadAsync(srcStream, overwrite: true, ct);
- BlobSasBuilder sasBuilder = new()
- {
- BlobContainerName = client.Name,
- BlobName = blobClient.Name,
- Identifier = AccessPolicyDownloadId,
- Protocol = SasProtocol.Https
- };
- Uri accessUri = blobClient.GenerateSasUri(sasBuilder);
-
using BlobDownloadStreamingResult blobStream = (await blobClient.DownloadStreamingAsync(cancellationToken: ct)).Value;
srcStream.Position = 0;
completed = await VerifyFileStreamsMatchAsync(srcStream, blobStream, ct);
- result = accessUri;
+ result = blobClient.Uri;
}
catch (IOException ioEx) when (ioEx is not PathTooLongException)
{
{
if (_client == null)
{
- BlobServiceClient serviceClient = new(AccountBlobUri, AccountCredential, BlobOptions);
+ BlobServiceClient serviceClient = new(AccountBlobUri, Credentials, BlobOptions);
_logger.LogInformation($"Attempting to connect to {serviceClient.Uri} to store blobs.");
BlobContainerClient newClient;
try
{
newClient = serviceClient.GetBlobContainerClient(_containerName);
- if (!(await newClient.ExistsAsync(ct)).Value)
+ if (!await newClient.ExistsAsync(ct))
{
- newClient = (await serviceClient.CreateBlobContainerAsync(_containerName, PublicAccessType.None, metadata: null, ct));
+ newClient = await serviceClient.CreateBlobContainerAsync(_containerName, PublicAccessType.None, metadata: null, ct);
}
}
catch (Exception ex)
continue;
}
- try
- {
- DateTime baseTime = DateTime.UtcNow;
- // Add the new (or update existing) "download" policy to the container
- // This is used to mint the SAS tokens without an expiration policy
- // Expiration can be added later by modifying this policy
- BlobSignedIdentifier downloadPolicyIdentifier = new()
- {
- Id = AccessPolicyDownloadId,
- AccessPolicy = new BlobAccessPolicy()
- {
- Permissions = "r",
- PolicyStartsOn = new DateTimeOffset(baseTime.AddSeconds(-ClockSkewSec)),
- PolicyExpiresOn = new DateTimeOffset(DateTime.UtcNow.AddDays(_sasValidDays).AddSeconds(ClockSkewSec)),
- }
- };
- _logger.LogInformation($"Writing download access policy: {AccessPolicyDownloadId} to {_containerName}.");
- await newClient.SetAccessPolicyAsync(PublicAccessType.None, new BlobSignedIdentifier[] { downloadPolicyIdentifier }, cancellationToken: ct);
- }
- catch (Exception ex)
- {
- _logger.LogWarning(ex, $"Failed to write access policy for {_containerName}, retrying.");
- continue;
- }
-
_logger.LogInformation($"Container {_containerName} is ready.");
_client = newClient;
break;
public DirectoryInfo StagingDirectory { get; }
public string ReleaseName { get; }
public string AccountName { get; }
- public string AccountKey { get; }
+ public string ClientId { get; }
public string ContainerName { get; }
- public int SasValidDays { get; }
public Config(
FileInfo toolManifest,
DirectoryInfo stagingDirectory,
string releaseName,
string accountName,
- string accountKey,
- string containerName,
- int sasValidDays)
+ string clientId,
+ string containerName)
{
ToolManifest = toolManifest;
ShouldVerifyManifest = verifyToolManifest;
StagingDirectory = stagingDirectory;
ReleaseName = releaseName;
AccountName = accountName;
- AccountKey = accountKey;
+ ClientId = clientId;
ContainerName = containerName;
- SasValidDays = sasValidDays;
}
}
}
ToolManifestVerificationOption(), DiagnosticLoggingOption(),
// Outputs
StagingPathOption(),
- AzureStorageAccountNameOption(), AzureStorageAccountKeyOption(), AzureStorageContainerNameOption(), AzureStorageSasExpirationOption()
+ AzureStorageAccountNameOption(), AzureStorageAccountKeyOption(), AzureStorageContainerNameOption()
};
private static Option<bool> DiagnosticLoggingOption() =>
new(
- aliases: new[] { "-v", "--verbose" },
+ aliases: ["-v", "--verbose"],
description: "Enables diagnostic logging",
getDefaultValue: () => false);
private static Option ToolManifestPathOption() =>
new Option<FileInfo>(
- aliases: new[] { "--tool-manifest", "-t" },
+ aliases: ["--tool-manifest", "-t"],
description: "Full path to the manifest of tools and packages to publish.")
{
IsRequired = true
private static Option<DirectoryInfo> InputDropPathOption() =>
new Option<DirectoryInfo>(
- aliases: new[] { "-i", "--input-drop-path" },
+ aliases: ["-i", "--input-drop-path"],
description: "Path to drop generated by `darc gather-drop`")
{
IsRequired = true
private static Option<string> ReleaseNameOption() =>
new(
- aliases: new[] { "-r", "--release-name" },
+ aliases: ["-r", "--release-name"],
description: "Name of this release.")
{
IsRequired = true,
private static Option StagingPathOption() =>
new Option<DirectoryInfo>(
- aliases: new[] { "--staging-directory", "-s" },
+ aliases: ["--staging-directory", "-s"],
description: "Full path to the staging path.",
getDefaultValue: () => new DirectoryInfo(
Path.Join(Path.GetTempPath(), Path.GetRandomFileName())))
private static Option<string> AzureStorageAccountNameOption() =>
new(
- aliases: new[] { "-n", "--account-name" },
+ aliases: ["-n", "--account-name"],
description: "Storage account name, must be in public azure cloud.")
{
IsRequired = true,
private static Option<string> AzureStorageAccountKeyOption() =>
new(
- aliases: new[] { "-k", "--account-key" },
- description: "Storage account key, in base 64 format.")
+ aliases: ["-k", "--client-id"],
+ description: "Identity Client ID. If left blank, ambient identity will be used.",
+ getDefaultValue: () => null)
{
IsRequired = true,
};
private static Option<string> AzureStorageContainerNameOption() =>
new(
- aliases: new[] { "-c", "--container-name" },
+ aliases: ["-c", "--container-name"],
description: "Storage account container name where the files will be uploaded.")
{
IsRequired = true,
};
-
- private static Option<int> AzureStorageSasExpirationOption() =>
- new(
- aliases: new[] { "--sas-valid-days" },
- description: "Number of days to allow access to the blobs via the provided SAS URIs.",
- getDefaultValue: () => 1);
}
}
DirectoryInfo basePublishDirectory = darcLayoutHelper.GetShippingDirectoryForSingleProjectVariants(DiagnosticsRepoHelpers.ProductNames);
string publishManifestPath = Path.Combine(releaseConfig.StagingDirectory.FullName, ManifestName);
- IPublisher releasePublisher = new AzureBlobBublisher(releaseConfig.AccountName, releaseConfig.AccountKey, releaseConfig.ContainerName, releaseConfig.ReleaseName, releaseConfig.SasValidDays, logger);
+ IPublisher releasePublisher = new AzureBlobBublisher(releaseConfig.AccountName, releaseConfig.ClientId, releaseConfig.ContainerName, releaseConfig.ReleaseName, logger);
IManifestGenerator manifestGenerator = new DiagnosticsManifestGenerator(releaseMetadata, releaseConfig.ToolManifest, logger);
using Release diagnosticsRelease = new(
<PackageReference Include="Microsoft.Extensions.Logging.Configuration" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0" />
- <PackageReference Include="Azure.Storage.Blobs" Version="[12.13.0]" />
+ <PackageReference Include="Azure.Identity" Version="[1.11.3]" />
+ <PackageReference Include="Azure.Storage.Blobs" Version="[12.20.0]" />
<PackageReference Include="System.CommandLine" Version="2.0.0-beta1.20468.1" />
</ItemGroup>
{
public static class DiagnosticsRepoHelpers
{
- public static readonly string[] ProductNames = new[] { "diagnostics", "dotnet-diagnostics" };
- public static readonly string[] RepositoryUrls = new[] { "https://github.com/dotnet/diagnostics", "https://dev.azure.com/dnceng/internal/_git/dotnet-diagnostics" };
+ public static readonly string[] ProductNames = ["diagnostics", "dotnet-diagnostics"];
+ public static readonly string[] RepositoryUrls = ["https://github.com/dotnet/diagnostics", "https://dev.azure.com/dnceng/internal/_git/dotnet-diagnostics"];
public static string BundleToolsPathInDrop => System.IO.Path.Combine("diagnostics", "bundledtools");
public const string BundledToolsPrefix = "diagnostic-tools-";
public const string BundledToolsCategory = "ToolBundleAssets";
[Parameter(Mandatory=$true)][string] $SasSuffixes,
[Parameter(Mandatory=$true)][string] $AzdoToken,
[Parameter(Mandatory=$true)][string] $MaestroToken,
- [Parameter(Mandatory=$true)][string] $GitHubToken,
[Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
[Parameter(Mandatory=$false)][string] $DarcVersion = $null,
[switch] $help,
Write-Host " -SasSuffixes <value> Comma separated list of potential uri suffixes that can be used if anonymous access to a blob uri fails. Appended directly to the end of the URI. Use full SAS syntax with ?."
Write-Host " -AzdoToken <value> Azure DevOps token to use for builds queries"
Write-Host " -MaestroToken <value> Maestro token to use for querying BAR"
- Write-Host " -GitHubToken <value> GitHub token to use for querying repository information"
Write-Host " -MaestroApiEndPoint <value> BAR endpoint to use for build queries."
Write-Host ""
}
--output-dir $DownloadTargetPath `
--overwrite `
--sas-suffixes $SasSuffixes `
- --github-pat $GitHubToken `
--azdev-pat $AzdoToken `
--bar-uri $MaestroApiEndPoint `
--password $MaestroToken `
projects / platform / core / dotnet / diagnostics.git / commitdiff