From: Juan Hoyos <19413848+hoyosjs@users.noreply.github.com> Date: Wed, 6 Oct 2021 11:00:50 +0000 (-0700) Subject: Add blob publishing capabilities to the release tool (#2640) X-Git-Tag: submit/tizen/20220302.040122~21^2^2~29 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=8ecf053f091f859c01f8ff80615d902a98e729e0;p=platform%2Fcore%2Fdotnet%2Fdiagnostics.git Add blob publishing capabilities to the release tool (#2640) * Add blob publishing capabilities to the release tool - Retarget to net6.0 - Cleanup Co-authored-by: Patrick Fenelon * Add release tool invocation * fixup! Add release tool invocation * Account for internal-only builds * Always verify hash if available Co-authored-by: Patrick Fenelon --- diff --git a/diagnostics.yml b/diagnostics.yml index b02b0b034..95f59a541 100644 --- a/diagnostics.yml +++ b/diagnostics.yml @@ -432,3 +432,6 @@ stages: -TsaPublish $True' artifactNames: - 'Packages' + + # This sets up the bits to do a Release. + - template: /eng/prepare-release.yml diff --git a/eng/prepare-release.yml b/eng/prepare-release.yml new file mode 100644 index 000000000..9709e7564 --- /dev/null +++ b/eng/prepare-release.yml @@ -0,0 +1,69 @@ +stages: +- stage: PrepareReleaseStage + displayName: Release Preparation + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}: + dependsOn: + - publish_using_darc + jobs: + - job: PrepareReleaseJob + displayName: Prepare release with Darc + pool: + vmImage: windows-latest + variables: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}: + - group: DotNet-Diagnostics-Storage + - group: DotNet-DotNetStage-Storage + - group: Release-Pipeline + - name: BARBuildId + value: $[ stageDependencies.publish_using_darc.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + steps: + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - script: '$(Build.Repository.LocalPath)\dotnet.cmd build $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release /bl' + workingDirectory: '$(System.ArtifactsDirectory)' + displayName: 'Build Manifest generation and asset publishing tool' + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '$(System.ArtifactsDirectory)' + publishLocation: 'pipeline' + artifact: 'DiagnosticsReleaseToolBin' + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}: + - task: UseDotNet@2 + displayName: 'Use .NET Core runtime 3.1.x' + inputs: + packageType: runtime + version: 3.1.x + installationPath: '$(Build.Repository.LocalPath)\.dotnet' + - task: PowerShell@2 + displayName: 'DARC Gather build' + inputs: + targetType: filePath + filePath: '$(Build.Repository.LocalPath)/eng/release/Scripts/AcquireBuild.ps1' + arguments: >- + -BarBuildId "$(BARBuildId)" + -AzdoToken "$(dn-bot-dotnet-all-scopes)" + -MaestroToken "$(MaestroAccessToken)" + -GitHubToken "$(BotAccount-dotnet-bot-repo-PAT)" + -DownloadTargetPath "$(System.ArtifactsDirectory)\ReleaseTarget" + -SasSuffixes "$(dotnetclichecksumsmsrc-dotnet-read-list-sas-token),$(dotnetclimsrc-read-sas-token)" + -ReleaseVersion "$(Build.BuildNumber)" + workingDirectory: '$(Build.Repository.LocalPath)' + - script: >- + dotnet.cmd run --project $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release + -- + prepare-release + --input-drop-path "$(System.ArtifactsDirectory)\ReleaseTarget" + --tool-manifest "$(Build.Repository.LocalPath)\eng\release\tool-list.json" + --staging-directory "$(System.ArtifactsDirectory)\ReleaseStaging" + --release-name "$(Build.BuildNumber)" + --account-name "$(dotnet-diagnostics-storage-accountname)" + --account-key "$(dotnetstage-storage-key)" + --container-name "$(dotnet-diagnostics-container-name)" + --sas-valid-days "$(dotnet-diagnostics-storage-retentiondays)" + -v True + workingDirectory: '$(Build.Repository.LocalPath)\' + displayName: 'Manifest generation and asset publishing' + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '$(System.ArtifactsDirectory)\ReleaseStaging' + publishLocation: 'pipeline' + artifact: 'DiagnosticsRelease' \ No newline at end of file diff --git a/eng/release/DiagnosticsReleaseTool/Common/AzureBlobPublisher.cs b/eng/release/DiagnosticsReleaseTool/Common/AzureBlobPublisher.cs new file mode 100644 index 000000000..06f8e49ef --- /dev/null +++ b/eng/release/DiagnosticsReleaseTool/Common/AzureBlobPublisher.cs @@ -0,0 +1,264 @@ +using Azure; +using Azure.Storage; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Sas; +using Microsoft.Extensions.Logging; +using System; +using System.Buffers; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace ReleaseTool.Core +{ + public class AzureBlobBublisher : IPublisher + { + private const int ClockSkewSec = 15 * 60; + private const int MaxRetries = 15; + private const int MaxFullLoopRetries = 5; + private readonly TimeSpan FullLoopRetryDelay = TimeSpan.FromSeconds(1); + private const string AccessPolicyDownloadId = "DownloadDrop"; + + private readonly string _accountName; + private readonly string _accountKey; + private readonly string _containerName; + private readonly string _releaseName; + private readonly int _sasValidDays; + private readonly ILogger _logger; + + private BlobContainerClient _client; + + private Uri AccountBlobUri + { + get + { + return new Uri(FormattableString.Invariant($"https://{_accountName}.blob.core.windows.net")); + } + } + + private StorageSharedKeyCredential AccountCredential + { + get + { + StorageSharedKeyCredential credential = new StorageSharedKeyCredential(_accountName, _accountKey); + return credential; + } + } + + private BlobClientOptions BlobOptions + { + get + { + // The Azure SDK client has it's own built in retry logic + // We want to allow more and longer retries because this + // is a publishing operation that happens once and can be + // allowed to take a very long time. We have a high + // tolerance for slow operations and a low tolerance for failure. + return new BlobClientOptions() + { + Retry = + { + MaxRetries = MaxRetries, + } + }; + } + } + + public AzureBlobBublisher(string accountName, string accountKey, string containerName, string releaseName, int sasValidDays, ILogger logger) + { + _accountName = accountName; + _accountKey = accountKey; + _containerName = containerName; + _releaseName = releaseName; + _sasValidDays = sasValidDays; + _logger = logger; + } + + public void Dispose() + { + } + + public async Task PublishFileAsync(FileMapping fileMap, CancellationToken ct) + { + Uri result = null; + int retriesLeft = MaxFullLoopRetries; + TimeSpan loopDelay = FullLoopRetryDelay; + bool completed = false; + + do + { + _logger.LogInformation($"Attempting to publish {fileMap.RelativeOutputPath}, {retriesLeft} tries left."); + try + { + BlobContainerClient client = await GetClient(ct); + if (client == null) + { + // client creation failed, return + return null; + } + + using var srcStream = new FileStream(fileMap.LocalSourcePath, FileMode.Open, FileAccess.Read); + + BlobClient blobClient = client.GetBlobClient(GetBlobName(_releaseName, fileMap.RelativeOutputPath)); + + await blobClient.UploadAsync(srcStream, overwrite: true, ct); + + BlobSasBuilder sasBuilder = new BlobSasBuilder() + { + BlobContainerName = client.Name, + BlobName = blobClient.Name, + Identifier = AccessPolicyDownloadId, + Protocol = SasProtocol.Https + }; + Uri accessUri = blobClient.GenerateSasUri(sasBuilder); + + using BlobDownloadStreamingResult blobStream = (await blobClient.DownloadStreamingAsync(cancellationToken: ct)).Value; + srcStream.Position = 0; + completed = await VerifyFileStreamsMatchAsync(srcStream, blobStream, ct); + + result = accessUri; + } + catch (IOException ioEx) when (!(ioEx is PathTooLongException)) + { + _logger.LogWarning(ioEx, $"Failed to publish {fileMap.LocalSourcePath}, retries remaining: {retriesLeft}."); + + /* Retry IO exceptions */ + retriesLeft--; + loopDelay *= 2; + + if (retriesLeft > 0) + { + await Task.Delay(loopDelay, ct); + } + } + catch (Exception ex) + { + // Azure errors have their own built-in retry logic, so just abort if we got an AzureResponseException + _logger.LogWarning(ex, $"Failed to publish {fileMap.LocalSourcePath}, unexpected error, aborting."); + return null; + } + } while (retriesLeft > 0 && !completed); + + return result?.OriginalString; + } + + private static string GetBlobName(string releaseName, string relativeFilePath) + { + return FormattableString.Invariant($"{releaseName}/{relativeFilePath}"); + } + + private async Task GetClient(CancellationToken ct) + { + if (_client == null) + { + BlobServiceClient serviceClient = new BlobServiceClient(AccountBlobUri, AccountCredential, BlobOptions); + _logger.LogInformation($"Attempting to connect to {serviceClient.Uri} to store blobs."); + + BlobContainerClient newClient; + int attemptCt = 0; + do + { + try + { + newClient = serviceClient.GetBlobContainerClient(_containerName); + if (!(await newClient.ExistsAsync(ct)).Value) + { + newClient = (await serviceClient.CreateBlobContainerAsync(_containerName, PublicAccessType.None, metadata: null, ct)); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, $"Failed to create or access {_containerName}, retrying with new name."); + continue; + } + + try + { + DateTime baseTime = DateTime.UtcNow; + // Add the new (or update existing) "download" policy to the container + // This is used to mint the SAS tokens without an expiration policy + // Expiration can be added later by modifying this policy + BlobSignedIdentifier downloadPolicyIdentifier = new BlobSignedIdentifier() + { + Id = AccessPolicyDownloadId, + AccessPolicy = new BlobAccessPolicy() + { + Permissions = "r", + PolicyStartsOn = new DateTimeOffset(baseTime.AddSeconds(-ClockSkewSec)), + PolicyExpiresOn = new DateTimeOffset(DateTime.UtcNow.AddDays(_sasValidDays).AddSeconds(ClockSkewSec)), + } + }; + _logger.LogInformation($"Writing download access policy: {AccessPolicyDownloadId} to {_containerName}."); + await newClient.SetAccessPolicyAsync(PublicAccessType.None, new BlobSignedIdentifier[] { downloadPolicyIdentifier }, cancellationToken: ct); + } + catch (Exception ex) + { + _logger.LogWarning(ex, $"Failed to write access policy for {_containerName}, retrying."); + continue; + } + + _logger.LogInformation($"Container {_containerName} is ready."); + _client = newClient; + break; + } while (++attemptCt < MaxFullLoopRetries); + } + + if (_client == null) + { + _logger.LogError("Failed to create or access container for publishing drop."); + } + return _client; + } + + private async Task VerifyFileStreamsMatchAsync(FileStream srcStream, BlobDownloadStreamingResult destBlobDownloadStream, CancellationToken ct) + { + if (srcStream.Length != destBlobDownloadStream.Details.ContentLength) + { + return false; + } + + using Stream destStream = destBlobDownloadStream.Content; + + using IMemoryOwner memOwnerSrc = MemoryPool.Shared.Rent(minBufferSize: 16_384); + using IMemoryOwner memOwnerDest = MemoryPool.Shared.Rent(minBufferSize: 16_384); + Memory memSrc = memOwnerSrc.Memory; + Memory memDest = memOwnerDest.Memory; + + int bytesProcessed = 0; + int srcBytesRemainingFromPrevRead = 0; + int destBytesRemainingFromPrevRead = 0; + + while (bytesProcessed != srcStream.Length) + { + int srcBytesRead = await srcStream.ReadAsync(memSrc.Slice(srcBytesRemainingFromPrevRead), ct); + srcBytesRead += srcBytesRemainingFromPrevRead; + int destBytesRead = await destStream.ReadAsync(memDest.Slice(destBytesRemainingFromPrevRead), ct); + destBytesRead += destBytesRemainingFromPrevRead; + + int bytesToCompare = Math.Min(srcBytesRead, destBytesRead); + + if (bytesToCompare == 0) + { + return false; + } + + bytesProcessed += bytesToCompare; + srcBytesRemainingFromPrevRead = srcBytesRead - bytesToCompare; + destBytesRemainingFromPrevRead = destBytesRead - bytesToCompare; + + bool isChunkEquals = memDest.Span.Slice(0, bytesToCompare).SequenceEqual(memSrc.Span.Slice(0, bytesToCompare)); + if (!isChunkEquals) + { + return false; + } + + memSrc.Slice(bytesToCompare, srcBytesRemainingFromPrevRead).CopyTo(memSrc); + memDest.Slice(bytesToCompare, destBytesRemainingFromPrevRead).CopyTo(memDest); + } + + return true; + } + } +} \ No newline at end of file diff --git a/eng/release/DiagnosticsReleaseTool/Common/NugetLayoutWorker.cs b/eng/release/DiagnosticsReleaseTool/Common/NugetLayoutWorker.cs index c48f522f7..d187b9001 100644 --- a/eng/release/DiagnosticsReleaseTool/Common/NugetLayoutWorker.cs +++ b/eng/release/DiagnosticsReleaseTool/Common/NugetLayoutWorker.cs @@ -5,13 +5,10 @@ namespace ReleaseTool.Core public sealed class NugetLayoutWorker : PassThroughLayoutWorker { public NugetLayoutWorker(string stagingPath) : base( - shouldHandleFileFunc: ShouldHandleFile, - getRelativePublishPathFromFileFunc: GetNugetPublishRelativePath, - getMetadataForFileFunc: (_) => new FileMetadata(FileClass.Nuget), + shouldHandleFileFunc: static file => file.Extension == ".nupkg" && !file.Name.EndsWith(".symbols.nupkg"), + getRelativePublishPathFromFileFunc: static file => Helpers.GetDefaultPathForFileCategory(file, FileClass.Nuget), + getMetadataForFileFunc: static file => Helpers.GetDefaultFileMetadata(file, FileClass.Nuget), stagingPath - ) {} - - private static bool ShouldHandleFile(FileInfo file) => file.Extension == ".nupkg" && !file.Name.EndsWith(".symbols.nupkg"); - private static string GetNugetPublishRelativePath(FileInfo file) => FileMetadata.GetDefaultCatgoryForClass(FileClass.Nuget); + ){} } } \ No newline at end of file diff --git a/eng/release/DiagnosticsReleaseTool/Common/PassThroughLayoutWorker.cs b/eng/release/DiagnosticsReleaseTool/Common/PassThroughLayoutWorker.cs index 508fc9030..dd67d6bd5 100644 --- a/eng/release/DiagnosticsReleaseTool/Common/PassThroughLayoutWorker.cs +++ b/eng/release/DiagnosticsReleaseTool/Common/PassThroughLayoutWorker.cs @@ -19,11 +19,11 @@ namespace ReleaseTool.Core string stagingPath) { - _shouldHandleFileFunc = shouldHandleFileFunc ?? (_ => true); + _shouldHandleFileFunc = shouldHandleFileFunc ?? (static _ => true); - _getRelativePublishPathFromFileFunc = getRelativePublishPathFromFileFunc ?? (file => Path.Combine(FileMetadata.GetDefaultCatgoryForClass(FileClass.Unknown), file.Name)); + _getRelativePublishPathFromFileFunc = getRelativePublishPathFromFileFunc ?? (static file => Helpers.GetDefaultPathForFileCategory(file, FileClass.Unknown)); - _getMetadataForFileFunc = getMetadataForFileFunc ?? (_ => new FileMetadata(FileClass.Unknown)); + _getMetadataForFileFunc = getMetadataForFileFunc ?? (static file => Helpers.GetDefaultFileMetadata(file, FileClass.Unknown)); _stagingPath = stagingPath; } @@ -37,7 +37,7 @@ namespace ReleaseTool.Core return new LayoutWorkerResult(LayoutResultStatus.FileNotHandled); } - string publishReleasePath = Path.Combine(_getRelativePublishPathFromFileFunc(file), file.Name); + string publishReleasePath = _getRelativePublishPathFromFileFunc(file); string localPath = file.FullName; diff --git a/eng/release/DiagnosticsReleaseTool/Common/ReleaseToolHelpers.cs b/eng/release/DiagnosticsReleaseTool/Common/ReleaseToolHelpers.cs new file mode 100644 index 000000000..4815150ee --- /dev/null +++ b/eng/release/DiagnosticsReleaseTool/Common/ReleaseToolHelpers.cs @@ -0,0 +1,32 @@ +using System; +using System.IO; + +namespace ReleaseTool.Core +{ + static class Helpers + { + internal static string GetDefaultPathForFileCategory(FileInfo file, FileClass fileClass) + { + string category = FileMetadata.GetDefaultCatgoryForClass(fileClass); + return FormattableString.Invariant($"{category}/{file.Name}"); + } + + internal static FileMetadata GetDefaultFileMetadata(FileInfo fileInfo, FileClass fileClass) + { + string sha512Hash = GetSha512(fileInfo); + FileMetadata result = new FileMetadata( + fileClass, + FileMetadata.GetDefaultCatgoryForClass(fileClass), + sha512: sha512Hash); + return result; + } + + internal static string GetSha512(FileInfo fileInfo) + { + using FileStream fileReadStream = fileInfo.OpenRead(); + using var sha = System.Security.Cryptography.SHA512.Create(); + byte[] hashValueBytes = sha.ComputeHash(fileReadStream); + return Convert.ToHexString(hashValueBytes); + } + } +} \ No newline at end of file diff --git a/eng/release/DiagnosticsReleaseTool/Common/SymbolPackageLayoutWorker.cs b/eng/release/DiagnosticsReleaseTool/Common/SymbolPackageLayoutWorker.cs index ae062960d..45274ffda 100644 --- a/eng/release/DiagnosticsReleaseTool/Common/SymbolPackageLayoutWorker.cs +++ b/eng/release/DiagnosticsReleaseTool/Common/SymbolPackageLayoutWorker.cs @@ -5,13 +5,10 @@ namespace ReleaseTool.Core public class SymbolPackageLayoutWorker : PassThroughLayoutWorker { public SymbolPackageLayoutWorker(string stagingPath) : base( - shouldHandleFileFunc: ShouldHandleFile, - getRelativePublishPathFromFileFunc: GetSymbolPackagePublishRelativePath, - getMetadataForFileFunc: (_) => new FileMetadata(FileClass.SymbolPackage), + shouldHandleFileFunc: static file => file.Name.EndsWith(".symbols.nupkg"), + getRelativePublishPathFromFileFunc: static file => Helpers.GetDefaultPathForFileCategory(file, FileClass.SymbolPackage), + getMetadataForFileFunc: static file => Helpers.GetDefaultFileMetadata(file, FileClass.SymbolPackage), stagingPath ) {} - - private static bool ShouldHandleFile(FileInfo file) => file.Name.EndsWith(".symbols.nupkg"); - private static string GetSymbolPackagePublishRelativePath(FileInfo file) => FileMetadata.GetDefaultCatgoryForClass(FileClass.SymbolPackage); } } \ No newline at end of file diff --git a/eng/release/DiagnosticsReleaseTool/Common/ZipLayoutWorker.cs b/eng/release/DiagnosticsReleaseTool/Common/ZipLayoutWorker.cs index 405479371..78f393297 100644 --- a/eng/release/DiagnosticsReleaseTool/Common/ZipLayoutWorker.cs +++ b/eng/release/DiagnosticsReleaseTool/Common/ZipLayoutWorker.cs @@ -22,9 +22,12 @@ namespace ReleaseTool.Core _shouldHandleFileFunc = shouldHandleFileFunc ?? (file => file.Extension == ".zip"); - _getRelativePathFromZipAndInnerFileFunc = getRelativePathFromZipAndInnerFileFunc ?? ((zipFile, innerFile) => Path.Combine(zipFile.Name, innerFile.Name)); + Func defaultgetRelPathFunc = static (zipFile, innerFile) => + FormattableString.Invariant($"{Path.GetFileNameWithoutExtension(zipFile.Name)}/{innerFile.Name}"); - _getMetadataForInnerFileFunc = getMetadataForInnerFileFunc ?? ((_, _) => new FileMetadata(FileClass.Blob)); + _getRelativePathFromZipAndInnerFileFunc = getRelativePathFromZipAndInnerFileFunc ?? defaultgetRelPathFunc; + + _getMetadataForInnerFileFunc = getMetadataForInnerFileFunc ?? (static (_, innerFile) => Helpers.GetDefaultFileMetadata(innerFile, FileClass.Blob)); _stagingPath = stagingPath; } @@ -66,8 +69,6 @@ namespace ReleaseTool.Core } string relativePath = _getRelativePathFromZipAndInnerFileFunc(file, extractedFile); - relativePath = Path.Combine(relativePath, extractedFile.Name); - string localPath = extractedFile.FullName; if (_stagingPath is not null) diff --git a/eng/release/DiagnosticsReleaseTool/Config.cs b/eng/release/DiagnosticsReleaseTool/Config.cs index b4f9b16c2..7e326b0ee 100644 --- a/eng/release/DiagnosticsReleaseTool/Config.cs +++ b/eng/release/DiagnosticsReleaseTool/Config.cs @@ -8,16 +8,32 @@ namespace DiagnosticsReleaseTool.Impl public bool ShouldVerifyManifest { get; } public DirectoryInfo DropPath { get; } public DirectoryInfo StagingDirectory { get; } - public string PublishPath { get; } + public string ReleaseName { get; } + public string AccountName { get; } + public string AccountKey { get; } + public string ContainerName { get; } + public int SasValidDays { get; } - public Config(FileInfo toolManifest, bool verifyToolManifest, - DirectoryInfo inputDropPath, DirectoryInfo stagingDirectory, string publishPath) + public Config( + FileInfo toolManifest, + bool verifyToolManifest, + DirectoryInfo inputDropPath, + DirectoryInfo stagingDirectory, + string releaseName, + string accountName, + string accountKey, + string containerName, + int sasValidDays) { ToolManifest = toolManifest; ShouldVerifyManifest = verifyToolManifest; DropPath = inputDropPath; StagingDirectory = stagingDirectory; - PublishPath = publishPath; + ReleaseName = releaseName; + AccountName = accountName; + AccountKey = accountKey; + ContainerName = containerName; + SasValidDays = sasValidDays; } } } \ No newline at end of file diff --git a/eng/release/DiagnosticsReleaseTool/Core/FileMetadata.cs b/eng/release/DiagnosticsReleaseTool/Core/FileMetadata.cs index 130f65e36..bcdc5ea72 100644 --- a/eng/release/DiagnosticsReleaseTool/Core/FileMetadata.cs +++ b/eng/release/DiagnosticsReleaseTool/Core/FileMetadata.cs @@ -14,7 +14,7 @@ namespace ReleaseTool.Core { public readonly FileClass Class { get; } - public readonly string AssetCategory { get; } + public readonly string AssetCategory { get; } public readonly bool ShouldPublishToCdn { get; } @@ -24,13 +24,10 @@ namespace ReleaseTool.Core // TODO: Add a metadata bag for Key,Value pairs. - public FileMetadata(FileClass fileClass) - : this(fileClass, GetDefaultCatgoryForClass(fileClass)) {} + public FileMetadata(FileClass fileClass, string assetCategory, string sha512) + : this(fileClass, assetCategory, shouldPublishToCdn: false, rid: "any", sha512: sha512) {} - public FileMetadata(FileClass fileClass, string assetCategory) - : this(fileClass, assetCategory, shouldPublishToCdn: false, rid: "any", sha512: null) {} - - public FileMetadata(FileClass fileClass, string assetCategory, bool shouldPublishToCdn, string rid, string sha512) + public FileMetadata(FileClass fileClass, string assetCategory, bool shouldPublishToCdn, string rid, string sha512) { if (string.IsNullOrEmpty(assetCategory)) { diff --git a/eng/release/DiagnosticsReleaseTool/DarcHelpers.cs b/eng/release/DiagnosticsReleaseTool/DarcHelpers.cs index 204f1201f..bdfc66a3e 100644 --- a/eng/release/DiagnosticsReleaseTool/DarcHelpers.cs +++ b/eng/release/DiagnosticsReleaseTool/DarcHelpers.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Text.Json; @@ -28,7 +29,7 @@ namespace DiagnosticsReleaseTool.Util } } - internal ReleaseMetadata GetDropMetadata(string repoUrl) + internal ReleaseMetadata GetDropMetadataForSingleRepoVariants(IEnumerable repoUrls) { string releaseVersion; using (Stream darcReleaseFile = File.OpenRead(ReleaseFilePath)) @@ -44,22 +45,25 @@ namespace DiagnosticsReleaseTool.Util // TODO: Schema validation. JsonElement buildList = jsonDoc.RootElement.GetProperty("builds"); - // TODO: This should be using Uri.Compare... + // This iteration is necessary due to the public/private nature repos. var repoBuilds = buildList.EnumerateArray() - .Where(build => build.GetProperty("repo").GetString() == repoUrl); + .Where(build => + { + var buildUri = new Uri(build.GetProperty("repo").GetString()); + return repoUrls.Any(repoUrl => buildUri == new Uri(repoUrl)); + }); if (repoBuilds.Count() != 1) { throw new InvalidOperationException( - $"There's either no build for {repoUrl} or more than one. Can't retrieve metadata."); + $"There's either no build for requested repos or more than one. Can't retrieve metadata."); } - JsonElement build = repoBuilds.ElementAt(0); + JsonElement build = repoBuilds.First(); - // TODO: If any of these were to fail... var releaseMetadata = new ReleaseMetadata( releaseVersion: releaseVersion, - repoUrl: repoUrl, + repoUrl: build.GetProperty("repo").GetString(), branch: build.GetProperty("branch").GetString(), commit: build.GetProperty("commit").GetString(), dateProduced: build.GetProperty("produced").GetString(), @@ -71,7 +75,7 @@ namespace DiagnosticsReleaseTool.Util } } - internal DirectoryInfo GetShippingDirectoryForProject(string projectName) + internal DirectoryInfo GetShippingDirectoryForSingleProjectVariants(IEnumerable projectNames) { using (Stream darcManifest = File.OpenRead(ReleaseFilePath)) using (JsonDocument jsonDoc = JsonDocument.Parse(darcManifest)) @@ -80,17 +84,16 @@ namespace DiagnosticsReleaseTool.Util // pretty stable schema. JsonElement productList = jsonDoc.RootElement[0].GetProperty("products"); - var directoryList = productList.EnumerateArray() - .Where(prod => prod.GetProperty("name").GetString() == projectName) - .Select(prod => prod.GetProperty("fileshare")); + var matchingProducts = productList.EnumerateArray() + .Where(prod => projectNames.Contains(prod.GetProperty("name").GetString())); - if (directoryList.Count() != 1) + if (matchingProducts.Count() != 1) { throw new InvalidOperationException( - $"There's either no product named {projectName} or more than one in the drop."); + $"There's either no product under the provided names or more than one in the drop."); } - return new DirectoryInfo(directoryList.ElementAt(0).GetString()); + return new DirectoryInfo(matchingProducts.First().GetProperty("fileshare").GetString()); } } } diff --git a/eng/release/DiagnosticsReleaseTool/DiagnosticsManifestGenerator.cs b/eng/release/DiagnosticsReleaseTool/DiagnosticsManifestGenerator.cs index ea96732b2..03e394625 100644 --- a/eng/release/DiagnosticsReleaseTool/DiagnosticsManifestGenerator.cs +++ b/eng/release/DiagnosticsReleaseTool/DiagnosticsManifestGenerator.cs @@ -2,7 +2,9 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text.Encodings.Web; using System.Text.Json; +using System.Text.Json.Serialization; using System.Text.RegularExpressions; using DiagnosticsReleaseTool.Util; using Microsoft.Extensions.Logging; @@ -33,7 +35,13 @@ namespace DiagnosticsReleaseTool.Impl { var stream = new MemoryStream(); - using (var writer = new Utf8JsonWriter(stream, new JsonWriterOptions{ Indented = true })) + var jro = new JsonWriterOptions + { + Indented = true, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; + + using (var writer = new Utf8JsonWriter(stream, jro)) { writer.WriteStartObject(); @@ -65,6 +73,7 @@ namespace DiagnosticsReleaseTool.Impl writer.WriteString("Rid", fileToRelease.FileMetadata.Rid); writer.WriteString("PublishRelativePath", fileToRelease.FileMap.RelativeOutputPath); writer.WriteString("PublishedPath", fileToRelease.PublishUri); + writer.WriteString("Sha512", fileToRelease.FileMetadata.Sha512); writer.WriteEndObject(); } @@ -83,6 +92,7 @@ namespace DiagnosticsReleaseTool.Impl writer.WriteStartObject(); writer.WriteString("PublishRelativePath", fileToRelease.FileMap.RelativeOutputPath); writer.WriteString("PublishedPath", fileToRelease.PublishUri); + writer.WriteString("Sha512", fileToRelease.FileMetadata.Sha512); writer.WriteEndObject(); } @@ -96,7 +106,7 @@ namespace DiagnosticsReleaseTool.Impl var options = new JsonSerializerOptions { - IgnoreNullValues = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, WriteIndented = true }; @@ -138,7 +148,7 @@ namespace DiagnosticsReleaseTool.Impl private string GenerateSubpath(FileReleaseData fileToRelease) { var fi = new FileInfo(fileToRelease.FileMap.LocalSourcePath); - using var hash = System.Security.Cryptography.SHA256Managed.Create(); + using var hash = System.Security.Cryptography.SHA256.Create(); var enc = System.Text.Encoding.UTF8; byte[] hashResult = hash.ComputeHash(enc.GetBytes(fileToRelease.FileMap.RelativeOutputPath)); string pathHash = BitConverter.ToString(hashResult).Replace("-", String.Empty); @@ -160,7 +170,7 @@ namespace DiagnosticsReleaseTool.Impl { if(!match.Groups.TryGetValue("metadata", out Group metadataGroup)) { - // Give up if the catpturing failed + // Give up if the capturing failed return null; } diff --git a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseCommandLine.cs b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseCommandLine.cs index fbd38ea6f..e6e37528e 100644 --- a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseCommandLine.cs +++ b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseCommandLine.cs @@ -30,13 +30,15 @@ namespace DiagnosticsReleaseTool.CommandLine { CommandHandler.Create(DiagnosticsReleaseRunner.PrepareRelease), // Inputs - InputDropPathOption(), ToolManifestPathOption(), + InputDropPathOption(), ToolManifestPathOption(), ReleaseNameOption(), // Toggles ToolManifestVerificationOption(), DiagnosticLoggingOption(), // Outputs - StagingPathOption(), PublishPathOption() + StagingPathOption(), + AzureStorageAccountNameOption(), AzureStorageAccountKeyOption(), AzureStorageContainerNameOption(), AzureStorageSasExpirationOption() }; + private static Option DiagnosticLoggingOption() => new Option( aliases: new[] { "-v", "--verbose" }, @@ -65,6 +67,14 @@ namespace DiagnosticsReleaseTool.CommandLine IsRequired = true }.ExistingOnly(); + private static Option ReleaseNameOption() => + new Option( + aliases: new[] { "-r", "--release-name" }, + description: "Name of this release.") + { + IsRequired = true, + }; + private static Option StagingPathOption() => new Option( aliases: new[] { "--staging-directory", "-s" }, @@ -73,12 +83,34 @@ namespace DiagnosticsReleaseTool.CommandLine Path.Join(Path.GetTempPath(), Path.GetRandomFileName()))) .LegalFilePathsOnly(); - private static Option PublishPathOption() => + private static Option AzureStorageAccountNameOption() => new Option( - aliases: new[] { "-o", "--publish-path" }, - description: "Path to publish the generated layout and publishing manifest to.") + aliases: new[] { "-n", "--account-name" }, + description: "Storage account name, must be in public azure cloud.") { - IsRequired = true + IsRequired = true, }; + + private static Option AzureStorageAccountKeyOption() => + new Option( + aliases: new[] { "-k", "--account-key" }, + description: "Storage account key, in base 64 format.") + { + IsRequired = true, + }; + + private static Option AzureStorageContainerNameOption() => + new Option( + aliases: new[] { "-c", "--container-name" }, + description: "Storage account container name where the files will be uploaded.") + { + IsRequired = true, + }; + + private static Option AzureStorageSasExpirationOption() => + new Option( + aliases: new[] { "--sas-valid-days" }, + description: "Number of days to allow access to the blobs via the provided SAS URIs.", + getDefaultValue: () => 1); } } diff --git a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseRunner.cs b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseRunner.cs index f5ecb4170..fe4808669 100644 --- a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseRunner.cs +++ b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseRunner.cs @@ -25,13 +25,13 @@ namespace DiagnosticsReleaseTool.Impl var layoutWorkerList = new List { // TODO: We may want to inject a logger. - new NugetLayoutWorker(stagingPath: null), - new SymbolPackageLayoutWorker(stagingPath: null), + new NugetLayoutWorker(stagingPath: releaseConfig.StagingDirectory.FullName), + new SymbolPackageLayoutWorker(stagingPath: releaseConfig.StagingDirectory.FullName), new ZipLayoutWorker( shouldHandleFileFunc: DiagnosticsRepoHelpers.IsBundledToolArchive, getRelativePathFromZipAndInnerFileFunc: DiagnosticsRepoHelpers.GetToolPublishRelativePath, getMetadataForInnerFileFunc: DiagnosticsRepoHelpers.GetMetadataForToolFile, - stagingPath: null + stagingPath: releaseConfig.StagingDirectory.FullName ) }; @@ -44,11 +44,11 @@ namespace DiagnosticsReleaseTool.Impl } // TODO: Probably should use BAR ID instead as an identifier for the metadata to gather. - ReleaseMetadata releaseMetadata = darcLayoutHelper.GetDropMetadata(DiagnosticsRepoHelpers.RepositoryName); - DirectoryInfo basePublishDirectory = darcLayoutHelper.GetShippingDirectoryForProject(DiagnosticsRepoHelpers.ProductName); + ReleaseMetadata releaseMetadata = darcLayoutHelper.GetDropMetadataForSingleRepoVariants(DiagnosticsRepoHelpers.RepositoryUrls); + DirectoryInfo basePublishDirectory = darcLayoutHelper.GetShippingDirectoryForSingleProjectVariants(DiagnosticsRepoHelpers.ProductNames); string publishManifestPath = Path.Combine(releaseConfig.StagingDirectory.FullName, ManifestName); - IPublisher releasePublisher = new FileSharePublisher(releaseConfig.PublishPath); + IPublisher releasePublisher = new AzureBlobBublisher(releaseConfig.AccountName, releaseConfig.AccountKey, releaseConfig.ContainerName, releaseConfig.ReleaseName, releaseConfig.SasValidDays, logger); IManifestGenerator manifestGenerator = new DiagnosticsManifestGenerator(releaseMetadata, releaseConfig.ToolManifest, logger); using var diagnosticsRelease = new Release( diff --git a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseTool.csproj b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseTool.csproj index 2214e25cd..08643e034 100644 --- a/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseTool.csproj +++ b/eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseTool.csproj @@ -2,7 +2,7 @@ Exe - net5.0 + net6.0 @@ -12,13 +12,14 @@ - - - - - - + + + + + + + diff --git a/eng/release/DiagnosticsReleaseTool/DiagnosticsRepoHelpers.cs b/eng/release/DiagnosticsReleaseTool/DiagnosticsRepoHelpers.cs index 97b8f5985..7e6ca75a8 100644 --- a/eng/release/DiagnosticsReleaseTool/DiagnosticsRepoHelpers.cs +++ b/eng/release/DiagnosticsReleaseTool/DiagnosticsRepoHelpers.cs @@ -7,8 +7,8 @@ namespace DiagnosticsReleaseTool.Util { public static class DiagnosticsRepoHelpers { - public const string ProductName = "diagnostics"; - public const string RepositoryName = "https://github.com/dotnet/diagnostics"; + public static readonly string[] ProductNames = new []{ "diagnostics", "dotnet-diagnostics" }; + public static readonly string[] RepositoryUrls = new [] { "https://github.com/dotnet/diagnostics", "https://dev.azure.com/dnceng/internal/_git/dotnet-diagnostics" }; public static string BundleToolsPathInDrop => System.IO.Path.Combine("diagnostics", "bundledtools"); public const string BundledToolsPrefix = "diagnostic-tools-"; public const string BundledToolsCategory = "ToolBundleAssets"; @@ -50,14 +50,9 @@ namespace DiagnosticsReleaseTool.Util _ => "UnknownAssets" }; - string sha512 = null; + string sha512 = GetSha512(fileInZip.FullName); string rid = GetRidFromBundleZip(zipFile); - if (category == BundledToolsCategory) - { - sha512 = GetSha512(fileInZip.FullName); - } - return new FileMetadata( FileClass.Blob, assetCategory: category, @@ -68,7 +63,7 @@ namespace DiagnosticsReleaseTool.Util public static string GetToolPublishRelativePath(FileInfo zipFile, FileInfo fileInZip) { - return Path.Combine(BundledToolsCategory, GetRidFromBundleZip(zipFile)); + return FormattableString.Invariant($"{BundledToolsCategory}/{GetRidFromBundleZip(zipFile)}/{fileInZip.Name}"); } public static bool IsBundledToolArchive(FileInfo file) @@ -80,12 +75,10 @@ namespace DiagnosticsReleaseTool.Util public static string GetSha512(string filePath) { - using (FileStream stream = System.IO.File.OpenRead(filePath)) - { - var sha = new System.Security.Cryptography.SHA512Managed(); - byte[] checksum = sha.ComputeHash(stream); - return BitConverter.ToString(checksum).Replace("-", String.Empty); - } + using FileStream stream = System.IO.File.OpenRead(filePath); + using var sha = System.Security.Cryptography.SHA512.Create(); + byte[] checksum = sha.ComputeHash(stream); + return Convert.ToHexString(checksum); } } } \ No newline at end of file diff --git a/eng/release/Scripts/PublishToNuget.ps1 b/eng/release/Scripts/PublishToNuget.ps1 index 226064384..ae9c6bdaa 100644 --- a/eng/release/Scripts/PublishToNuget.ps1 +++ b/eng/release/Scripts/PublishToNuget.ps1 @@ -57,6 +57,12 @@ foreach ($nugetPack in $manifestJson.NugetAssets) Invoke-WebRequest -Uri $nugetPack.PublishedPath -OutFile (New-Item -Path $packagePath -Force) $progressPreference = 'Continue' + if ($nugetPack.PSobject.Properties.Name.Contains("Sha512")-and $(Get-FileHash -Algorithm sha512 $packagePath).Hash -ne $nugetPack.Sha512) { + Write-Host "Sha512 verification failed for $($nugetPack.PublishRelativePath)." + $failedToPublish++ + continue + } + Write-Host "Publishing $packagePath." & "$PSScriptRoot/../../../dotnet.cmd" nuget push $packagePath --source $FeedEndpoint --api-key $FeedPat if ($LastExitCode -ne 0)