From 508f8a798d8d4972bcbadc2aa45c51e8be58da2d Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Tue, 25 May 2021 15:49:29 +0200 Subject: [PATCH] Deterministic bundling issue 3601 (#52930) https://github.com/dotnet/runtime/issues/3601 Bundling should generate id based on the content in order to secure unique but reproducible ids --- .../Microsoft.NET.HostModel/Bundle/Bundler.cs | 2 +- .../Microsoft.NET.HostModel/Bundle/Manifest.cs | 40 ++++++++++++++++++-- .../BundlerConsistencyTests.cs | 44 ++++++++++++++++++++++ 3 files changed, 81 insertions(+), 5 deletions(-) diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs index 83a6762..603ddba 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs @@ -322,7 +322,7 @@ namespace Microsoft.NET.HostModel.Bundle { FileType targetType = Target.TargetSpecificFileType(type); (long startOffset, long compressedSize) = AddToBundle(bundle, file, targetType); - FileEntry entry = BundleManifest.AddEntry(targetType, relativePath, startOffset, file.Length, compressedSize, Target.BundleMajorVersion); + FileEntry entry = BundleManifest.AddEntry(targetType, file, relativePath, startOffset, compressedSize, Target.BundleMajorVersion); Tracer.Log($"Embed: {entry}"); } } diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs index 73e7b3b..5d58732 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Security.Cryptography; namespace Microsoft.NET.HostModel.Bundle { @@ -65,7 +66,10 @@ namespace Microsoft.NET.HostModel.Bundle // identify this bundle. It is choosen to be compatible // with path-names so that the AppHost can use it in // extraction path. - public readonly string BundleID; + public string BundleID { get; private set; } + //Same as Path.GetRandomFileName + private const int BundleIdLength = 12; + private SHA256 bundleHash = SHA256.Create(); public readonly uint BundleMajorVersion; // The Minor version is currently unused, and is always zero public const uint BundleMinorVersion = 0; @@ -79,15 +83,23 @@ namespace Microsoft.NET.HostModel.Bundle { BundleMajorVersion = bundleMajorVersion; Files = new List(); - BundleID = Path.GetRandomFileName(); Flags = (netcoreapp3CompatMode) ? HeaderFlags.NetcoreApp3CompatMode : HeaderFlags.None; } - public FileEntry AddEntry(FileType type, string relativePath, long offset, long size, long compressedSize, uint bundleMajorVersion) + public FileEntry AddEntry(FileType type, FileStream fileContent, string relativePath, long offset, long compressedSize, uint bundleMajorVersion) { - FileEntry entry = new FileEntry(type, relativePath, offset, size, compressedSize, bundleMajorVersion); + if (bundleHash == null) + { + throw new InvalidOperationException("It is forbidden to change Manifest state after it was written or BundleId was obtained."); + } + + FileEntry entry = new FileEntry(type, relativePath, offset, fileContent.Length, compressedSize, bundleMajorVersion); Files.Add(entry); + fileContent.Position = 0; + byte[] hashBytes = ComputeSha256Hash(fileContent); + bundleHash.TransformBlock(hashBytes, 0, hashBytes.Length, hashBytes, 0); + switch (entry.Type) { case FileType.DepsJson: @@ -107,8 +119,28 @@ namespace Microsoft.NET.HostModel.Bundle return entry; } + private static byte[] ComputeSha256Hash(Stream stream) + { + using (SHA256 sha = SHA256.Create()) + { + return sha.ComputeHash(stream); + } + } + + private string GenerateDeterministicId() + { + bundleHash.TransformFinalBlock(Array.Empty(), 0, 0); + byte[] manifestHash = bundleHash.Hash; + bundleHash.Dispose(); + bundleHash = null; + + return Convert.ToBase64String(manifestHash).Substring(BundleIdLength).Replace('/', '_'); + } + public long Write(BinaryWriter writer) { + BundleID = BundleID ?? GenerateDeterministicId(); + long startOffset = writer.BaseStream.Position; // Write the bundle header diff --git a/src/installer/tests/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs b/src/installer/tests/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs index 75d9e44..6a84380 100644 --- a/src/installer/tests/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs +++ b/src/installer/tests/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs @@ -144,6 +144,50 @@ namespace Microsoft.NET.HostModel.Tests bundler.BundleManifest.Files.Where(entry => entry.RelativePath.Equals("rel/app.Repeat.dll")).Single().Type.Should().Be(FileType.Assembly); } + private (string bundleFileName, string bundleId) CreateSampleBundle(bool bundleMultipleFiles) + { + var fixture = sharedTestState.TestFixture.Copy(); + + var hostName = BundleHelper.GetHostName(fixture); + var bundleDir = Directory.CreateDirectory( + Path.Combine(BundleHelper.GetBundleDir(fixture).FullName, Path.GetRandomFileName())); + var targetOS = BundleHelper.GetTargetOS(fixture.CurrentRid); + var targetArch = BundleHelper.GetTargetArch(fixture.CurrentRid); + + var fileSpecs = new List(); + fileSpecs.Add(new FileSpec(BundleHelper.GetHostPath(fixture), BundleHelper.GetHostName(fixture))); + if (bundleMultipleFiles) + { + fileSpecs.Add(new FileSpec(BundleHelper.GetAppPath(fixture), "rel/app.repeat.dll")); + } + + Bundler bundler = new Bundler(hostName, bundleDir.FullName, targetOS: targetOS, targetArch: targetArch); + return (bundler.GenerateBundle(fileSpecs), bundler.BundleManifest.BundleID); + } + + [Fact] + public void TestWithIdenticalBundlesShouldBeBinaryEqualPasses() + { + var firstBundle = CreateSampleBundle(true); + byte[] firstBundleContent = File.ReadAllBytes(firstBundle.bundleFileName); + var secondBundle = CreateSampleBundle(true); + byte[] secondBundleContent = File.ReadAllBytes(secondBundle.bundleFileName); + + firstBundle.bundleId.ShouldBeEquivalentTo(secondBundle.bundleId, + "Deterministic/Reproducible build should produce identical bundle id for identical inputs"); + firstBundleContent.ShouldBeEquivalentTo(secondBundleContent, + "Deterministic/Reproducible build should produce identical binary for identical inputs"); + } + + [Fact] + public void TestWithUniqueBundlesShouldHaveUniqueBundleIdsPasses() + { + string firstBundle = CreateSampleBundle(true).bundleId; + string secondBundle = CreateSampleBundle(false).bundleId; + + Assert.NotEqual(firstBundle, secondBundle, StringComparer.Ordinal); + } + [Fact] public void TestWithMultipleDuplicateEntriesFails() { -- 2.7.4