Deterministic bundling issue 3601 (#52930)

https://github.com/dotnet/runtime/issues/3601
Bundling should generate id based on the content in order to secure unique but reproducible ids

diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs
index 83a6762..603ddba 100644 (file)
--- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs
+++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs
@@ -322,7 +322,7 @@ namespace Microsoft.NET.HostModel.Bundle
                     {
                         FileType targetType = Target.TargetSpecificFileType(type);
                         (long startOffset, long compressedSize) = AddToBundle(bundle, file, targetType);
-                        FileEntry entry = BundleManifest.AddEntry(targetType, relativePath, startOffset, file.Length, compressedSize, Target.BundleMajorVersion);
+                        FileEntry entry = BundleManifest.AddEntry(targetType, file, relativePath, startOffset, compressedSize, Target.BundleMajorVersion);
                         Tracer.Log($"Embed: {entry}");
                     }
                 }

diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs
index 73e7b3b..5d58732 100644 (file)
--- a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs
+++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs
@@ -5,6 +5,7 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Security.Cryptography;
 
 namespace Microsoft.NET.HostModel.Bundle
 {
@@ -65,7 +66,10 @@ namespace Microsoft.NET.HostModel.Bundle
         // identify this bundle. It is choosen to be compatible
         // with path-names so that the AppHost can use it in
         // extraction path.
-        public readonly string BundleID;
+        public string BundleID { get; private set; }
+        //Same as Path.GetRandomFileName
+        private const int BundleIdLength = 12;
+        private SHA256 bundleHash = SHA256.Create();
         public readonly uint BundleMajorVersion;
         // The Minor version is currently unused, and is always zero
         public const uint BundleMinorVersion = 0;
@@ -79,15 +83,23 @@ namespace Microsoft.NET.HostModel.Bundle
         {
             BundleMajorVersion = bundleMajorVersion;
             Files = new List<FileEntry>();
-            BundleID = Path.GetRandomFileName();
             Flags = (netcoreapp3CompatMode) ? HeaderFlags.NetcoreApp3CompatMode : HeaderFlags.None;
         }
 
-        public FileEntry AddEntry(FileType type, string relativePath, long offset, long size, long compressedSize, uint bundleMajorVersion)
+        public FileEntry AddEntry(FileType type, FileStream fileContent, string relativePath, long offset, long compressedSize, uint bundleMajorVersion)
         {
-            FileEntry entry = new FileEntry(type, relativePath, offset, size, compressedSize, bundleMajorVersion);
+            if (bundleHash == null)
+            {
+                throw new InvalidOperationException("It is forbidden to change Manifest state after it was written or BundleId was obtained.");
+            }
+
+            FileEntry entry = new FileEntry(type, relativePath, offset, fileContent.Length, compressedSize, bundleMajorVersion);
             Files.Add(entry);
 
+            fileContent.Position = 0;
+            byte[] hashBytes = ComputeSha256Hash(fileContent);
+            bundleHash.TransformBlock(hashBytes, 0, hashBytes.Length, hashBytes, 0);
+
             switch (entry.Type)
             {
                 case FileType.DepsJson:
@@ -107,8 +119,28 @@ namespace Microsoft.NET.HostModel.Bundle
             return entry;
         }
 
+        private static byte[] ComputeSha256Hash(Stream stream)
+        {
+            using (SHA256 sha = SHA256.Create())
+            {
+                return sha.ComputeHash(stream);
+            }
+        }
+
+        private string GenerateDeterministicId()
+        {
+            bundleHash.TransformFinalBlock(Array.Empty<byte>(), 0, 0);
+            byte[] manifestHash = bundleHash.Hash;
+            bundleHash.Dispose();
+            bundleHash = null;
+
+            return Convert.ToBase64String(manifestHash).Substring(BundleIdLength).Replace('/', '_');
+        }
+
         public long Write(BinaryWriter writer)
         {
+            BundleID = BundleID ?? GenerateDeterministicId();
+
             long startOffset = writer.BaseStream.Position;
 
             // Write the bundle header

diff --git a/src/installer/tests/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs b/src/installer/tests/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs
index 75d9e44..6a84380 100644 (file)
--- a/src/installer/tests/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs
+++ b/src/installer/tests/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Bundle.Tests/BundlerConsistencyTests.cs
@@ -144,6 +144,50 @@ namespace Microsoft.NET.HostModel.Tests
             bundler.BundleManifest.Files.Where(entry => entry.RelativePath.Equals("rel/app.Repeat.dll")).Single().Type.Should().Be(FileType.Assembly);
         }
 
+        private (string bundleFileName, string bundleId) CreateSampleBundle(bool bundleMultipleFiles)
+        {
+            var fixture = sharedTestState.TestFixture.Copy();
+
+            var hostName = BundleHelper.GetHostName(fixture);
+            var bundleDir = Directory.CreateDirectory(
+                Path.Combine(BundleHelper.GetBundleDir(fixture).FullName, Path.GetRandomFileName()));
+            var targetOS = BundleHelper.GetTargetOS(fixture.CurrentRid);
+            var targetArch = BundleHelper.GetTargetArch(fixture.CurrentRid);
+
+            var fileSpecs = new List<FileSpec>();
+            fileSpecs.Add(new FileSpec(BundleHelper.GetHostPath(fixture), BundleHelper.GetHostName(fixture)));
+            if (bundleMultipleFiles)
+            {
+                fileSpecs.Add(new FileSpec(BundleHelper.GetAppPath(fixture), "rel/app.repeat.dll"));
+            }
+
+            Bundler bundler = new Bundler(hostName, bundleDir.FullName, targetOS: targetOS, targetArch: targetArch);
+            return (bundler.GenerateBundle(fileSpecs), bundler.BundleManifest.BundleID);
+        }
+
+        [Fact]
+        public void TestWithIdenticalBundlesShouldBeBinaryEqualPasses()
+        {
+            var firstBundle = CreateSampleBundle(true);
+            byte[] firstBundleContent = File.ReadAllBytes(firstBundle.bundleFileName);
+            var secondBundle = CreateSampleBundle(true);
+            byte[] secondBundleContent = File.ReadAllBytes(secondBundle.bundleFileName);
+
+            firstBundle.bundleId.ShouldBeEquivalentTo(secondBundle.bundleId,
+                "Deterministic/Reproducible build should produce identical bundle id for identical inputs");
+            firstBundleContent.ShouldBeEquivalentTo(secondBundleContent,
+                "Deterministic/Reproducible build should produce identical binary for identical inputs");
+        }
+
+        [Fact]
+        public void TestWithUniqueBundlesShouldHaveUniqueBundleIdsPasses()
+        {
+            string firstBundle = CreateSampleBundle(true).bundleId;
+            string secondBundle = CreateSampleBundle(false).bundleId;
+
+            Assert.NotEqual(firstBundle, secondBundle, StringComparer.Ordinal);
+        }
+
         [Fact]
         public void TestWithMultipleDuplicateEntriesFails()
         {