Singlefile compression of native files (#49855)
authorVladimir Sadov <vsadov@microsoft.com>
Mon, 29 Mar 2021 23:16:14 +0000 (16:16 -0700)
committerGitHub <noreply@github.com>
Mon, 29 Mar 2021 23:16:14 +0000 (23:16 +0000)
* versioning

* no need to support 2.0 bundle version

* enable compression

* fixes after rebase

* build fix for Unix

* Fix build with GCC

* disable a test temporarily to see what else breaks

* Add EnableCompression option to Bundler + PR feedback

* Couple more places should use version 6.0

* PR feedback (header versioning, more tests, fixed an assert)

* Suggestion from PR review

Co-authored-by: Vitek Karas <vitek.karas@microsoft.com>
* sorted usings

* should be bundle_major_version in two more places.

* More PR feedback

Co-authored-by: Vitek Karas <vitek.karas@microsoft.com>
20 files changed:
src/installer/managed/Microsoft.NET.HostModel/Bundle/BundleOptions.cs
src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs
src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs
src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs
src/installer/managed/Microsoft.NET.HostModel/Bundle/TargetInfo.cs
src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs
src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleTestBase.cs
src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundledAppWithSubDirs.cs
src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/NetCoreApp3CompatModeTests.cs
src/installer/tests/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/SingleFileApiTests.cs
src/installer/tests/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs
src/native/corehost/apphost/static/CMakeLists.txt
src/native/corehost/bundle/extractor.cpp
src/native/corehost/bundle/file_entry.cpp
src/native/corehost/bundle/file_entry.h
src/native/corehost/bundle/header.cpp
src/native/corehost/bundle/header.h
src/native/corehost/bundle/info.cpp
src/native/corehost/bundle/manifest.cpp
src/native/corehost/hostmisc/utils.h

index 646e1b0..2203038 100644 (file)
@@ -17,5 +17,6 @@ namespace Microsoft.NET.HostModel.Bundle
         BundleOtherFiles = 2,
         BundleSymbolFiles = 4,
         BundleAllContent = BundleNativeBinaries | BundleOtherFiles,
+        EnableCompression = 8,
     };
 }
index 0505d2d..bee21dd 100644 (file)
@@ -1,14 +1,15 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using Microsoft.NET.HostModel.AppHost;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
 using System.IO;
+using System.IO.Compression;
+using System.Linq;
 using System.Reflection.PortableExecutable;
 using System.Runtime.InteropServices;
+using Microsoft.NET.HostModel.AppHost;
 
 namespace Microsoft.NET.HostModel.Bundle
 {
@@ -18,6 +19,9 @@ namespace Microsoft.NET.HostModel.Bundle
     /// </summary>
     public class Bundler
     {
+        public const uint BundlerMajorVersion = 6;
+        public const uint BundlerMinorVersion = 0;
+
         private readonly string HostName;
         private readonly string OutputDir;
         private readonly string DepsJson;
@@ -44,22 +48,72 @@ namespace Microsoft.NET.HostModel.Bundle
             OutputDir = Path.GetFullPath(string.IsNullOrEmpty(outputDir) ? Environment.CurrentDirectory : outputDir);
             Target = new TargetInfo(targetOS, targetArch, targetFrameworkVersion);
 
+            if (Target.BundleMajorVersion < 6 &&
+                (options & BundleOptions.EnableCompression) != 0)
+            {
+                throw new ArgumentException("Compression requires framework version 6.0 or above", nameof(options));
+            }
+
             appAssemblyName ??= Target.GetAssemblyName(hostName);
             DepsJson = appAssemblyName + ".deps.json";
             RuntimeConfigJson = appAssemblyName + ".runtimeconfig.json";
             RuntimeConfigDevJson = appAssemblyName + ".runtimeconfig.dev.json";
 
-            BundleManifest = new Manifest(Target.BundleVersion, netcoreapp3CompatMode: options.HasFlag(BundleOptions.BundleAllContent));
+            BundleManifest = new Manifest(Target.BundleMajorVersion, netcoreapp3CompatMode: options.HasFlag(BundleOptions.BundleAllContent));
             Options = Target.DefaultOptions | options;
         }
 
+        private bool ShouldCompress(FileType type)
+        {
+            if (!Options.HasFlag(BundleOptions.EnableCompression))
+            {
+                return false;
+            }
+
+            switch (type)
+            {
+                case FileType.Symbols:
+                case FileType.NativeBinary:
+                    return true;
+
+                default:
+                    return false;
+            }
+        }
+
         /// <summary>
         /// Embed 'file' into 'bundle'
         /// </summary>
-        /// <returns>Returns the offset of the start 'file' within 'bundle'</returns>
-
-        private long AddToBundle(Stream bundle, Stream file, FileType type)
+        /// <returns>
+        /// startOffset: offset of the start 'file' within 'bundle'
+        /// compressedSize: size of the compressed data, if entry was compressed, otherwise 0
+        /// </returns>
+        private (long startOffset, long compressedSize) AddToBundle(Stream bundle, Stream file, FileType type)
         {
+            long startOffset = bundle.Position;
+            if (ShouldCompress(type))
+            {
+                long fileLength = file.Length;
+                file.Position = 0;
+
+                // We use DeflateStream here.
+                // It uses GZip algorithm, but with a trivial header that does not contain file info.
+                using (DeflateStream compressionStream = new DeflateStream(bundle, CompressionLevel.Optimal, leaveOpen: true))
+                {
+                    file.CopyTo(compressionStream);
+                }
+
+                long compressedSize = bundle.Position - startOffset;
+                if (compressedSize < fileLength * 0.75)
+                {
+                    return (startOffset, compressedSize);
+                }
+
+                // compression rate was not good enough
+                // roll back the bundle offset and let the uncompressed code path take care of the entry.
+                bundle.Seek(startOffset, SeekOrigin.Begin);
+            }
+
             if (type == FileType.Assembly)
             {
                 long misalignment = (bundle.Position % Target.AssemblyAlignment);
@@ -72,10 +126,10 @@ namespace Microsoft.NET.HostModel.Bundle
             }
 
             file.Position = 0;
-            long startOffset = bundle.Position;
+            startOffset = bundle.Position;
             file.CopyTo(bundle);
 
-            return startOffset;
+            return (startOffset, 0);
         }
 
         private bool IsHost(string fileRelativePath)
@@ -186,8 +240,8 @@ namespace Microsoft.NET.HostModel.Bundle
         /// </exceptions>
         public string GenerateBundle(IReadOnlyList<FileSpec> fileSpecs)
         {
-            Tracer.Log($"Bundler version: {Manifest.CurrentVersion}");
-            Tracer.Log($"Bundler Header: {BundleManifest.DesiredVersion}");
+            Tracer.Log($"Bundler Version: {BundlerMajorVersion}.{BundlerMinorVersion}");
+            Tracer.Log($"Bundle  Version: {BundleManifest.BundleVersion}");
             Tracer.Log($"Target Runtime: {Target}");
             Tracer.Log($"Bundler Options: {Options}");
 
@@ -254,8 +308,8 @@ namespace Microsoft.NET.HostModel.Bundle
                     using (FileStream file = File.OpenRead(fileSpec.SourcePath))
                     {
                         FileType targetType = Target.TargetSpecificFileType(type);
-                        long startOffset = AddToBundle(bundle, file, targetType);
-                        FileEntry entry = BundleManifest.AddEntry(targetType, relativePath, startOffset, file.Length);
+                        (long startOffset, long compressedSize) = AddToBundle(bundle, file, targetType);
+                        FileEntry entry = BundleManifest.AddEntry(targetType, relativePath, startOffset, file.Length, compressedSize, Target.BundleMajorVersion);
                         Tracer.Log($"Embed: {entry}");
                     }
                 }
index 7315c2a..e71a7fa 100644 (file)
@@ -15,32 +15,44 @@ namespace Microsoft.NET.HostModel.Bundle
     /// * Name       ("NameLength" Bytes)
     /// * Offset     (Int64)
     /// * Size       (Int64)
+    /// === present only in bundle version 3+
+    /// * CompressedSize   (Int64)  0 indicates No Compression
     /// </summary>
     public class FileEntry
     {
+        public readonly uint BundleMajorVersion;
+
         public readonly long Offset;
         public readonly long Size;
+        public readonly long CompressedSize;
         public readonly FileType Type;
         public readonly string RelativePath; // Path of an embedded file, relative to the Bundle source-directory.
 
         public const char DirectorySeparatorChar = '/';
 
-        public FileEntry(FileType fileType, string relativePath, long offset, long size)
+        public FileEntry(FileType fileType, string relativePath, long offset, long size, long compressedSize, uint bundleMajorVersion)
         {
+            BundleMajorVersion = bundleMajorVersion;
             Type = fileType;
             RelativePath = relativePath.Replace('\\', DirectorySeparatorChar);
             Offset = offset;
             Size = size;
+            CompressedSize = compressedSize;
         }
 
         public void Write(BinaryWriter writer)
         {
             writer.Write(Offset);
             writer.Write(Size);
+            // compression is used only in version 6.0+
+            if (BundleMajorVersion >= 6)
+            {
+                writer.Write(CompressedSize);
+            }
             writer.Write((byte)Type);
             writer.Write(RelativePath);
         }
 
-        public override string ToString() => $"{RelativePath} [{Type}] @{Offset} Sz={Size}";
+        public override string ToString() => $"{RelativePath} [{Type}] @{Offset} Sz={Size} CompressedSz={CompressedSize}";
     }
 }
index 0beddb2..73e7b3b 100644 (file)
@@ -66,32 +66,26 @@ namespace Microsoft.NET.HostModel.Bundle
         // with path-names so that the AppHost can use it in
         // extraction path.
         public readonly string BundleID;
-
-        public const uint CurrentMajorVersion = 2;
-        public readonly uint DesiredMajorVersion;
+        public readonly uint BundleMajorVersion;
         // The Minor version is currently unused, and is always zero
-        public const uint MinorVersion = 0;
-
-        public static string CurrentVersion => $"{CurrentMajorVersion}.{MinorVersion}";
-        public string DesiredVersion => $"{DesiredMajorVersion}.{MinorVersion}";
-
+        public const uint BundleMinorVersion = 0;
         private FileEntry DepsJsonEntry;
         private FileEntry RuntimeConfigJsonEntry;
         private HeaderFlags Flags;
-
         public List<FileEntry> Files;
+        public string BundleVersion => $"{BundleMajorVersion}.{BundleMinorVersion}";
 
-        public Manifest(uint desiredVersion, bool netcoreapp3CompatMode = false)
+        public Manifest(uint bundleMajorVersion, bool netcoreapp3CompatMode = false)
         {
-            DesiredMajorVersion = desiredVersion;
+            BundleMajorVersion = bundleMajorVersion;
             Files = new List<FileEntry>();
             BundleID = Path.GetRandomFileName();
-            Flags = (netcoreapp3CompatMode) ? HeaderFlags.NetcoreApp3CompatMode: HeaderFlags.None;
+            Flags = (netcoreapp3CompatMode) ? HeaderFlags.NetcoreApp3CompatMode : HeaderFlags.None;
         }
 
-        public FileEntry AddEntry(FileType type, string relativePath, long offset, long size)
+        public FileEntry AddEntry(FileType type, string relativePath, long offset, long size, long compressedSize, uint bundleMajorVersion)
         {
-            FileEntry entry = new FileEntry(type, relativePath, offset, size);
+            FileEntry entry = new FileEntry(type, relativePath, offset, size, compressedSize, bundleMajorVersion);
             Files.Add(entry);
 
             switch (entry.Type)
@@ -118,12 +112,12 @@ namespace Microsoft.NET.HostModel.Bundle
             long startOffset = writer.BaseStream.Position;
 
             // Write the bundle header
-            writer.Write(DesiredMajorVersion);
-            writer.Write(MinorVersion);
+            writer.Write(BundleMajorVersion);
+            writer.Write(BundleMinorVersion);
             writer.Write(Files.Count);
             writer.Write(BundleID);
 
-            if (DesiredMajorVersion == 2)
+            if (BundleMajorVersion >= 2)
             {
                 writer.Write((DepsJsonEntry != null) ? DepsJsonEntry.Offset : 0);
                 writer.Write((DepsJsonEntry != null) ? DepsJsonEntry.Size : 0);
index 05d5ff9..2d3f656 100644 (file)
@@ -25,7 +25,7 @@ namespace Microsoft.NET.HostModel.Bundle
         public readonly OSPlatform OS;
         public readonly Architecture Arch;
         public readonly Version FrameworkVersion;
-        public readonly uint BundleVersion;
+        public readonly uint BundleMajorVersion;
         public readonly BundleOptions DefaultOptions;
         public readonly int AssemblyAlignment;
 
@@ -33,18 +33,23 @@ namespace Microsoft.NET.HostModel.Bundle
         {
             OS = os ?? HostOS;
             Arch = arch ?? RuntimeInformation.OSArchitecture;
-            FrameworkVersion = targetFrameworkVersion ?? net50;
+            FrameworkVersion = targetFrameworkVersion ?? net60;
 
             Debug.Assert(IsLinux || IsOSX || IsWindows);
 
-            if (FrameworkVersion.CompareTo(net50) >= 0)
+            if (FrameworkVersion.CompareTo(net60) >= 0)
             {
-                BundleVersion = 2u;
+                BundleMajorVersion = 6u;
+                DefaultOptions = BundleOptions.None;
+            }
+            else if (FrameworkVersion.CompareTo(net50) >= 0)
+            {
+                BundleMajorVersion = 2u;
                 DefaultOptions = BundleOptions.None;
             }
             else if (FrameworkVersion.Major == 3 && (FrameworkVersion.Minor == 0 || FrameworkVersion.Minor == 1))
             {
-                BundleVersion = 1u;
+                BundleMajorVersion = 1u;
                 DefaultOptions = BundleOptions.BundleAllContent;
             }
             else
@@ -94,7 +99,7 @@ namespace Microsoft.NET.HostModel.Bundle
 
         // The .net core 3 apphost doesn't care about semantics of FileType -- all files are extracted at startup.
         // However, the apphost checks that the FileType value is within expected bounds, so set it to the first enumeration.
-        public FileType TargetSpecificFileType(FileType fileType) => (BundleVersion == 1) ? FileType.Unknown : fileType;
+        public FileType TargetSpecificFileType(FileType fileType) => (BundleMajorVersion == 1) ? FileType.Unknown : fileType;
 
         // In .net core 3.x, bundle processing happens within the AppHost.
         // Therefore HostFxr and HostPolicy can be bundled within the single-file app.
@@ -105,6 +110,7 @@ namespace Microsoft.NET.HostModel.Bundle
         public bool ShouldExclude(string relativePath) =>
             (FrameworkVersion.Major != 3) && (relativePath.Equals(HostFxr) || relativePath.Equals(HostPolicy));
 
+        private readonly Version net60 = new Version(6, 0);
         private readonly Version net50 = new Version(5, 0);
         private string HostFxr => IsWindows ? "hostfxr.dll" : IsLinux ? "libhostfxr.so" : "libhostfxr.dylib";
         private string HostPolicy => IsWindows ? "hostpolicy.dll" : IsLinux ? "libhostpolicy.so" : "libhostpolicy.dylib";
index 1741c95..33c8955 100644 (file)
@@ -31,8 +31,8 @@ namespace AppHost.Bundle.Tests
             var hostName = BundleHelper.GetHostName(fixture);
 
             // Publish the bundle
-            UseSingleFileSelfContainedHost(fixture);
-            Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, options: BundleOptions.BundleNativeBinaries);
+            BundleOptions options = BundleOptions.BundleNativeBinaries;
+            Bundler bundler = BundleSelfContainedApp(fixture, out string singleFile, options);
 
             // Verify expected files in the bundle directory
             var bundleDir = BundleHelper.GetBundleDir(fixture);
@@ -80,8 +80,7 @@ namespace AppHost.Bundle.Tests
                 return;
 
             var fixture = sharedTestState.TestFixture.Copy();
-            UseSingleFileSelfContainedHost(fixture);
-            var bundler = BundleHelper.BundleApp(fixture, out var singleFile, bundleOptions);
+            var bundler = BundleSelfContainedApp(fixture, out var singleFile, bundleOptions);
 
             // Run the bundled app (extract files to <path>)
             var cmd = Command.Create(singleFile);
@@ -110,8 +109,8 @@ namespace AppHost.Bundle.Tests
             var fixture = sharedTestState.TestFixture.Copy();
 
             // Publish the bundle
-            UseSingleFileSelfContainedHost(fixture);
-            Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, BundleOptions.BundleNativeBinaries);
+            BundleOptions options = BundleOptions.BundleNativeBinaries;
+            Bundler bundler = BundleSelfContainedApp(fixture, out string singleFile, options);
 
             // Create a directory for extraction.
             var extractBaseDir = BundleHelper.GetExtractionRootDir(fixture);
@@ -160,13 +159,12 @@ namespace AppHost.Bundle.Tests
             var appName = Path.GetFileNameWithoutExtension(hostName);
 
             // Publish the bundle
-            UseSingleFileSelfContainedHost(fixture);
-            Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, BundleOptions.BundleNativeBinaries);
+            BundleOptions options = BundleOptions.BundleNativeBinaries;
+            Bundler bundler = BundleSelfContainedApp(fixture, out string singleFile, options);
 
             // Create a directory for extraction.
             var extractBaseDir = BundleHelper.GetExtractionRootDir(fixture);
 
-
             // Run the bunded app for the first time, and extract files to 
             // $DOTNET_BUNDLE_EXTRACT_BASE_DIR/<app>/bundle-id
             Command.Create(singleFile)
index c71aaa3..18aba06 100644 (file)
@@ -44,10 +44,28 @@ namespace AppHost.Bundle.Tests
         public static string BundleSelfContainedApp(
             TestProjectFixture testFixture,
             BundleOptions options = BundleOptions.None,
-            Version targetFrameworkVersion = null)
+            Version targetFrameworkVersion = null,
+            bool disableCompression = false)
+        {
+            string singleFile;
+            BundleSelfContainedApp(testFixture, out singleFile, options, targetFrameworkVersion);
+            return singleFile;
+        }
+
+        public static Bundler BundleSelfContainedApp(
+            TestProjectFixture testFixture,
+            out string singleFile,
+            BundleOptions options = BundleOptions.None,
+            Version targetFrameworkVersion = null,
+            bool disableCompression = false)
         {
             UseSingleFileSelfContainedHost(testFixture);
-            return BundleHelper.BundleApp(testFixture, options, targetFrameworkVersion);
+            if (targetFrameworkVersion == null || targetFrameworkVersion >= new Version(6, 0))
+            {
+                options |= BundleOptions.EnableCompression;
+            }
+
+            return BundleHelper.BundleApp(testFixture, out singleFile, options, targetFrameworkVersion);
         }
 
         public abstract class SharedTestStateBase
index 2a0dc9a..9ede20c 100644 (file)
@@ -59,7 +59,54 @@ namespace AppHost.Bundle.Tests
         public void Bundled_Self_Contained_App_Run_Succeeds(BundleOptions options)
         {
             var fixture = sharedTestState.TestSelfContainedFixture.Copy();
-            var singleFile = BundleHelper.BundleApp(fixture, options);
+            var singleFile = BundleSelfContainedApp(fixture, options);
+
+            // Run the bundled app (extract files)
+            RunTheApp(singleFile, fixture);
+
+            // Run the bundled app again (reuse extracted files)
+            RunTheApp(singleFile, fixture);
+        }
+
+        [InlineData(BundleOptions.None)]
+        [InlineData(BundleOptions.BundleNativeBinaries)]
+        [InlineData(BundleOptions.BundleAllContent)]
+        [Theory]
+        public void Bundled_Self_Contained_NoCompression_App_Run_Succeeds(BundleOptions options)
+        {
+            var fixture = sharedTestState.TestSelfContainedFixture.Copy();
+            var singleFile = BundleSelfContainedApp(fixture, options, disableCompression: true);
+
+            // Run the bundled app (extract files)
+            RunTheApp(singleFile, fixture);
+
+            // Run the bundled app again (reuse extracted files)
+            RunTheApp(singleFile, fixture);
+        }
+
+        [InlineData(BundleOptions.None)]
+        [InlineData(BundleOptions.BundleNativeBinaries)]
+        [InlineData(BundleOptions.BundleAllContent)]
+        [Theory]
+        public void Bundled_Self_Contained_Targeting50_App_Run_Succeeds(BundleOptions options)
+        {
+            var fixture = sharedTestState.TestSelfContainedFixture.Copy();
+            var singleFile = BundleSelfContainedApp(fixture, options, new Version(5, 0));
+
+            // Run the bundled app (extract files)
+            RunTheApp(singleFile, fixture);
+
+            // Run the bundled app again (reuse extracted files)
+            RunTheApp(singleFile, fixture);
+        }
+
+        [InlineData(BundleOptions.BundleAllContent)]
+        [Theory]
+        public void Bundled_Framework_dependent_Targeting50_App_Run_Succeeds(BundleOptions options)
+        {
+            var fixture = sharedTestState.TestSelfContainedFixture.Copy();
+            UseFrameworkDependentHost(fixture);
+            var singleFile = BundleHelper.BundleApp(fixture, options, new Version(5, 0));
 
             // Run the bundled app (extract files)
             RunTheApp(singleFile, fixture);
@@ -68,6 +115,17 @@ namespace AppHost.Bundle.Tests
             RunTheApp(singleFile, fixture);
         }
 
+        [Fact]
+        public void Bundled_Self_Contained_Targeting50_WithCompression_Throws()
+        {
+            var fixture = sharedTestState.TestSelfContainedFixture.Copy();
+            UseSingleFileSelfContainedHost(fixture);
+            // compression must be off when targeting 5.0
+            var options = BundleOptions.EnableCompression;
+
+            Assert.Throws<ArgumentException>(()=>BundleHelper.BundleApp(fixture, options, new Version(5, 0)));
+        }
+
         [InlineData(BundleOptions.None)]
         [InlineData(BundleOptions.BundleNativeBinaries)]
         [InlineData(BundleOptions.BundleAllContent)]
@@ -75,7 +133,7 @@ namespace AppHost.Bundle.Tests
         public void Bundled_With_Empty_File_Succeeds(BundleOptions options)
         {
             var fixture = sharedTestState.TestAppWithEmptyFileFixture.Copy();
-            var singleFile = BundleHelper.BundleApp(fixture, options);
+            var singleFile = BundleSelfContainedApp(fixture, options);
 
             // Run the app
             RunTheApp(singleFile, fixture);
index 95e1061..cb8bdf9 100644 (file)
@@ -26,8 +26,8 @@ namespace AppHost.Bundle.Tests
         public void Bundle_Is_Extracted()
         {
             var fixture = sharedTestState.TestFixture.Copy();
-            UseSingleFileSelfContainedHost(fixture);
-            Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, BundleOptions.BundleAllContent);
+            BundleOptions options = BundleOptions.BundleAllContent;
+            Bundler bundler = BundleSelfContainedApp(fixture, out string singleFile, options);
             var extractionBaseDir = BundleHelper.GetExtractionRootDir(fixture);
 
             Command.Create(singleFile, "executing_assembly_location trusted_platform_assemblies assembly_location System.Console")
index 60425b9..0efa9a7 100644 (file)
@@ -91,7 +91,7 @@ namespace AppHost.Bundle.Tests
         public void AppContext_Native_Search_Dirs_Contains_Bundle_Dir()
         {
             var fixture = sharedTestState.TestFixture.Copy();
-            Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile);
+            Bundler bundler = BundleSelfContainedApp(fixture, out string singleFile);
             string extractionDir = BundleHelper.GetExtractionDir(fixture, bundler).Name;
             string bundleDir = BundleHelper.GetBundleDir(fixture).FullName;
 
@@ -110,7 +110,7 @@ namespace AppHost.Bundle.Tests
         public void AppContext_Native_Search_Dirs_Contains_Bundle_And_Extraction_Dirs()
         {
             var fixture = sharedTestState.TestFixture.Copy();
-            Bundler bundler = BundleHelper.BundleApp(fixture, out string singleFile, BundleOptions.BundleNativeBinaries);
+            Bundler bundler = BundleSelfContainedApp(fixture, out string singleFile, BundleOptions.BundleNativeBinaries);
             string extractionDir = BundleHelper.GetExtractionDir(fixture, bundler).Name;
             string bundleDir = BundleHelper.GetBundleDir(fixture).FullName;
 
index 6a102dd..ecd00f7 100644 (file)
@@ -60,7 +60,7 @@ namespace BundleTests.Helpers
 
         public static string[] GetExtractedFiles(TestProjectFixture fixture, BundleOptions bundleOptions)
         {
-            switch (bundleOptions)
+            switch (bundleOptions & ~BundleOptions.EnableCompression)
             {
                 case BundleOptions.None:
                 case BundleOptions.BundleOtherFiles:
index c922205..d7e129c 100644 (file)
@@ -19,6 +19,10 @@ set(SKIP_VERSIONING 1)
 include_directories(..)
 include_directories(../../json)
 
+if(NOT CLR_CMAKE_TARGET_WIN32)
+    include_directories(../../../../libraries/Native/Unix/Common)
+endif()
+
 set(SOURCES
     ../bundle_marker.cpp
     ./hostfxr_resolver.cpp
index be3f207..e33daf8 100644 (file)
@@ -7,6 +7,16 @@
 #include "pal.h"
 #include "utils.h"
 
+#if defined(NATIVE_LIBS_EMBEDDED)
+extern "C"
+{
+#include "../../../libraries/Native/AnyOS/zlib/pal_zlib.h"
+}
+#endif
+
+// Suppress prefast warning #6255: alloca indicates failure by raising a stack overflow exception
+#pragma warning(disable:6255)
+
 using namespace bundle;
 
 pal::string_t& extractor_t::extraction_dir()
@@ -105,9 +115,66 @@ void extractor_t::extract(const file_entry_t &entry, reader_t &reader)
     reader.set_offset(entry.offset());
     int64_t size = entry.size();
     size_t cast_size = to_size_t_dbgchecked(size);
-    if (fwrite(reader, 1, cast_size, file) != cast_size)
+    size_t extracted_size = 0;
+
+    if (entry.compressedSize() != 0)
+    {
+#if defined(NATIVE_LIBS_EMBEDDED)
+        PAL_ZStream zStream;
+        zStream.nextIn = (uint8_t*)(const void*)reader;
+        zStream.availIn = entry.compressedSize();
+
+        const int Deflate_DefaultWindowBits = -15; // Legal values are 8..15 and -8..-15. 15 is the window size,
+                                                   // negative val causes deflate to produce raw deflate data (no zlib header).
+
+        int ret = CompressionNative_InflateInit2_(&zStream, Deflate_DefaultWindowBits);
+        if (ret != PAL_Z_OK)
+        {
+            trace::error(_X("Failure initializing zLib stream."));
+            throw StatusCode::BundleExtractionIOError;
+        }
+
+        const int bufSize = 4096;
+        uint8_t* buf = (uint8_t*)alloca(bufSize);
+
+        do
+        {
+            zStream.nextOut = buf;
+            zStream.availOut = bufSize;
+
+            ret = CompressionNative_Inflate(&zStream, PAL_Z_NOFLUSH);
+            if (ret < 0)
+            {
+                CompressionNative_InflateEnd(&zStream);
+                trace::error(_X("Failure inflating zLib stream. %s"), zStream.msg);
+                throw StatusCode::BundleExtractionIOError;
+            }
+
+            int produced = bufSize - zStream.availOut;
+            if (fwrite(buf, 1, produced, file) != (size_t)produced)
+            {
+                CompressionNative_InflateEnd(&zStream);
+                trace::error(_X("I/O failure when writing decompressed file."));
+                throw StatusCode::BundleExtractionIOError;
+            }
+
+            extracted_size += produced;
+        } while (zStream.availOut == 0);
+
+        CompressionNative_InflateEnd(&zStream);
+#else
+        trace::error(_X("Failure extracting contents of the application bundle. Compressed files used with a standalone (not singlefile) apphost."));
+        throw StatusCode::BundleExtractionIOError;
+#endif
+    }
+    else
+    {
+        extracted_size = fwrite(reader, 1, cast_size, file);
+    }
+
+    if (extracted_size != cast_size)
     {
-        trace::error(_X("Failure extracting contents of the application bundle."));
+        trace::error(_X("Failure extracting contents of the application bundle. Expected size:%d Actual size:%d"), size, extracted_size);
         trace::error(_X("I/O failure when writing extracted files."));
         throw StatusCode::BundleExtractionIOError;
     }
index e33b2cf..ace0ef5 100644 (file)
@@ -10,15 +10,26 @@ using namespace bundle;
 
 bool file_entry_t::is_valid() const
 {
-    return m_offset > 0 && m_size >= 0 &&
+    return m_offset > 0 && m_size >= 0 && m_compressedSize >= 0 &&
         static_cast<file_type_t>(m_type) < file_type_t::__last;
 }
 
-file_entry_t file_entry_t::read(reader_t &reader, bool force_extraction)
+file_entry_t file_entry_t::read(reader_t &reader, uint32_t bundle_major_version, bool force_extraction)
 {
     // First read the fixed-sized portion of file-entry
-    const file_entry_fixed_t* fixed_data = reinterpret_cast<const file_entry_fixed_t*>(reader.read_direct(sizeof(file_entry_fixed_t)));
-    file_entry_t entry(fixed_data, force_extraction);
+    file_entry_fixed_t fixed_data;
+
+    fixed_data.offset = *(int64_t*)reader.read_direct(sizeof(int64_t));
+    fixed_data.size   = *(int64_t*)reader.read_direct(sizeof(int64_t));
+
+    // compressedSize is present only in v6+ headers
+    fixed_data.compressedSize = bundle_major_version >= 6 ?
+                        *(int64_t*)reader.read_direct(sizeof(int64_t)) :
+                        0;
+
+    fixed_data.type   = *(file_type_t*)reader.read_direct(sizeof(file_type_t));
+
+    file_entry_t entry(&fixed_data, force_extraction);
 
     if (!entry.is_valid())
     {
index eb122ef..225cf22 100644 (file)
@@ -16,6 +16,7 @@ namespace bundle
     // Fixed size portion (file_entry_fixed_t)
     //   - Offset     
     //   - Size       
+    //   - CompressedSize  - only in bundleVersion 6+
     //   - File Entry Type       
     // Variable Size portion
     //   - relative path (7-bit extension encoded length prefixed string)
@@ -25,6 +26,7 @@ namespace bundle
     {
         int64_t offset;
         int64_t size;
+        int64_t compressedSize;
         file_type_t type;
     };
 #pragma pack(pop)
@@ -56,23 +58,26 @@ namespace bundle
 
             m_offset = fixed_data->offset;
             m_size = fixed_data->size;
+            m_compressedSize = fixed_data->compressedSize;
             m_type = fixed_data->type;
         }
 
         const pal::string_t relative_path() const { return m_relative_path; }
         int64_t offset() const { return m_offset; }
         int64_t size() const { return m_size; }
+        int64_t compressedSize() const { return m_compressedSize; }
         file_type_t type() const { return m_type; }
         void disable() { m_disabled = true; }
         bool is_disabled() const { return m_disabled; }
         bool needs_extraction() const;
         bool matches(const pal::string_t& path) const { return (pal::pathcmp(relative_path(), path) == 0) && !is_disabled(); }
 
-        static file_entry_t read(reader_t &reader, bool force_extraction);
+        static file_entry_t read(reader_t &reader, uint32_t bundle_major_version, bool force_extraction);
 
     private:
         int64_t m_offset;
         int64_t m_size;
+        int64_t m_compressedSize;
         file_type_t m_type;
         pal::string_t m_relative_path; // Path of an embedded file, relative to the extraction directory.
         // If the file represented by this entry is also found in a servicing location, the servicing location must take precedence.
index 05aa736..268ef86 100644 (file)
@@ -15,9 +15,11 @@ bool header_fixed_t::is_valid() const
         return false;
     }
 
+    // .net 6 host expects the version information to be 6.0
     // .net 5 host expects the version information to be 2.0
     // .net core 3 single-file bundles are handled within the netcoreapp3.x apphost, and are not processed here in the framework.
-    return (major_version == header_t::major_version) && (minor_version == header_t::minor_version);
+    return ((major_version == 6) && (minor_version == 0)) ||
+           ((major_version == 2) && (minor_version == 0));
 }
 
 header_t header_t::read(reader_t& reader)
@@ -32,7 +34,7 @@ header_t header_t::read(reader_t& reader)
         throw StatusCode::BundleExtractionFailure;
     }
 
-    header_t header(fixed_header->num_embedded_files);
+    header_t header(fixed_header->major_version, fixed_header->minor_version, fixed_header->num_embedded_files);
 
     // bundle_id is a component of the extraction path
     reader.read_path_string(header.m_bundle_id);
index f785203..1bc9241 100644 (file)
@@ -34,7 +34,7 @@ namespace bundle
         bool is_valid() const;
     };
 
-    // netcoreapp3_compat_mode flag is set on a .net5 app, which chooses to build single-file apps in .netcore3.x compat mode,
+    // netcoreapp3_compat_mode flag is set on a .net5+ app, which chooses to build single-file apps in .netcore3.x compat mode,
     // This indicates that:
     //   All published files are bundled into the app; some of them will be extracted to disk.
     //   AppContext.BaseDirectory is set to the extraction directory (and not the AppHost directory).
@@ -72,8 +72,10 @@ namespace bundle
     struct header_t
     {
     public:
-        header_t(int32_t num_embedded_files = 0)
+        header_t(uint32_t major_version, uint32_t minor_version, int32_t num_embedded_files)
             : m_num_embedded_files(num_embedded_files)
+            , m_major_version(major_version)
+            , m_minor_version(minor_version)
             , m_bundle_id()
             , m_v2_header()
         {
@@ -87,11 +89,13 @@ namespace bundle
         const location_t& runtimeconfig_json_location() const { return m_v2_header.runtimeconfig_json_location; }
         bool is_netcoreapp3_compat_mode() const { return m_v2_header.is_netcoreapp3_compat_mode(); }
 
-        static const uint32_t major_version = 2;
-        static const uint32_t minor_version = 0;
+        const uint32_t major_version() const { return m_major_version; };
+        const uint32_t minor_version() const { return m_minor_version; };
 
     private:
         int32_t m_num_embedded_files;
+        uint32_t m_major_version;
+        uint32_t m_minor_version;
         pal::string_t m_bundle_id;
         header_fixed_v2_t m_v2_header;
     };
index afb1ee1..5f9f0da 100644 (file)
@@ -11,11 +11,12 @@ using namespace bundle;
 const info_t* info_t::the_app = nullptr;
 
 info_t::info_t(const pal::char_t* bundle_path,
-               const pal::char_t* app_path,
-               int64_t header_offset)
+    const pal::char_t* app_path,
+    int64_t header_offset)
     : m_bundle_path(bundle_path)
     , m_bundle_size(0)
     , m_header_offset(header_offset)
+    , m_header(0, 0, 0)
 {
     m_base_path = get_directory(m_bundle_path);
 
index cf43995..ed7ced6 100644 (file)
@@ -11,7 +11,7 @@ manifest_t manifest_t::read(reader_t& reader, const header_t& header)
 
     for (int32_t i = 0; i < header.num_embedded_files(); i++)
     {
-        file_entry_t entry = file_entry_t::read(reader, header.is_netcoreapp3_compat_mode());
+        file_entry_t entry = file_entry_t::read(reader, header.major_version(), header.is_netcoreapp3_compat_mode());
         manifest.files.push_back(std::move(entry));
         manifest.m_files_need_extraction |= entry.needs_extraction();
     }
index 932cb80..d0dc381 100644 (file)
@@ -120,8 +120,9 @@ template<typename T>
 size_t to_size_t_dbgchecked(T value)
 {
     assert(value >= 0);
-    assert(value < static_cast<T>(std::numeric_limits<size_t>::max()));
-    return static_cast<size_t>(value);
+    size_t result = static_cast<size_t>(value);
+    assert(static_cast<T>(result) == value);
+    return result;
 }
 
 #endif