From dffaad78314044072068d8e253a51ef28e66b14a Mon Sep 17 00:00:00 2001 From: Swaroop Sridhar Date: Thu, 11 Apr 2019 21:17:18 -0700 Subject: [PATCH] Build the Bundler as a library (dotnet/core-setup#5798) * Build the Bundler as a library Currently, the single-file bundler is * Built as an app instead of a library * Invoked from the SDK out-of-proc in order to enforce strict API boundaries However, since the SDK implementation progressed, we've had to revisit the above decisions. First, the bundler currently is implemented to take a folder vs a list of files to bundle. * This causes unnecessary extra copies to the publish directory. * The files to be bundled need not be first copied to the publish directory. * Instead they can be written directly to the bundle. * Stale items remaining in the publish directory may be written into the bundle unnecessarily. Therefore, we decided that the bundler should take tuples of (source file-path, relative-path within bundle) instead of the publish folder as input. * While it is possible to provide this input on the command-line/response file, it is convenient and more efficient to do it in-proc. * Further, the SDK team agrees that the bundler is simple enough to run in-proc with the build system. Therefore, the changes proposed are: * Bundler will be built into a library Microsoft.NET.HostModel in core-setup repo (similar to DependencyModel) * It is not a task directly because we don't have to worry about multi-targeting * Other apphost related services like stamping the app.dll name into the apphost will be moded into this library * A task in the SDK repo will consume this library through package reference (from myget) * (WIP) SDK changes: https://github.com/dotnet/sdk/compare/master...swaroop-sridhar:bundle-lib?expand=1 Commit migrated from https://github.com/dotnet/core-setup/commit/345061995531c6f8de00189842a27298d5d1a9b1 --- Microsoft.DotNet.CoreSetup.sln | 74 ++++---- .../Microsoft.NET.Build.Bundle.csproj | 21 --- .../managed/Microsoft.NET.Build.Bundle/Program.cs | 206 --------------------- .../managed/Microsoft.NET.Build.Bundle/README.md | 23 --- .../managed/Microsoft.NET.Build.Bundle/Sdk.props | 18 -- .../Bundle}/BundleException.cs | 2 +- .../Bundle}/Bundler.cs | 168 +++++++++-------- .../Bundle}/Extractor.cs | 31 +++- .../Bundle}/FileEntry.cs | 4 +- .../Microsoft.NET.HostModel/Bundle/FileSpec.cs | 28 +++ .../Bundle}/FileType.cs | 5 +- .../Bundle}/Manifest.cs | 8 +- .../Microsoft.NET.HostModel/Bundle/README.md | 12 ++ .../Microsoft.NET.HostModel/Bundle/Trace.cs | 35 ++++ .../Microsoft.NET.HostModel.csproj | 18 ++ src/installer/managed/dir.proj | 2 +- src/installer/pkg/packaging/dir.proj | 2 +- .../BundleAndExtract.cs | 32 +--- .../Microsoft.NET.HostModel.Tests.csproj} | 8 +- src/installer/test/dir.props | 2 +- 20 files changed, 262 insertions(+), 437 deletions(-) delete mode 100644 src/installer/managed/Microsoft.NET.Build.Bundle/Microsoft.NET.Build.Bundle.csproj delete mode 100644 src/installer/managed/Microsoft.NET.Build.Bundle/Program.cs delete mode 100644 src/installer/managed/Microsoft.NET.Build.Bundle/README.md delete mode 100644 src/installer/managed/Microsoft.NET.Build.Bundle/Sdk.props rename src/installer/managed/{Microsoft.NET.Build.Bundle => Microsoft.NET.HostModel/Bundle}/BundleException.cs (92%) rename src/installer/managed/{Microsoft.NET.Build.Bundle => Microsoft.NET.HostModel/Bundle}/Bundler.cs (51%) rename src/installer/managed/{Microsoft.NET.Build.Bundle => Microsoft.NET.HostModel/Bundle}/Extractor.cs (68%) rename src/installer/managed/{Microsoft.NET.Build.Bundle => Microsoft.NET.HostModel/Bundle}/FileEntry.cs (95%) create mode 100644 src/installer/managed/Microsoft.NET.HostModel/Bundle/FileSpec.cs rename src/installer/managed/{Microsoft.NET.Build.Bundle => Microsoft.NET.HostModel/Bundle}/FileType.cs (87%) rename src/installer/managed/{Microsoft.NET.Build.Bundle => Microsoft.NET.HostModel/Bundle}/Manifest.cs (94%) create mode 100644 src/installer/managed/Microsoft.NET.HostModel/Bundle/README.md create mode 100644 src/installer/managed/Microsoft.NET.HostModel/Bundle/Trace.cs create mode 100644 src/installer/managed/Microsoft.NET.HostModel/Microsoft.NET.HostModel.csproj rename src/installer/test/{Microsoft.NET.Build.Bundle.Tests => Microsoft.NET.HostModel.Tests}/BundleAndExtract.cs (79%) rename src/installer/test/{Microsoft.NET.Build.Bundle.Tests/Microsoft.NET.Build.Bundle.Tests.csproj => Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Tests.csproj} (64%) diff --git a/Microsoft.DotNet.CoreSetup.sln b/Microsoft.DotNet.CoreSetup.sln index b00872d..c3a149b 100644 --- a/Microsoft.DotNet.CoreSetup.sln +++ b/Microsoft.DotNet.CoreSetup.sln @@ -1,6 +1,6 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 +# Visual Studio Version 15 VisualStudioVersion = 15.0.27527.1 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{5A29E8E3-A0FC-4C57-81DD-297B56D1A119}" @@ -23,9 +23,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Extensions.Depend EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestUtils", "src\test\TestUtils\TestUtils.csproj", "{D6676666-D14D-4DFA-88FB-76E3E823E2E1}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.NET.Build.Bundle", "src\managed\Microsoft.NET.Build.Bundle\Microsoft.NET.Build.Bundle.csproj", "{37B7E731-BEDE-45BF-AEC7-457333F23E53}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.NET.HostModel", "src\managed\Microsoft.NET.HostModel\Microsoft.NET.HostModel.csproj", "{325FB7F2-2E2E-422D-ADAA-F0B63E84CF24}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.NET.Build.Bundle.Tests", "src\test\Microsoft.NET.Build.Bundle.Tests\Microsoft.NET.Build.Bundle.Tests.csproj", "{8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.NET.HostModel.Tests", "src\test\Microsoft.NET.HostModel.Tests\Microsoft.NET.HostModel.Tests.csproj", "{3D07933E-8A4B-4C9A-92FD-473B5C4F71E2}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -119,38 +119,38 @@ Global {D6676666-D14D-4DFA-88FB-76E3E823E2E1}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU {D6676666-D14D-4DFA-88FB-76E3E823E2E1}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU {D6676666-D14D-4DFA-88FB-76E3E823E2E1}.RelWithDebInfo|x64.Build.0 = Release|Any CPU - {37B7E731-BEDE-45BF-AEC7-457333F23E53}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {37B7E731-BEDE-45BF-AEC7-457333F23E53}.Debug|Any CPU.Build.0 = Debug|Any CPU - {37B7E731-BEDE-45BF-AEC7-457333F23E53}.Debug|x64.ActiveCfg = Debug|Any CPU - {37B7E731-BEDE-45BF-AEC7-457333F23E53}.Debug|x64.Build.0 = Debug|Any CPU - {37B7E731-BEDE-45BF-AEC7-457333F23E53}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU - {37B7E731-BEDE-45BF-AEC7-457333F23E53}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU - {37B7E731-BEDE-45BF-AEC7-457333F23E53}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU - {37B7E731-BEDE-45BF-AEC7-457333F23E53}.MinSizeRel|x64.Build.0 = Debug|Any CPU - {37B7E731-BEDE-45BF-AEC7-457333F23E53}.Release|Any CPU.ActiveCfg = Release|Any CPU - {37B7E731-BEDE-45BF-AEC7-457333F23E53}.Release|Any CPU.Build.0 = Release|Any CPU - {37B7E731-BEDE-45BF-AEC7-457333F23E53}.Release|x64.ActiveCfg = Release|Any CPU - {37B7E731-BEDE-45BF-AEC7-457333F23E53}.Release|x64.Build.0 = Release|Any CPU - {37B7E731-BEDE-45BF-AEC7-457333F23E53}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU - {37B7E731-BEDE-45BF-AEC7-457333F23E53}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU - {37B7E731-BEDE-45BF-AEC7-457333F23E53}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU - {37B7E731-BEDE-45BF-AEC7-457333F23E53}.RelWithDebInfo|x64.Build.0 = Release|Any CPU - {8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF}.Debug|Any CPU.Build.0 = Debug|Any CPU - {8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF}.Debug|x64.ActiveCfg = Debug|Any CPU - {8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF}.Debug|x64.Build.0 = Debug|Any CPU - {8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU - {8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU - {8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU - {8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF}.MinSizeRel|x64.Build.0 = Debug|Any CPU - {8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF}.Release|Any CPU.ActiveCfg = Release|Any CPU - {8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF}.Release|Any CPU.Build.0 = Release|Any CPU - {8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF}.Release|x64.ActiveCfg = Release|Any CPU - {8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF}.Release|x64.Build.0 = Release|Any CPU - {8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU - {8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU - {8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU - {8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF}.RelWithDebInfo|x64.Build.0 = Release|Any CPU + {325FB7F2-2E2E-422D-ADAA-F0B63E84CF24}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {325FB7F2-2E2E-422D-ADAA-F0B63E84CF24}.Debug|Any CPU.Build.0 = Debug|Any CPU + {325FB7F2-2E2E-422D-ADAA-F0B63E84CF24}.Debug|x64.ActiveCfg = Debug|Any CPU + {325FB7F2-2E2E-422D-ADAA-F0B63E84CF24}.Debug|x64.Build.0 = Debug|Any CPU + {325FB7F2-2E2E-422D-ADAA-F0B63E84CF24}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU + {325FB7F2-2E2E-422D-ADAA-F0B63E84CF24}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU + {325FB7F2-2E2E-422D-ADAA-F0B63E84CF24}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU + {325FB7F2-2E2E-422D-ADAA-F0B63E84CF24}.MinSizeRel|x64.Build.0 = Debug|Any CPU + {325FB7F2-2E2E-422D-ADAA-F0B63E84CF24}.Release|Any CPU.ActiveCfg = Release|Any CPU + {325FB7F2-2E2E-422D-ADAA-F0B63E84CF24}.Release|Any CPU.Build.0 = Release|Any CPU + {325FB7F2-2E2E-422D-ADAA-F0B63E84CF24}.Release|x64.ActiveCfg = Release|Any CPU + {325FB7F2-2E2E-422D-ADAA-F0B63E84CF24}.Release|x64.Build.0 = Release|Any CPU + {325FB7F2-2E2E-422D-ADAA-F0B63E84CF24}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU + {325FB7F2-2E2E-422D-ADAA-F0B63E84CF24}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU + {325FB7F2-2E2E-422D-ADAA-F0B63E84CF24}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU + {325FB7F2-2E2E-422D-ADAA-F0B63E84CF24}.RelWithDebInfo|x64.Build.0 = Release|Any CPU + {3D07933E-8A4B-4C9A-92FD-473B5C4F71E2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3D07933E-8A4B-4C9A-92FD-473B5C4F71E2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3D07933E-8A4B-4C9A-92FD-473B5C4F71E2}.Debug|x64.ActiveCfg = Debug|Any CPU + {3D07933E-8A4B-4C9A-92FD-473B5C4F71E2}.Debug|x64.Build.0 = Debug|Any CPU + {3D07933E-8A4B-4C9A-92FD-473B5C4F71E2}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU + {3D07933E-8A4B-4C9A-92FD-473B5C4F71E2}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU + {3D07933E-8A4B-4C9A-92FD-473B5C4F71E2}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU + {3D07933E-8A4B-4C9A-92FD-473B5C4F71E2}.MinSizeRel|x64.Build.0 = Debug|Any CPU + {3D07933E-8A4B-4C9A-92FD-473B5C4F71E2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3D07933E-8A4B-4C9A-92FD-473B5C4F71E2}.Release|Any CPU.Build.0 = Release|Any CPU + {3D07933E-8A4B-4C9A-92FD-473B5C4F71E2}.Release|x64.ActiveCfg = Release|Any CPU + {3D07933E-8A4B-4C9A-92FD-473B5C4F71E2}.Release|x64.Build.0 = Release|Any CPU + {3D07933E-8A4B-4C9A-92FD-473B5C4F71E2}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU + {3D07933E-8A4B-4C9A-92FD-473B5C4F71E2}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU + {3D07933E-8A4B-4C9A-92FD-473B5C4F71E2}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU + {3D07933E-8A4B-4C9A-92FD-473B5C4F71E2}.RelWithDebInfo|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -161,8 +161,8 @@ Global {23F4AB97-D15C-4C51-A641-DF5C5D5EF70F} = {5CE8410C-3100-4F41-8FA9-E6B4132D9703} {D86A859D-E6FA-4E73-A255-5776FC473A25} = {5CE8410C-3100-4F41-8FA9-E6B4132D9703} {D6676666-D14D-4DFA-88FB-76E3E823E2E1} = {5CE8410C-3100-4F41-8FA9-E6B4132D9703} - {37B7E731-BEDE-45BF-AEC7-457333F23E53} = {FAA448DA-7D1C-4481-915D-5765BF906332} - {8E21F355-54D0-46C3-ACC6-B1BC5D11FDCF} = {5CE8410C-3100-4F41-8FA9-E6B4132D9703} + {325FB7F2-2E2E-422D-ADAA-F0B63E84CF24} = {FAA448DA-7D1C-4481-915D-5765BF906332} + {3D07933E-8A4B-4C9A-92FD-473B5C4F71E2} = {5CE8410C-3100-4F41-8FA9-E6B4132D9703} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {28B9726D-802B-478D-AF7A-B9243B9E180B} diff --git a/src/installer/managed/Microsoft.NET.Build.Bundle/Microsoft.NET.Build.Bundle.csproj b/src/installer/managed/Microsoft.NET.Build.Bundle/Microsoft.NET.Build.Bundle.csproj deleted file mode 100644 index 9cca0de..0000000 --- a/src/installer/managed/Microsoft.NET.Build.Bundle/Microsoft.NET.Build.Bundle.csproj +++ /dev/null @@ -1,21 +0,0 @@ - - - - Exe - netcoreapp2.0 - .Net Core Single File Bundler - true - - - - - - - true - Sdk - - - - - - diff --git a/src/installer/managed/Microsoft.NET.Build.Bundle/Program.cs b/src/installer/managed/Microsoft.NET.Build.Bundle/Program.cs deleted file mode 100644 index f56b0e3..0000000 --- a/src/installer/managed/Microsoft.NET.Build.Bundle/Program.cs +++ /dev/null @@ -1,206 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; - -namespace Microsoft.NET.Build.Bundle -{ - /// - /// The main driver for Bundle and Extract operations. - /// - public static class Program - { - enum RunMode - { - Help, - Bundle, - Extract - }; - - static RunMode Mode = RunMode.Bundle; - - // Common Options: - static bool Verbose = false; - static string OutputDir; - - // Bundle options: - static bool EmbedPDBs = false; - static string HostName; - static string SourceDir; - - // Extract options: - static string BundleToExtract; - - static void Usage() - { - Console.WriteLine($".NET Core Bundler (version {Bundler.Version})"); - Console.WriteLine("Usage: bundle "); - Console.WriteLine(""); - Console.WriteLine("Bundle options:"); - Console.WriteLine(" --source Directory containing files to bundle (required)."); - Console.WriteLine(" --apphost Application host within source directory (required)."); - Console.WriteLine(" --pdb Embed PDB files."); - Console.WriteLine(""); - Console.WriteLine("Extract options:"); - Console.WriteLine(" --extract Extract files from the specified bundle."); - Console.WriteLine(""); - Console.WriteLine("Common options:"); - Console.WriteLine(" -o|--output Output directory (default: current)."); - Console.WriteLine(" -d|--diagnostics Enable diagnostic output."); - Console.WriteLine(" -?|-h|--help Display usage information."); - Console.WriteLine(""); - Console.WriteLine("Examples:"); - Console.WriteLine("Bundle: bundle --source --apphost -o "); - Console.WriteLine("Extract: bundle --extract -o "); - } - - public static void Log(string fmt, params object[] args) - { - if (Verbose) - { - Console.WriteLine("LOG: " + fmt, args); - } - } - - static void Fail(string type, string message) - { - Console.Error.WriteLine($"{type}: {message}"); - } - - static void ParseArgs(string[] args) - { - int i = 0; - Func NextArg = (string option) => - { - if (++i >= args.Length) - { - throw new BundleException("Argument missing for" + option); - } - return args[i]; - }; - - for (; i < args.Length; i++) - { - string arg = args[i]; - switch (arg.ToLower()) - { - case "-?": - case "-h": - case "--help": - Mode = RunMode.Help; - break; - - case "--extract": - Mode = RunMode.Extract; - BundleToExtract = NextArg(arg); - break; - - case "-d": - case "--diagnostics": - Verbose = true; - break; - - case "--apphost": - HostName = NextArg(arg); - break; - - case "--source": - SourceDir = NextArg(arg); - break; - - case "-o": - case "--output": - OutputDir = NextArg(arg); - break; - - case "--pdb": - EmbedPDBs = true; - break; - - default: - throw new BundleException("Invalid option: " + arg); - } - } - - if (Mode == RunMode.Bundle) - { - if (SourceDir == null) - { - throw new BundleException("Missing argument: source directory"); - } - - if (HostName == null) - { - throw new BundleException("Missing argument: host"); - } - } - - if (OutputDir == null) - { - OutputDir = Environment.CurrentDirectory; - } - } - - static void Run() - { - switch (Mode) - { - case RunMode.Help: - Usage(); - break; - - case RunMode.Bundle: - Log($"Bundle from dir: {SourceDir}"); - Log($"Output Directory: {OutputDir}"); - Bundler bundle = new Bundler(HostName, SourceDir, OutputDir, EmbedPDBs); - bundle.MakeBundle(); - break; - - case RunMode.Extract: - Log($"Extract from file: {BundleToExtract}"); - Log($"Output Directory: {OutputDir}"); - Extractor extract = new Extractor(BundleToExtract, OutputDir); - extract.Spill(); - break; - } - } - - public static int Main(string[] args) - { - try - { - Log($"Bundler version: {Bundler.Version}"); - - try - { - ParseArgs(args); - } - catch (BundleException e) - { - Fail("ERROR", e.Message); - Usage(); - return -1; - } - - try - { - Run(); - } - catch (BundleException e) - { - Fail("ERROR", e.Message); - return -2; - } - } - catch (Exception e) - { - Fail("INTERNAL ERROR", e.Message); - return -3; - } - - return 0; - } - } -} - diff --git a/src/installer/managed/Microsoft.NET.Build.Bundle/README.md b/src/installer/managed/Microsoft.NET.Build.Bundle/README.md deleted file mode 100644 index 8a5ef61..0000000 --- a/src/installer/managed/Microsoft.NET.Build.Bundle/README.md +++ /dev/null @@ -1,23 +0,0 @@ -.NET Core Bundler -=================================== - -The Bundler is a tool that embeds an application and its dependencies into the AppHost executable. This tool is used to publish apps as a single-file, as described in this [design document](https://github.com/dotnet/designs/blob/master/accepted/single-file/design.md). - -### Why is the Bundler in core-setup repo? - -The bundler is an independent tool for merging several files into one. -The bundler code lives in the core-setup repo because: -* It is closely related to the AppHost code, which facilitates easy development, update, and testing. -* The `dotnet/cli` and `dotnet/sdk` repos were considered unsuitable because of repo ownership and maintainence concerns. -* It is not worth creating an managing an independent repo just for the tool. - -### Why is the Bundler a tool, not a managed library? - -Users typically only interact with the bundler via dotnet CLI (`dotnet publish /p:PublishSingleFile=true`). The connection between the bundler tool and msbuild is facilitated by MsBuild artifacts in the SDK. The Bundler itself is an executable tool with a command-line interface because: - -1. A library hosted in-proc by the MSBuild process needs to carefully address concerns such as target framework / multitargeting requirements, dependency collision with other tasks / libraries, etc. -2. It forces a crisp contract of inputs and outputs of the MSBuild task. -3. It can be run without spinning up a full build in "ad-hoc" scenarios. -4. It facilitates easy testing. - -The [IL Linker](https://github.com/mono/linker) and [Crossgen compiler](https://github.com/dotnet/coreclr/tree/master/src/tools/crossgen) are similarly implemented as an independent command line tools. \ No newline at end of file diff --git a/src/installer/managed/Microsoft.NET.Build.Bundle/Sdk.props b/src/installer/managed/Microsoft.NET.Build.Bundle/Sdk.props deleted file mode 100644 index 280c976..0000000 --- a/src/installer/managed/Microsoft.NET.Build.Bundle/Sdk.props +++ /dev/null @@ -1,18 +0,0 @@ - - - - - $(MSBuildThisFileDirectory)..\tools\Microsoft.NET.Build.Bundle.dll - - - diff --git a/src/installer/managed/Microsoft.NET.Build.Bundle/BundleException.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/BundleException.cs similarity index 92% rename from src/installer/managed/Microsoft.NET.Build.Bundle/BundleException.cs rename to src/installer/managed/Microsoft.NET.HostModel/Bundle/BundleException.cs index 3725098..486291f 100644 --- a/src/installer/managed/Microsoft.NET.Build.Bundle/BundleException.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/BundleException.cs @@ -4,7 +4,7 @@ using System; -namespace Microsoft.NET.Build.Bundle +namespace Microsoft.NET.HostModel.Bundle { /// /// This exception is thrown when a bundle/extraction diff --git a/src/installer/managed/Microsoft.NET.Build.Bundle/Bundler.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs similarity index 51% rename from src/installer/managed/Microsoft.NET.Build.Bundle/Bundler.cs rename to src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs index a551017..fb42c65 100644 --- a/src/installer/managed/Microsoft.NET.Build.Bundle/Bundler.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Bundler.cs @@ -3,10 +3,12 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; +using System.Linq; using System.IO; using System.Reflection.PortableExecutable; -namespace Microsoft.NET.Build.Bundle +namespace Microsoft.NET.HostModel.Bundle { /// /// Bundler: Functionality to embed the managed app and its dependencies @@ -14,15 +16,14 @@ namespace Microsoft.NET.Build.Bundle /// public class Bundler { - string HostName; - string SourceDir; - string OutputDir; - bool EmbedPDBs; + readonly string HostName; + readonly string OutputDir; + readonly bool EmbedPDBs; + readonly string DepsJson; + readonly string RuntimeConfigJson; + readonly string RuntimeConfigDevJson; - string Application; - string DepsJson; - string RuntimeConfigJson; - string RuntimeConfigDevJson; + readonly Trace trace; /// /// Align embedded assemblies such that they can be loaded @@ -34,50 +35,18 @@ namespace Microsoft.NET.Build.Bundle public static string Version => (Manifest.MajorVersion + "." + Manifest.MinorVersion); - public Bundler(string hostName, string sourceDir, string outputDir, bool embedPDBs) + public Bundler(string hostName, string outputDir, bool embedPDBs = false, bool diagnosticOutput = false) { - SourceDir = sourceDir; - OutputDir = outputDir; HostName = hostName; - EmbedPDBs = embedPDBs; - } + OutputDir = Path.GetFullPath(string.IsNullOrEmpty(outputDir) ? Environment.CurrentDirectory : outputDir); - void ValidateFiles() - { - // Check required directories - if (!Directory.Exists(SourceDir)) - { - throw new BundleException("Dirctory not found: " + SourceDir); - } - if (!Directory.Exists(OutputDir)) - { - throw new BundleException("Dirctory not found: " + OutputDir); - } - - // Convert relative paths to absolute paths. - SourceDir = Path.GetFullPath(SourceDir); - OutputDir = Path.GetFullPath(OutputDir); - - // Set default names string baseName = Path.GetFileNameWithoutExtension(HostName); - Application = baseName + ".dll"; DepsJson = baseName + ".deps.json"; RuntimeConfigJson = baseName + ".runtimeconfig.json"; RuntimeConfigDevJson = baseName + ".runtimeconfig.dev.json"; - // Check that required files exist on disk. - Action checkFileExists = (string name) => - { - string path = Path.Combine(SourceDir, name); - if (!File.Exists(path)) - { - throw new BundleException("File not found: " + path); - } - }; - - checkFileExists(HostName); - checkFileExists(Application); - // The *.json files may or may not exist. + EmbedPDBs = embedPDBs; + trace = new Trace(diagnosticOutput); } /// @@ -140,11 +109,6 @@ namespace Microsoft.NET.Build.Bundle return FileType.RuntimeConfigJson; } - if (fileRelativePath.Equals(Application)) - { - return FileType.Application; - } - try { PEReader peReader = new PEReader(file); @@ -161,63 +125,111 @@ namespace Microsoft.NET.Build.Bundle return FileType.Extract; } - void GenerateBundle() + /// + /// Generate a bundle, given the specification of embedded files + /// + /// + /// An enumeration FileSpecs for the files to be embedded. + /// + /// + /// The full path the the generated bundle file + /// + /// + /// ArgumentException if input is invalid + /// IOExceptions and ArgumentExceptions from callees flow to the caller. + /// + public string GenerateBundle(IReadOnlyList fileSpecs) { - string bundlePath = Path.Combine(OutputDir, HostName); + trace.Log($"Bundler version {Version}"); + string bundlePath = Path.Combine(OutputDir, HostName); if (File.Exists(bundlePath)) { - Program.Log($"Ovewriting existing File {bundlePath}"); + trace.Log($"Ovewriting existing File {bundlePath}"); + } + + string hostSource; + try + { + hostSource = fileSpecs.Where(x => x.BundleRelativePath.Equals(HostName)).Single().SourcePath; + } + catch (InvalidOperationException) + { + throw new ArgumentException("Input must uniquely specify the host binary"); } // Start with a copy of the host executable. // Copy the file to preserve its permissions. - File.Copy(Path.Combine(SourceDir, HostName), bundlePath, overwrite: true); + File.Copy(hostSource, bundlePath, overwrite: true); using (BinaryWriter writer = new BinaryWriter(File.OpenWrite(bundlePath))) { - Stream bundle = writer.BaseStream; Manifest manifest = new Manifest(); - + Stream bundle = writer.BaseStream; bundle.Position = bundle.Length; - // Get all files in the source directory and all sub-directories. - string[] sources = Directory.GetFiles(SourceDir, searchPattern: "*", searchOption: SearchOption.AllDirectories); - - // Sort the file names to keep the bundle construction deterministic. - Array.Sort(sources, StringComparer.Ordinal); - - foreach (string filePath in sources) + // Write the files from the specification into the bundle + foreach (var fileSpec in fileSpecs) { - // filePath is the full-path of files within source directory, and any of its sub-directories. - // We need the relative paths with respect to the source directory. - string relativePath = Path.GetRelativePath(SourceDir, filePath); - - if (!ShouldEmbed(relativePath)) + if (!ShouldEmbed(fileSpec.BundleRelativePath)) { - Program.Log($"Skip: {relativePath}"); + trace.Log($"Skip: {fileSpec.BundleRelativePath}"); continue; } - using (FileStream file = File.OpenRead(filePath)) + using (FileStream file = File.OpenRead(fileSpec.SourcePath)) { - FileType type = InferType(relativePath, file); + FileType type = InferType(fileSpec.BundleRelativePath, file); long startOffset = AddToBundle(bundle, file, type); - FileEntry entry = new FileEntry(type, relativePath, startOffset, file.Length); + FileEntry entry = new FileEntry(type, fileSpec.BundleRelativePath, startOffset, file.Length); manifest.Files.Add(entry); - Program.Log($"Embed: {entry}"); + trace.Log($"Embed: {entry}"); } } - manifest.Write(writer); - Program.Log($"Bundle: Path={bundlePath} Size={bundle.Length}"); + // Write the bundle manifest + long manifestOffset = manifest.Write(writer); + trace.Log($"Manifest: Offset={manifestOffset}, Size={writer.BaseStream.Position - manifestOffset}"); + trace.Log($"Bundle: Path={bundlePath} Size={bundle.Length}"); } + + return bundlePath; } - public void MakeBundle() + string RelativePath(string dirFullPath, string fileFullPath) { - ValidateFiles(); - GenerateBundle(); + // This function is used in lieu of Path.GetRelativePath because + // * Path.GetRelativePath() doesn't exist in netstandard2.0 + // * This implementation is pretty much only intended for testing. + // SDK integration invokes GenerateBundle(fileSpecs) directly. + // + // In later revisions, we should target netstandard2.1, and replace + // this function with Path.GetRelativePath(). + + return fileFullPath.Substring(dirFullPath.TrimEnd(Path.DirectorySeparatorChar).Length).TrimStart(Path.DirectorySeparatorChar); + } + + /// + /// Generate a bundle containind the (embeddable) files in sourceDir + /// + public string GenerateBundle(string sourceDir) + { + // Convert sourceDir to absolute path + sourceDir = Path.GetFullPath(sourceDir); + + // Get all files in the source directory and all sub-directories. + string[] sources = Directory.GetFiles(sourceDir, searchPattern: "*", searchOption: SearchOption.AllDirectories); + + // Sort the file names to keep the bundle construction deterministic. + Array.Sort(sources, StringComparer.Ordinal); + + List fileSpecs = new List(sources.Length); + foreach(var file in sources) + { + fileSpecs.Add(new FileSpec(file, RelativePath(sourceDir, file))); + } + + return GenerateBundle(fileSpecs); } } } diff --git a/src/installer/managed/Microsoft.NET.Build.Bundle/Extractor.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Extractor.cs similarity index 68% rename from src/installer/managed/Microsoft.NET.Build.Bundle/Extractor.cs rename to src/installer/managed/Microsoft.NET.HostModel/Bundle/Extractor.cs index 5b41f88..66b11af 100644 --- a/src/installer/managed/Microsoft.NET.Build.Bundle/Extractor.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Extractor.cs @@ -5,7 +5,7 @@ using System; using System.IO; -namespace Microsoft.NET.Build.Bundle +namespace Microsoft.NET.HostModel.Bundle { /// /// Extractor: The functionality to extract the files embedded @@ -16,20 +16,30 @@ namespace Microsoft.NET.Build.Bundle string OutputDir; string BundlePath; - public Extractor(string bundlePath, string outputDir) + readonly Trace trace; + + public Extractor(string bundlePath, string outputDir, + bool diagnosticOutput = false) { BundlePath = bundlePath; - OutputDir = outputDir; + OutputDir = Path.GetFullPath(string.IsNullOrEmpty(outputDir) ? Environment.CurrentDirectory : outputDir); + trace = new Trace(diagnosticOutput); } - public void Spill() + /// + /// Extract all files in the bundle to disk + /// + /// + /// BundleException if the bundle is invalid or malformed. + /// IOExceptions and ArgumentExceptions from callees flow to the caller. + /// + public void ExtractFiles() { try { - if (!File.Exists(BundlePath)) - { - throw new BundleException("File not found: " + BundlePath); - } + trace.Log($"Bundler version {Bundler.Version}"); + trace.Log($"Extract from file: {BundlePath}"); + trace.Log($"Output Directory: {OutputDir}"); using (BinaryReader reader = new BinaryReader(File.OpenRead(BundlePath))) { @@ -37,7 +47,7 @@ namespace Microsoft.NET.Build.Bundle foreach (FileEntry entry in manifest.Files) { - Program.Log($"Spill: {entry}"); + trace.Log($"Extract: {entry}"); string fileRelativePath = entry.RelativePath.Replace(Manifest.DirectorySeparatorChar, Path.DirectorySeparatorChar); string filePath = Path.Combine(OutputDir, fileRelativePath); @@ -62,8 +72,9 @@ namespace Microsoft.NET.Build.Bundle } } } - catch (IOException) + catch (EndOfStreamException) { + // Trying to read non-existant bits in the bundle throw new BundleException("Malformed Bundle"); } catch (ArgumentOutOfRangeException) diff --git a/src/installer/managed/Microsoft.NET.Build.Bundle/FileEntry.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs similarity index 95% rename from src/installer/managed/Microsoft.NET.Build.Bundle/FileEntry.cs rename to src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs index d2361f8..7804608 100644 --- a/src/installer/managed/Microsoft.NET.Build.Bundle/FileEntry.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileEntry.cs @@ -5,7 +5,7 @@ using System; using System.IO; -namespace Microsoft.NET.Build.Bundle +namespace Microsoft.NET.HostModel.Bundle { /// /// FileEntry: Records information about embedded files. @@ -21,7 +21,7 @@ namespace Microsoft.NET.Build.Bundle public class FileEntry { public FileType Type; - public string RelativePath; // Path of an embedded file, relative to the Bundle source-directory. + public string RelativePath; // Path of an embedded file, relative to the dll. public long Offset; public long Size; diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileSpec.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileSpec.cs new file mode 100644 index 0000000..7b18f57 --- /dev/null +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileSpec.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.NET.HostModel.Bundle +{ + /// + /// Information about files to embed into the Bundle (input to the Bundler). + /// + /// SourcePath: path to the file to be bundled at compile time + /// BundleRelativePath: path where the file is expected at run time, + /// relative to the app DLL. + /// + public struct FileSpec + { + public string SourcePath; + public string BundleRelativePath; + + public FileSpec(string sourcePath, string bundleRelativePath) + { + SourcePath = sourcePath; + BundleRelativePath = bundleRelativePath; + } + } +} + diff --git a/src/installer/managed/Microsoft.NET.Build.Bundle/FileType.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileType.cs similarity index 87% rename from src/installer/managed/Microsoft.NET.Build.Bundle/FileType.cs rename to src/installer/managed/Microsoft.NET.HostModel/Bundle/FileType.cs index f191789..164eb5a 100644 --- a/src/installer/managed/Microsoft.NET.Build.Bundle/FileType.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/FileType.cs @@ -2,7 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -namespace Microsoft.NET.Build.Bundle +namespace Microsoft.NET.HostModel.Bundle { /// /// FileType: Identifies the type of file embedded into the bundle. @@ -19,12 +19,11 @@ namespace Microsoft.NET.Build.Bundle /// public enum FileType : byte { - Application, // Represents the main app, also an assembly Assembly, // IL Assemblies, which will be processed from bundle Ready2Run, // R2R assemblies, currently unused, spilled to disk. DepsJson, // Configuration file, processed from bundle RuntimeConfigJson, // Configuration file, processed from bundle - Extract // Files spilled to disk by the host + Extract // Files spilled to disk by the host }; } diff --git a/src/installer/managed/Microsoft.NET.Build.Bundle/Manifest.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs similarity index 94% rename from src/installer/managed/Microsoft.NET.Build.Bundle/Manifest.cs rename to src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs index 2d0ef2e..880213c 100644 --- a/src/installer/managed/Microsoft.NET.Build.Bundle/Manifest.cs +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Manifest.cs @@ -2,12 +2,11 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; using System.Collections.Generic; using System.IO; using System.Linq; -namespace Microsoft.NET.Build.Bundle +namespace Microsoft.NET.HostModel.Bundle { /// /// BundleManifest is a description of the contents of a bundle file. @@ -59,7 +58,7 @@ namespace Microsoft.NET.Build.Bundle Files = new List(); } - public void Write(BinaryWriter writer) + public long Write(BinaryWriter writer) { long startOffset = writer.BaseStream.Position; @@ -78,8 +77,7 @@ namespace Microsoft.NET.Build.Bundle writer.Write(startOffset); writer.Write(Signature); - long size = writer.BaseStream.Position - startOffset; - Program.Log($"Manifest: Offset={startOffset}, Size={size}"); + return startOffset; } public static Manifest Read(BinaryReader reader) diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/README.md b/src/installer/managed/Microsoft.NET.HostModel/Bundle/README.md new file mode 100644 index 0000000..a25b62d --- /dev/null +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/README.md @@ -0,0 +1,12 @@ +.NET Core Bundler +=================================== + +The Bundler is a tool that embeds an application and its dependencies into the AppHost executable. This tool is used to publish apps as a single-file, as described in this [design document](https://github.com/dotnet/designs/blob/master/accepted/single-file/design.md). + +### Why is the Bundler in core-setup repo? + +The bundler is an independent tool for merging several files into one. +The bundler code lives in the core-setup repo because: +* It is closely related to the AppHost code, which facilitates easy development, update, and testing. +* The `dotnet/cli` and `dotnet/sdk` repos were considered unsuitable because of repo ownership and maintainence concerns. +* It is not worth creating an managing an independent repo just for the tool. diff --git a/src/installer/managed/Microsoft.NET.HostModel/Bundle/Trace.cs b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Trace.cs new file mode 100644 index 0000000..bb251dfa --- /dev/null +++ b/src/installer/managed/Microsoft.NET.HostModel/Bundle/Trace.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.NET.HostModel.Bundle +{ + /// + /// Tracing utilities for diagnostic output + /// + public class Trace + { + readonly bool Verbose = false; + + public Trace(bool verbose) + { + Verbose = verbose; + } + + public void Log(string fmt, params object[] args) + { + if (Verbose) + { + Console.WriteLine("LOG: " + fmt, args); + } + } + + public void Error(string type, string message) + { + Console.Error.WriteLine($"ERROR: {message}"); + } + } +} + diff --git a/src/installer/managed/Microsoft.NET.HostModel/Microsoft.NET.HostModel.csproj b/src/installer/managed/Microsoft.NET.HostModel/Microsoft.NET.HostModel.csproj new file mode 100644 index 0000000..d7bc044 --- /dev/null +++ b/src/installer/managed/Microsoft.NET.HostModel/Microsoft.NET.HostModel.csproj @@ -0,0 +1,18 @@ + + + + netstandard2.0 + Abstractions for modifying .net core host-binaries + + + + + + + 1.5.0 + + + + + + diff --git a/src/installer/managed/dir.proj b/src/installer/managed/dir.proj index 3ebae8f..0733d39 100644 --- a/src/installer/managed/dir.proj +++ b/src/installer/managed/dir.proj @@ -10,7 +10,7 @@ - + diff --git a/src/installer/pkg/packaging/dir.proj b/src/installer/pkg/packaging/dir.proj index ad968f1..5b368e6 100644 --- a/src/installer/pkg/packaging/dir.proj +++ b/src/installer/pkg/packaging/dir.proj @@ -198,7 +198,7 @@ - + diff --git a/src/installer/test/Microsoft.NET.Build.Bundle.Tests/BundleAndExtract.cs b/src/installer/test/Microsoft.NET.HostModel.Tests/BundleAndExtract.cs similarity index 79% rename from src/installer/test/Microsoft.NET.Build.Bundle.Tests/BundleAndExtract.cs rename to src/installer/test/Microsoft.NET.HostModel.Tests/BundleAndExtract.cs index 81b1f65..51246b0 100644 --- a/src/installer/test/Microsoft.NET.Build.Bundle.Tests/BundleAndExtract.cs +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/BundleAndExtract.cs @@ -6,6 +6,7 @@ using System; using System.IO; using Xunit; using Microsoft.DotNet.Cli.Build.Framework; +using Microsoft.NET.HostModel.Bundle; namespace Microsoft.DotNet.CoreSetup.Test.BundleTests.BundleExtract { @@ -18,7 +19,6 @@ namespace Microsoft.DotNet.CoreSetup.Test.BundleTests.BundleExtract sharedTestState = fixture; } - private void Run(TestProjectFixture fixture, string publishDir, string singleFileDir) { var dotnet = fixture.SdkDotnet; @@ -35,32 +35,12 @@ namespace Microsoft.DotNet.CoreSetup.Test.BundleTests.BundleExtract .HaveStdOutContaining("Wow! We now say hello to the big world and you."); // Bundle to a single-file - string bundleDll = Path.Combine(sharedTestState.RepoDirectories.Artifacts, - "Microsoft.NET.Build.Bundle", - "netcoreapp2.0", - "Microsoft.NET.Build.Bundle.dll"); - string[] bundleArgs = { "--source", publishDir, - "--apphost", hostName, - "--output", singleFileDir }; - - dotnet.Exec(bundleDll, bundleArgs) - .CaptureStdErr() - .CaptureStdOut() - .Execute() - .Should() - .Pass(); + Bundler bundler = new Bundler(hostName, singleFileDir); + string singleFile = bundler.GenerateBundle(publishDir); - // Extract the contents - string singleFile = Path.Combine(singleFileDir, hostName); - string[] extractArgs = { "--extract", singleFile, - "--output", singleFileDir }; - - dotnet.Exec(bundleDll, extractArgs) - .CaptureStdErr() - .CaptureStdOut() - .Execute() - .Should() - .Pass(); + // Extract the file + Extractor extractor = new Extractor(singleFile, singleFileDir); + extractor.ExtractFiles(); // Run the extracted app Command.Create(singleFile) diff --git a/src/installer/test/Microsoft.NET.Build.Bundle.Tests/Microsoft.NET.Build.Bundle.Tests.csproj b/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Tests.csproj similarity index 64% rename from src/installer/test/Microsoft.NET.Build.Bundle.Tests/Microsoft.NET.Build.Bundle.Tests.csproj rename to src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Tests.csproj index 37d123d..6cc5490 100644 --- a/src/installer/test/Microsoft.NET.Build.Bundle.Tests/Microsoft.NET.Build.Bundle.Tests.csproj +++ b/src/installer/test/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.Tests.csproj @@ -1,10 +1,10 @@  - Microsoft.NET.Build.Bundle.Tests + Microsoft.NET.HostModel.Tests netcoreapp2.0 - Microsoft.NET.Build.Bundle.Tests - Microsoft.NET.Build.Bundle.Tests + Microsoft.NET.HostModel.Tests + Microsoft.NET.HostModel.Tests true @@ -16,7 +16,7 @@ - + diff --git a/src/installer/test/dir.props b/src/installer/test/dir.props index bc270c9..9c13c25 100644 --- a/src/installer/test/dir.props +++ b/src/installer/test/dir.props @@ -8,9 +8,9 @@ - + -- 2.7.4