Wasm dedup (#49538)

* [wasm] Disable the linkonce linking of wrappers for now, the aot metadata about wrappers is not dedup-ed, so a separate dedup pass still saves more space.

* [wasm] Add dedup support to the build to avoid generating generic instances etc. multiple times.

This works as follows:
* Controlled by a new public 'WasmDedup' property, defaults to true.
* All assemblies are AOT compiled with the 'dedup-skip' option, which causes
the AOT compiler to avoid generating generic instances, certain wrappers etc.
* A new internal assembly called aot-instances.dll is added to the build.
* When aot-instances.dll is AOTed, all the other assemblies are AOTed together,
but only the generic instances/wrappers are emitted into its AOT image.

diff --git a/src/mono/mono/mini/aot-runtime.c b/src/mono/mono/mini/aot-runtime.c
index 7bbeb40..816d539 100644 (file)
--- a/src/mono/mono/mini/aot-runtime.c
+++ b/src/mono/mono/mini/aot-runtime.c
@@ -4337,9 +4337,12 @@ mono_aot_can_dedup (MonoMethod *method)
                        info->subtype == WRAPPER_SUBTYPE_INTERP_LMF ||
                        info->subtype == WRAPPER_SUBTYPE_AOT_INIT)
                        return FALSE;
+#if 0
+               // See is_linkonce_method () in mini-llvm.c
                if (info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_IN_SIG || info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_OUT_SIG)
                        /* Handled using linkonce */
                        return FALSE;
+#endif
                return TRUE;
        }
        default:

diff --git a/src/mono/mono/mini/llvm-runtime.cpp b/src/mono/mono/mini/llvm-runtime.cpp
index 9982ccc..a621ae8 100644 (file)
--- a/src/mono/mono/mini/llvm-runtime.cpp
+++ b/src/mono/mono/mini/llvm-runtime.cpp
@@ -3,6 +3,10 @@
 
 #include <glib.h>
 
+#ifdef HOST_WASM
+#include <emscripten.h>
+#endif
+
 extern "C" {
 
 void
@@ -10,6 +14,14 @@ mono_llvm_cpp_throw_exception (void)
 {
        gint32 *ex = NULL;
 
+#ifdef HOST_WASM
+               EM_ASM(
+                          var err = new Error();
+                          console.log ("Throw stacktrace: \n");
+                          console.log (err.stack);
+                          );
+#endif
+
        /* The generated code catches an int32* */
        throw ex;
 }

diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c
index 0d72b59..e87e53b 100644 (file)
--- a/src/mono/mono/mini/mini-llvm.c
+++ b/src/mono/mono/mini/mini-llvm.c
@@ -11260,12 +11260,15 @@ is_linkonce_method (MonoMethod *method)
         * FIXME: Fails System.Core tests
         * -> amodule->sorted_methods contains duplicates, screwing up jit tables.
         */
+       // FIXME: This works, but the aot data for the methods is still kept, so size still increases
+#if 0
        if (method->wrapper_type == MONO_WRAPPER_OTHER) {
                WrapperInfo *info = mono_marshal_get_wrapper_info (method);
                if (info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_IN_SIG || info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_OUT_SIG)
                        return TRUE;
        }
 #endif
+#endif
        return FALSE;
 }
 

diff --git a/src/mono/wasm/build/WasmApp.targets b/src/mono/wasm/build/WasmApp.targets
index 34732b7..964b162 100644 (file)
--- a/src/mono/wasm/build/WasmApp.targets
+++ b/src/mono/wasm/build/WasmApp.targets
@@ -24,6 +24,7 @@
       - $(WasmDebugLevel)
       - $(WasmNativeDebugSymbols) - Build with native debug symbols, useful only with `$(RunAOTCompilation)`, or `$(WasmBuildNative)`
                                     Defaults to true.
+      - $(WasmDedup)         - Whenever to dedup generic instances when using AOT. Defaults to true.
 
       - $(WasmProfilers)     - Profilers to use
       - $(AOTMode)           - Defaults to `AotInterp`
@@ -64,6 +65,7 @@
 
   <PropertyGroup>
     <WasmStripAOTAssemblies>false</WasmStripAOTAssemblies>
+    <WasmDedup Condition="'$(WasmDedup)' == ''">$(RunAOTCompilation)</WasmDedup>
 
     <!--<WasmStripAOTAssemblies Condition="'$(AOTMode)' == 'AotInterp'">false</WasmStripAOTAssemblies>-->
     <!--<WasmStripAOTAssemblies Condition="'$(WasmStripAOTAssemblies)' == ''">$(RunAOTCompilation)</WasmStripAOTAssemblies>-->
@@ -106,6 +108,27 @@
            Text="Builing in AOTMode=LLVMonly, but found some assemblies marked as _InternalForceInterpret: @(_AOT_InternalForceInterpretAssemblies)" />
 
     <Message Text="AOT'ing @(_AotInputAssemblies->Count()) assemblies" Importance="High" />
+
+    <!-- Dedup -->
+    <PropertyGroup Condition="'$(WasmDedup)' == 'true'">
+      <_WasmDedupAssembly>$(_WasmIntermediateOutputPath)\aot-instances.dll</_WasmDedupAssembly>
+    </PropertyGroup>
+    <WriteLinesToFile Condition="'$(WasmDedup)' == 'true'" File="$(_WasmIntermediateOutputPath)/aot-instances.cs" Overwrite="true" Lines="" />
+    <Csc
+      Condition="'$(WasmDedup)' == 'true'"
+      Sources="$(_WasmIntermediateOutputPath)\aot-instances.cs"
+      OutputAssembly="$(_WasmDedupAssembly)"
+      TargetType="library"
+      References="@(ReferencePath)"
+      ToolExe="$(CscToolExe)"
+      ToolPath="$(CscToolPath)" />
+    <ItemGroup Condition="'$(WasmDedup)' == 'true'">
+      <_AotInputAssemblies Include="$(_WasmDedupAssembly)">
+        <AotArguments>@(MonoAOTCompilerDefaultAotArguments, ';')</AotArguments>
+        <ProcessArguments>@(MonoAOTCompilerDefaultProcessArguments, ';')</ProcessArguments>
+      </_AotInputAssemblies>
+    </ItemGroup>
+
     <MonoAOTCompiler
       CompilerBinaryPath="$(MonoAotCrossCompilerPath)"
       OutputDir="$(_WasmIntermediateOutputPath)"
@@ -119,6 +142,7 @@
       AotModulesTablePath="$(_WasmIntermediateOutputPath)driver-gen.c"
       UseLLVM="true"
       DisableParallelAot="true"
+      DedupAssembly="$(_WasmDedupAssembly)"
       LLVMPath="$(EMSDK_PATH)\upstream\bin">
       <Output TaskParameter="CompiledAssemblies" ItemName="_WasmAssembliesInternal" />
       <Output TaskParameter="FileWrites" ItemName="FileWrites" />
@@ -407,7 +431,7 @@ EMSCRIPTEN_KEEPALIVE void mono_wasm_load_profiler_aot (const char *desc) { mono_
     </ItemGroup>
 
     <!-- $(WasmResolveAssembliesBeforeBuild) can add more assemblies, so no point checking the count in that case -->
-    <Error Condition="'$(WasmResolveAssembliesBeforeBuild)' != 'true' and @(WasmAssembliesFinal->Distinct()->Count()) != @(WasmAssembliesToBundle->Distinct()->Count())"
+    <Error Condition="'$(WasmResolveAssembliesBeforeBuild)' != 'true' and @(WasmAssembliesFinal->Distinct()->Count()) != @(WasmAssembliesToBundle->Distinct()->Count()) and '$(WasmDedup)' != 'true'"
       Text="Bug: The wasm build started with @(WasmAssembliesToBundle->Count()) assemblies, but completed with @(WasmAssembliesFinal->Count()).
       WasmAssembliesToBundle: @(WasmAssembliesToBundle)
       WasmAssembliesFinal: @(WasmAssembliesFinal)" />

diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs
index d7a7ab6..fc5f9b7 100644 (file)
--- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs
+++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs
@@ -121,6 +121,11 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task
     /// </summary>
     public string? MsymPath { get; set; }
 
+    /// <summary>
+    /// The assembly whose AOT image will contained dedup-ed generic instances
+    /// </summary>
+    public string? DedupAssembly { get; set; }
+
     [Output]
     public string[]? FileWrites { get; private set; }
 
@@ -242,6 +247,7 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task
         var aotAssembly = new TaskItem(assembly);
         var aotArgs = new List<string>();
         var processArgs = new List<string>();
+        bool isDedup = assembly == DedupAssembly;
 
         var a = assemblyItem.GetMetadata("AotArguments");
         if (a != null)
@@ -274,6 +280,15 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task
 
         string assemblyFilename = Path.GetFileName(assembly);
 
+        if (isDedup)
+        {
+            aotArgs.Add($"dedup-include={assemblyFilename}");
+        }
+        else if (!string.IsNullOrEmpty (DedupAssembly))
+        {
+            aotArgs.Add("dedup-skip");
+        }
+
         // compute output mode and file names
         if (parsedAotMode == MonoAotMode.LLVMOnly || parsedAotMode == MonoAotMode.AotInterp)
         {
@@ -350,14 +365,43 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task
         // values, which wont work.
         processArgs.Add($"\"--aot={string.Join(",", aotArgs)}\"");
 
-        processArgs.Add($"\"{assemblyFilename}\"");
+        string paths = "";
+        if (isDedup)
+        {
+            StringBuilder sb = new StringBuilder();
+            HashSet<string> allPaths = new HashSet<string>();
+            foreach (var aItem in Assemblies)
+            {
+                string filename = aItem.ItemSpec;
+                processArgs.Add(filename);
+                string dir = Path.GetDirectoryName(filename)!;
+                if (!allPaths.Contains(dir))
+                {
+                    allPaths.Add(dir);
+                    if (sb.Length > 0)
+                        sb.Append(Path.PathSeparator);
+                    sb.Append(dir);
+                }
+            }
+            if (sb.Length > 0)
+                sb.Append(Path.PathSeparator);
+            sb.Append(monoPaths);
+            paths = sb.ToString();
+        }
+        else
+        {
+            paths = $"{assemblyDir}{Path.PathSeparator}{monoPaths}";
+            processArgs.Add(assemblyFilename);
+        }
 
         var envVariables = new Dictionary<string, string>
         {
-            {"MONO_PATH", $"{assemblyDir}{Path.PathSeparator}{monoPaths}"},
+            {"MONO_PATH", paths},
             {"MONO_ENV_OPTIONS", string.Empty} // we do not want options to be provided out of band to the cross compilers
         };
 
+        Utils.LogInfo ($"MONO_PATH={paths}");
+
         try
         {
             // run the AOT compiler