From: Fan Yang <52458914+fanyang-mono@users.noreply.github.com> Date: Thu, 10 Aug 2023 19:45:07 +0000 (-0400) Subject: [WASM] Add ILStrip task to wasm app build process (#88926) X-Git-Tag: accepted/tizen/unified/riscv/20231226.055536~365 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=867e185209d329c406ab57aa7c9706e4ccbc9d56;p=platform%2Fupstream%2Fdotnet%2Fruntime.git [WASM] Add ILStrip task to wasm app build process (#88926) * Add ILStrip task to wasm app build process * Make it work for wasm app building workflow * Interp: stop inlining stripped methods. ILStrip: set code size to zero for tiny methods * [mono][aot] Avoid adding some methods to the compiled method file. * Methods which have 'deopt' set can enter the interpreter during EH. * Methods which have 'interp_entry_only' set are AOTed, but the AOT code is only used to enter the interpreter. * Only trim the methods that interpreter is able to call the aot'ed verion of it * Add default value and documentation for WASMStripIL * Move jit_call_can_be_supported to interp.c * Minor format fix * Add a test * For testing * Fix typo * Skip TestUtilities Reference * Address review feedback * Change it to true * Change name to trimming eligible * Remove testing * Address review feedback * Address review feedback from Kate * Add a var for llvm_only --------- Co-authored-by: Zoltan Varga --- diff --git a/docs/workflow/building/mono/README.md b/docs/workflow/building/mono/README.md index 318c23c..19dfcc4 100644 --- a/docs/workflow/building/mono/README.md +++ b/docs/workflow/building/mono/README.md @@ -66,7 +66,7 @@ For `build.sh` `/p:DisableCrossgen=true` - Skips building the installer if you don't need it (builds faster) -`-p:KeepNativeSymbols=true` - Keep the symbols in the binary instead of stripping them out to a separate file. This helps with debugging Mono with lldb. +`/p:KeepNativeSymbols=true` - Keep the symbols in the binary instead of stripping them out to a separate file. This helps with debugging Mono with lldb. The build has a number of options that you can learn about using build -?. diff --git a/src/mono/mono/mini/aot-compiler.c b/src/mono/mono/mini/aot-compiler.c index c3568f3..1944a77 100644 --- a/src/mono/mono/mini/aot-compiler.c +++ b/src/mono/mono/mini/aot-compiler.c @@ -188,7 +188,7 @@ typedef struct MonoAotOptions { char *llvm_outfile; char *data_outfile; char *export_symbols_outfile; - char *compiled_methods_outfile; + char *trimming_eligible_methods_outfile; GList *profile_files; GList *mibc_profile_files; gboolean save_temps; @@ -420,7 +420,7 @@ typedef struct MonoAotCompile { FILE *logfile; FILE *instances_logfile; FILE *data_outfile; - FILE *compiled_methods_outfile; + FILE *trimming_eligible_methods_outfile; int datafile_offset; int gc_name_offset; @@ -8804,8 +8804,8 @@ mono_aot_parse_options (const char *aot_options, MonoAotOptions *opts) opts->llvm_outfile = g_strdup (arg + strlen ("llvm-outfile=")); } else if (str_begins_with (arg, "export-symbols-outfile=")) { opts->export_symbols_outfile = g_strdup (arg + strlen ("export-symbols-outfile=")); - } else if (str_begins_with (arg, "compiled-methods-outfile=")) { - opts->compiled_methods_outfile = g_strdup (arg + strlen ("compiled-methods-outfile=")); + } else if (str_begins_with (arg, "trimming-eligible-methods-outfile=")) { + opts->trimming_eligible_methods_outfile = g_strdup (arg + strlen ("trimming-eligible-methods-outfile=")); } else if (str_begins_with (arg, "temp-path=")) { opts->temp_path = clean_path (g_strdup (arg + strlen ("temp-path="))); } else if (str_begins_with (arg, "save-temps")) { @@ -9864,9 +9864,11 @@ compile_method (MonoAotCompile *acfg, MonoMethod *method) mono_atomic_inc_i32 (&acfg->stats.ccount); - if (acfg->aot_opts.compiled_methods_outfile && acfg->compiled_methods_outfile != NULL) { - if (!mono_method_is_generic_impl (method) && method->token != 0) { - fprintf (acfg->compiled_methods_outfile, "%x\n", method->token); + if (acfg->aot_opts.trimming_eligible_methods_outfile && acfg->trimming_eligible_methods_outfile != NULL) { + if (!mono_method_is_generic_impl (method) && method->token != 0 && !cfg->deopt && !cfg->interp_entry_only && mini_get_interp_callbacks ()->jit_call_can_be_supported (method, mono_method_signature_internal (method), acfg->aot_opts.llvm_only)) { + // The call back to jit_call_can_be_supported is necessary for WASM, because it would still interprete some methods sometimes even though they were already AOT'ed. + // When that happens, interpreter needs to have the capability to call the AOT'ed version of that method, since the method body has already been trimmed. + fprintf (acfg->trimming_eligible_methods_outfile, "%x\n", method->token); } } } @@ -14884,13 +14886,13 @@ aot_assembly (MonoAssembly *ass, guint32 jit_opts, MonoAotOptions *aot_options) acfg->logfile = fopen (acfg->aot_opts.logfile, "a+"); } - if (acfg->aot_opts.compiled_methods_outfile && acfg->dedup_phase != DEDUP_COLLECT) { - acfg->compiled_methods_outfile = fopen (acfg->aot_opts.compiled_methods_outfile, "w+"); - if (!acfg->compiled_methods_outfile) - aot_printerrf (acfg, "Unable to open compiled-methods-outfile specified file %s\n", acfg->aot_opts.compiled_methods_outfile); + if (acfg->aot_opts.trimming_eligible_methods_outfile && acfg->dedup_phase != DEDUP_COLLECT) { + acfg->trimming_eligible_methods_outfile = fopen (acfg->aot_opts.trimming_eligible_methods_outfile, "w+"); + if (!acfg->trimming_eligible_methods_outfile) + aot_printerrf (acfg, "Unable to open trimming-eligible-methods-outfile specified file %s\n", acfg->aot_opts.trimming_eligible_methods_outfile); else { - fprintf(acfg->compiled_methods_outfile, "%s\n", ass->image->filename); - fprintf(acfg->compiled_methods_outfile, "%s\n", ass->image->guid); + fprintf(acfg->trimming_eligible_methods_outfile, "%s\n", ass->image->filename); + fprintf(acfg->trimming_eligible_methods_outfile, "%s\n", ass->image->guid); } } @@ -15244,8 +15246,8 @@ aot_assembly (MonoAssembly *ass, guint32 jit_opts, MonoAotOptions *aot_options) current_acfg = NULL; - if (acfg->compiled_methods_outfile) { - fclose (acfg->compiled_methods_outfile); + if (acfg->trimming_eligible_methods_outfile) { + fclose (acfg->trimming_eligible_methods_outfile); } return emit_aot_image (acfg); @@ -15623,9 +15625,9 @@ mono_aot_assemblies (MonoAssembly **assemblies, int nassemblies, guint32 jit_opt dedup_methods = g_hash_table_new (NULL, NULL); } - if (aot_opts.compiled_methods_outfile) { - if (g_ensure_directory_exists (aot_opts.compiled_methods_outfile) == FALSE) { - fprintf (stderr, "AOT : failed to create the directory to save the compiled method names: %s\n", aot_opts.compiled_methods_outfile); + if (aot_opts.trimming_eligible_methods_outfile) { + if (g_ensure_directory_exists (aot_opts.trimming_eligible_methods_outfile) == FALSE) { + fprintf (stderr, "AOT : failed to create the directory to save the trimmable method names: %s\n", aot_opts.trimming_eligible_methods_outfile); exit (1); } } diff --git a/src/mono/mono/mini/ee.h b/src/mono/mono/mini/ee.h index caa04be..1e88e9a 100644 --- a/src/mono/mono/mini/ee.h +++ b/src/mono/mono/mini/ee.h @@ -65,6 +65,7 @@ typedef gpointer MonoInterpFrameHandle; MONO_EE_CALLBACK (void, entry_llvmonly, (gpointer res, gpointer *args, gpointer imethod)) \ MONO_EE_CALLBACK (gpointer, get_interp_method, (MonoMethod *method)) \ MONO_EE_CALLBACK (MonoJitInfo*, compile_interp_method, (MonoMethod *method, MonoError *error)) \ + MONO_EE_CALLBACK (gboolean, jit_call_can_be_supported, (MonoMethod *method, MonoMethodSignature *sig, gboolean is_llvm_only)) \ typedef struct _MonoEECallbacks { diff --git a/src/mono/mono/mini/interp-stubs.c b/src/mono/mono/mini/interp-stubs.c index 8d1487c..0ade0c9 100644 --- a/src/mono/mono/mini/interp-stubs.c +++ b/src/mono/mono/mini/interp-stubs.c @@ -252,6 +252,12 @@ stub_compile_interp_method (MonoMethod *method, MonoError *error) return NULL; } +static gboolean +stub_jit_call_can_be_supported (MonoMethod *method, MonoMethodSignature *sig, gboolean is_llvm_only) +{ + return TRUE; +} + #undef MONO_EE_CALLBACK #define MONO_EE_CALLBACK(ret, name, sig) stub_ ## name, diff --git a/src/mono/mono/mini/interp/interp-internals.h b/src/mono/mono/mini/interp/interp-internals.h index 9be34b4..4e0be7d 100644 --- a/src/mono/mono/mini/interp/interp-internals.h +++ b/src/mono/mono/mini/interp/interp-internals.h @@ -320,6 +320,9 @@ mono_mint_type (MonoType *type); int mono_interp_type_size (MonoType *type, int mt, int *align_p); +gboolean +interp_jit_call_can_be_supported (MonoMethod *method, MonoMethodSignature *sig, gboolean is_llvm_only); + #if HOST_BROWSER gboolean diff --git a/src/mono/mono/mini/interp/interp.c b/src/mono/mono/mini/interp/interp.c index 7b6715f..6fb6bdd 100644 --- a/src/mono/mono/mini/interp/interp.c +++ b/src/mono/mono/mini/interp/interp.c @@ -4081,7 +4081,39 @@ main_loop: } ip += 5; - goto call; + InterpMethodCodeType code_type = cmethod->code_type; + + g_assert (code_type == IMETHOD_CODE_UNKNOWN || + code_type == IMETHOD_CODE_INTERP || + code_type == IMETHOD_CODE_COMPILED); + + if (G_UNLIKELY (code_type == IMETHOD_CODE_UNKNOWN)) { + // FIXME push/pop LMF + MonoMethodSignature *sig = mono_method_signature_internal (cmethod->method); + if (mono_interp_jit_call_supported (cmethod->method, sig)) + code_type = IMETHOD_CODE_COMPILED; + else + code_type = IMETHOD_CODE_INTERP; + cmethod->code_type = code_type; + } + + if (code_type == IMETHOD_CODE_INTERP) { + + goto call; + + } else if (code_type == IMETHOD_CODE_COMPILED) { + frame->state.ip = ip; + error_init_reuse (error); + do_jit_call (context, (stackval*)(locals + return_offset), (stackval*)(locals + call_args_offset), frame, cmethod, error); + if (!is_ok (error)) { + MonoException *call_ex = interp_error_convert_to_exception (frame, error, ip); + THROW_EX (call_ex, ip); + } + + CHECK_RESUME_STATE (context); + } + + MINT_IN_BREAK; } MINT_IN_CASE(MINT_CALLI) { gboolean need_unbox; @@ -8595,6 +8627,31 @@ interp_sufficient_stack (gsize size) return (context->stack_pointer + size) < (context->stack_start + INTERP_STACK_SIZE); } +gboolean +interp_jit_call_can_be_supported (MonoMethod *method, MonoMethodSignature *sig, gboolean is_llvm_only) +{ + if (sig->param_count > 10) + return FALSE; + if (sig->pinvoke) + return FALSE; + if (method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL) + return FALSE; + if (method->iflags & METHOD_IMPL_ATTRIBUTE_INTERNAL_CALL) + return FALSE; + if (!is_llvm_only && method->is_inflated) + return FALSE; + if (method->string_ctor) + return FALSE; + if (method->wrapper_type != MONO_WRAPPER_NONE) + return FALSE; + + if (method->flags & METHOD_ATTRIBUTE_REQSECOBJ) + /* Used to mark methods containing StackCrawlMark locals */ + return FALSE; + + return TRUE; +} + static void interp_cleanup (void) { diff --git a/src/mono/mono/mini/interp/transform.c b/src/mono/mono/mini/interp/transform.c index 655dad4..d4b7a94 100644 --- a/src/mono/mono/mini/interp/transform.c +++ b/src/mono/mono/mini/interp/transform.c @@ -1213,23 +1213,7 @@ mono_interp_jit_call_supported (MonoMethod *method, MonoMethodSignature *sig) { GSList *l; - if (sig->param_count > 10) - return FALSE; - if (sig->pinvoke) - return FALSE; - if (method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL) - return FALSE; - if (method->iflags & METHOD_IMPL_ATTRIBUTE_INTERNAL_CALL) - return FALSE; - if (!mono_llvm_only && method->is_inflated) - return FALSE; - if (method->string_ctor) - return FALSE; - if (method->wrapper_type != MONO_WRAPPER_NONE) - return FALSE; - - if (method->flags & METHOD_ATTRIBUTE_REQSECOBJ) - /* Used to mark methods containing StackCrawlMark locals */ + if (!interp_jit_call_can_be_supported (method, sig, mono_llvm_only)) return FALSE; if (mono_aot_only && m_class_get_image (method->klass)->aot_module && !(method->iflags & METHOD_IMPL_ATTRIBUTE_SYNCHRONIZED)) { @@ -2991,6 +2975,10 @@ interp_inline_method (TransformData *td, MonoMethod *target_method, MonoMethodHe int nargs = csignature->param_count + !!csignature->hasthis; InterpInst *prev_last_ins; + if (header->code_size == 0) + /* IL stripped */ + return FALSE; + if (csignature->is_inflated) generic_context = mono_method_get_context (target_method); else { diff --git a/src/mono/sample/HelloWorld/HelloWorld.csproj b/src/mono/sample/HelloWorld/HelloWorld.csproj index e72a411..cc05249 100644 --- a/src/mono/sample/HelloWorld/HelloWorld.csproj +++ b/src/mono/sample/HelloWorld/HelloWorld.csproj @@ -26,8 +26,8 @@ LibraryFormat="$(_AotLibraryFormat)" Assemblies="@(AotInputAssemblies)" OutputDir="$(PublishDir)" - CollectCompiledMethods="$(StripILCode)" - CompiledMethodsOutputDirectory="$(CompiledMethodsOutputDirectory)" + CollectTrimmingEligibleMethods="$(StripILCode)" + TrimmingEligibleMethodsOutputDirectory="$(TrimmingEligibleMethodsOutputDirectory)" IntermediateOutputPath="$(IntermediateOutputPath)" UseAotDataFile="$(UseAotDataFile)" MibcProfilePath="$(MibcProfilePath)" diff --git a/src/mono/sample/HelloWorld/Makefile b/src/mono/sample/HelloWorld/Makefile index cfb6f07..bb380b0 100644 --- a/src/mono/sample/HelloWorld/Makefile +++ b/src/mono/sample/HelloWorld/Makefile @@ -8,7 +8,7 @@ TARGET_OS?=$(shell . $(TOP)eng/native/init-os-and-arch.sh && echo $${os}) AOT?=false USE_LLVM?=false StripILCode?=false -CompiledMethodsOutputDirectory?= # +TrimmingEligibleMethodsOutputDirectory?= # #MIBC_PROFILE_PATH= @@ -21,7 +21,7 @@ publish: /p:RunAOTCompilation=$(AOT) \ /p:MonoEnableLLVM=$(USE_LLVM) \ /p:StripILCode=$(StripILCode) \ - /p:CompiledMethodsOutputDirectory=$(CompiledMethodsOutputDirectory) \ + /p:TrimmingEligibleMethodsOutputDirectory=$(TrimmingEligibleMethodsOutputDirectory) \ '/p:MibcProfilePath="$(MIBC_PROFILE_PATH)"' \ /bl diff --git a/src/mono/wasm/build/WasmApp.InTree.targets b/src/mono/wasm/build/WasmApp.InTree.targets index 0a2f332..da873ba 100644 --- a/src/mono/wasm/build/WasmApp.InTree.targets +++ b/src/mono/wasm/build/WasmApp.InTree.targets @@ -1,6 +1,7 @@ + diff --git a/src/mono/wasm/build/WasmApp.Native.targets b/src/mono/wasm/build/WasmApp.Native.targets index 5ab27d6..1a894ad 100644 --- a/src/mono/wasm/build/WasmApp.Native.targets +++ b/src/mono/wasm/build/WasmApp.Native.targets @@ -685,12 +685,25 @@ CacheFilePath="$(_AOTCompilerCacheFile)" LLVMDebug="dwarfdebug" LLVMPath="$(EmscriptenUpstreamBinPath)" + CollectTrimmingEligibleMethods="$(WasmStripILAfterAOT)" + TrimmingEligibleMethodsOutputDirectory="$(_WasmIntermediateOutputPath)tokens" IntermediateOutputPath="$(_WasmIntermediateOutputPath)" AotProfilePath="@(AotProfilePath)"> + + + + <_BitcodeFile Include="%(_WasmAssembliesInternal.LlvmBitcodeFile)" /> diff --git a/src/mono/wasm/build/WasmApp.targets b/src/mono/wasm/build/WasmApp.targets index e6e0585..36d6aeb 100644 --- a/src/mono/wasm/build/WasmApp.targets +++ b/src/mono/wasm/build/WasmApp.targets @@ -85,6 +85,8 @@ - AppBundle directly contains user files - AppBundle/_framework contains generated files (dlls, runtime scripts, icu) - AppBundle/_content contains web files from nuget packages (css, js, etc) + - $(WasmStripILAfterAOT) - Set to true to enable trimming away AOT compiled methods body (IL code) + Defaults to false. Public items: - @(WasmExtraFilesToDeploy) - Files to copy to $(WasmAppDir). @@ -146,6 +148,9 @@ .wasm .dll + + false + _framework diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index ffc95c0..9e52322 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -67,7 +67,7 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task /// - LlvmObjectFile (if using LLVM) /// - LlvmBitcodeFile (if using LLVM-only) /// - ExportsFile (used in LibraryMode only) - /// - MethodTokenFile (when using CollectCompiledMethods=true) + /// - MethodTokenFile (when using CollectTrimmingEligibleMethods=true) /// [Output] public ITaskItem[]? CompiledAssemblies { get; set; } @@ -156,12 +156,12 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task /// /// Instructs the AOT compiler to print the list of aot compiled methods /// - public bool CollectCompiledMethods { get; set; } + public bool CollectTrimmingEligibleMethods { get; set; } /// - /// Directory to store the aot output when using switch compiled-methods-outfile + /// Directory to store the aot output when using switch trimming-eligible-methods-outfile /// - public string? CompiledMethodsOutputDirectory { get; set; } + public string? TrimmingEligibleMethodsOutputDirectory { get; set; } /// /// File to use for profile-guided optimization, *only* the methods described in the file will be AOT compiled. @@ -453,14 +453,14 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task throw new LogAsErrorException($"Could not find {fullPath} to AOT"); } - if (CollectCompiledMethods) + if (CollectTrimmingEligibleMethods) { - if (string.IsNullOrEmpty(CompiledMethodsOutputDirectory)) - throw new LogAsErrorException($"{nameof(CompiledMethodsOutputDirectory)} is empty. When {nameof(CollectCompiledMethods)} is set to true, the user needs to provide a directory for {nameof(CompiledMethodsOutputDirectory)}."); + if (string.IsNullOrEmpty(TrimmingEligibleMethodsOutputDirectory)) + throw new LogAsErrorException($"{nameof(TrimmingEligibleMethodsOutputDirectory)} is empty. When {nameof(CollectTrimmingEligibleMethods)} is set to true, the user needs to provide a directory for {nameof(TrimmingEligibleMethodsOutputDirectory)}."); - if (!Directory.Exists(CompiledMethodsOutputDirectory)) + if (!Directory.Exists(TrimmingEligibleMethodsOutputDirectory)) { - Directory.CreateDirectory(CompiledMethodsOutputDirectory); + Directory.CreateDirectory(TrimmingEligibleMethodsOutputDirectory); } } @@ -744,20 +744,20 @@ public class MonoAOTCompiler : Microsoft.Build.Utilities.Task aotArgs.Add("dedup-skip"); } - if (CollectCompiledMethods) + if (CollectTrimmingEligibleMethods) { string assemblyName = assemblyFilename.Replace(".", "_"); string outputFileName = assemblyName + "_compiled_methods.txt"; string outputFilePath; - if (string.IsNullOrEmpty(CompiledMethodsOutputDirectory)) + if (string.IsNullOrEmpty(TrimmingEligibleMethodsOutputDirectory)) { outputFilePath = outputFileName; } else { - outputFilePath = Path.Combine(CompiledMethodsOutputDirectory, outputFileName); + outputFilePath = Path.Combine(TrimmingEligibleMethodsOutputDirectory, outputFileName); } - aotArgs.Add($"compiled-methods-outfile={outputFilePath}"); + aotArgs.Add($"trimming-eligible-methods-outfile={outputFilePath}"); aotAssembly.SetMetadata("MethodTokenFile", outputFilePath); } diff --git a/src/tasks/MonoTargetsTasks/ILStrip/ILStrip.cs b/src/tasks/MonoTargetsTasks/ILStrip/ILStrip.cs index 34c81b6..719064d 100644 --- a/src/tasks/MonoTargetsTasks/ILStrip/ILStrip.cs +++ b/src/tasks/MonoTargetsTasks/ILStrip/ILStrip.cs @@ -116,6 +116,7 @@ public class ILStrip : Microsoft.Build.Utilities.Task private bool TrimMethods(ITaskItem assemblyItem) { + string assemblyFilePathArg = assemblyItem.ItemSpec; string methodTokenFile = assemblyItem.GetMetadata("MethodTokenFile"); if (string.IsNullOrEmpty(methodTokenFile)) { @@ -165,7 +166,7 @@ public class ILStrip : Microsoft.Build.Utilities.Task if (isTrimmed) { - AddItemToTrimmedList(assemblyFilePath, trimmedAssemblyFilePath); + AddItemToTrimmedList(assemblyFilePathArg, trimmedAssemblyFilePath); } return true; @@ -252,6 +253,8 @@ public class ILStrip : Microsoft.Build.Utilities.Task int methodSize = ComputeMethodSize(peReader, rva); int actualLoc = ComputeMethodHash(peReader, rva); int headerSize = ComputeMethodHeaderSize(memStream, actualLoc); + if (headerSize == 1) //Set code size to zero for TinyFormat + SetCodeSizeToZeroForTiny(ref memStream, actualLoc); ZeroOutMethodBody(ref memStream, methodSize, actualLoc, headerSize); } else if (count < 0) @@ -282,6 +285,13 @@ public class ILStrip : Microsoft.Build.Utilities.Task return (headerFlag == 2 ? 1 : 4); } + private static void SetCodeSizeToZeroForTiny(ref MemoryStream memStream, int actualLoc) + { + memStream.Position = actualLoc; + byte[] header = {0b10}; + memStream.Write(header, 0, 1); + } + private static void ZeroOutMethodBody(ref MemoryStream memStream, int methodSize, int actualLoc, int headerSize) { memStream.Position = actualLoc + headerSize;