[mono][exceptions] Don't attempt to catch StackOverflowException (#87601)

* [mono][exceptions] Don't attempt to catch StackOverflowException

* [mono][exceptions] Don't invoke ToString when logging StackOveflowException

Invoking ToString can trigger method compilations and put pressure on the stack space. Instead, build the output directly from the `exc->trace_ips` with already existing methods.

* Remove unused variable

diff --git a/src/mono/mono/metadata/exception.c b/src/mono/mono/metadata/exception.c
index 0095d8766b9492204ca5ebbf3cd9d3bf95b763f0..89c9faede2efb1315f72e8417095cbc66cc141c0 100644 (file)
--- a/src/mono/mono/metadata/exception.c
+++ b/src/mono/mono/metadata/exception.c
@@ -1219,27 +1219,37 @@ mono_invoke_unhandled_exception_hook (MonoObject *exc)
                unhandled_exception_hook (exc, unhandled_exception_hook_data);
        } else {
                ERROR_DECL (inner_error);
-               MonoObject *other = NULL;
-               MonoString *str = mono_object_try_to_string (exc, &other, inner_error);
                char *msg = NULL;
 
-               if (str && is_ok (inner_error)) {
-                       msg = mono_string_to_utf8_checked_internal (str, inner_error);
-                       if (!is_ok (inner_error)) {
-                               msg = g_strdup_printf ("Nested exception while formatting original exception");
-                               mono_error_cleanup (inner_error);
-                       }
-               } else if (other) {
-                       char *original_backtrace = mono_exception_get_managed_backtrace ((MonoException*)exc);
-                       char *nested_backtrace = mono_exception_get_managed_backtrace ((MonoException*)other);
-
-                       msg = g_strdup_printf ("Nested exception detected.\nOriginal Exception: %s\nNested exception:%s\n",
-                               original_backtrace, nested_backtrace);
+               if (exc == (MonoObject*)mono_domain_get ()->stack_overflow_ex) {
+                       // Build stack trace directly instead of calling ToString so we don't put
+                       // additional pressure on the limited stack
+                       char *backtrace = mono_exception_get_managed_backtrace ((MonoException*)exc);
 
-                       g_free (original_backtrace);
-                       g_free (nested_backtrace);
+                       msg = g_strdup_printf ("System.StackOverflowException: The requested operation caused a stack overflow.\n%s\n",
+                               backtrace);
                } else {
-                       msg = g_strdup ("Nested exception trying to figure out what went wrong");
+                       MonoObject *other = NULL;
+                       MonoString *str = mono_object_try_to_string (exc, &other, inner_error);
+
+                       if (str && is_ok (inner_error)) {
+                               msg = mono_string_to_utf8_checked_internal (str, inner_error);
+                               if (!is_ok (inner_error)) {
+                                       msg = g_strdup_printf ("Nested exception while formatting original exception");
+                                       mono_error_cleanup (inner_error);
+                               }
+                       } else if (other) {
+                               char *original_backtrace = mono_exception_get_managed_backtrace ((MonoException*)exc);
+                               char *nested_backtrace = mono_exception_get_managed_backtrace ((MonoException*)other);
+
+                               msg = g_strdup_printf ("Nested exception detected.\nOriginal Exception: %s\nNested exception:%s\n",
+                                       original_backtrace, nested_backtrace);
+
+                               g_free (original_backtrace);
+                               g_free (nested_backtrace);
+                       } else {
+                               msg = g_strdup ("Nested exception trying to figure out what went wrong");
+                       }
                }
                mono_runtime_printf_err ("[ERROR] FATAL UNHANDLED EXCEPTION: %s", msg);
                g_free (msg);

diff --git a/src/mono/mono/mini/mini-exceptions.c b/src/mono/mono/mini/mini-exceptions.c
index c75c124b7d400bbd6e6aafadb31a291337325d1b..1b6f17555dbd594825f4e5ee3ee0bf562cc13010 100644 (file)
--- a/src/mono/mono/mini/mini-exceptions.c
+++ b/src/mono/mono/mini/mini-exceptions.c
@@ -1839,7 +1839,7 @@ handle_exception_first_pass (MonoContext *ctx, MonoObject *obj, gint32 *out_filt
 
        while (1) {
                MonoContext new_ctx;
-               guint32 free_stack, clause_index_start = 0;
+               guint32 clause_index_start = 0;
                gboolean unwind_res = TRUE;
 
                StackFrameInfo frame;
@@ -1907,12 +1907,6 @@ handle_exception_first_pass (MonoContext *ctx, MonoObject *obj, gint32 *out_filt
                if (method->dynamic)
                        dynamic_methods = g_slist_prepend (dynamic_methods, method);
 
-               if (stack_overflow) {
-                       free_stack = (guint32)((guint8*)(MONO_CONTEXT_GET_SP (ctx)) - (guint8*)(MONO_CONTEXT_GET_SP (&initial_ctx)));
-               } else {
-                       free_stack = 0xffffff;
-               }
-
                if (method->wrapper_type == MONO_WRAPPER_NATIVE_TO_MANAGED && ftnptr_eh_callback) {
                        result = MONO_FIRST_PASS_CALLBACK_TO_NATIVE;
                }
@@ -1921,11 +1915,8 @@ handle_exception_first_pass (MonoContext *ctx, MonoObject *obj, gint32 *out_filt
                        MonoJitExceptionInfo *ei = &ji->clauses [i];
                        gboolean filtered = FALSE;
 
-                       /*
-                        * During stack overflow, wait till the unwinding frees some stack
-                        * space before running handlers/finalizers.
-                        */
-                       if (free_stack <= (64 * 1024))
+                       // StackOverflowException shouldn't be caught
+                       if (stack_overflow)
                                continue;
 
                        if (is_address_protected (ji, ei, ip)) {
@@ -2085,7 +2076,6 @@ mono_handle_exception_internal (MonoContext *ctx, MonoObject *obj, gboolean resu
        MonoLMF *lmf = mono_get_lmf ();
        MonoException *mono_ex;
        gboolean stack_overflow = FALSE;
-       MonoContext initial_ctx;
        MonoMethod *method;
        // int frame_count = 0; // used for debugging
        gint32 filter_idx, first_filter_idx = 0;
@@ -2270,13 +2260,11 @@ mono_handle_exception_internal (MonoContext *ctx, MonoObject *obj, gboolean resu
        if (out_ji)
                *out_ji = NULL;
        filter_idx = 0;
-       initial_ctx = *ctx;
 
        unwinder_init (&unwinder);
 
        while (1) {
                MonoContext new_ctx;
-               guint32 free_stack;
                int clause_index_start = 0;
                gboolean unwind_res = TRUE;
                StackFrameInfo frame;
@@ -2334,12 +2322,6 @@ mono_handle_exception_internal (MonoContext *ctx, MonoObject *obj, gboolean resu
                // frame_count ++;
                // printf ("[%d] %s.\n", frame_count, mono_method_full_name (method, TRUE));
 
-               if (stack_overflow) {
-                       free_stack = (guint32)((guint8*)(MONO_CONTEXT_GET_SP (ctx)) - (guint8*)(MONO_CONTEXT_GET_SP (&initial_ctx)));
-               } else {
-                       free_stack = 0xffffff;
-               }
-
                if (method->wrapper_type == MONO_WRAPPER_NATIVE_TO_MANAGED && ftnptr_eh_callback) {
                        MonoGCHandle handle = mono_gchandle_new_internal (obj, FALSE);
                        MONO_STACKDATA (stackptr);
@@ -2354,11 +2336,8 @@ mono_handle_exception_internal (MonoContext *ctx, MonoObject *obj, gboolean resu
                        MonoJitExceptionInfo *ei = &ji->clauses [i];
                        gboolean filtered = FALSE;
 
-                       /*
-                        * During stack overflow, wait till the unwinding frees some stack
-                        * space before running handlers/finalizers.
-                        */
-                       if (free_stack <= (64 * 1024))
+                       // StackOverflowException shouldn't be caught
+                       if (stack_overflow)
                                continue;
 
                        if (is_address_protected (ji, ei, ip)) {