[release/6.0] [mini] Dynamically allocate a buffer for large runtime invoke results...

* [mini] Dynamically allocate a buffer for large runtime invoke results

If the return type is a struct that's bigger than our buffer, malloc a
buffer for it instead of using a fixed-size stack buffer

* Throw nullbyrefreturn exception for non-LLVM dyn invoke

Fixes various tests in InvokeRefReturnNetcoreTests

Co-authored-by: Aleksey Kliger <alklig@microsoft.com>

diff --git a/src/mono/mono/mini/mini-runtime.c b/src/mono/mono/mini/mini-runtime.c
index 2ca296b..2abfe01 100644 (file)
--- a/src/mono/mono/mini/mini-runtime.c
+++ b/src/mono/mono/mini/mini-runtime.c
@@ -2999,6 +2999,8 @@ typedef struct {
        gpointer *wrapper_arg;
 } RuntimeInvokeInfo;
 
+#define MONO_SIZEOF_DYN_CALL_RET_BUF 256
+
 static RuntimeInvokeInfo*
 create_runtime_invoke_info (MonoMethod *method, gpointer compiled_method, gboolean callee_gsharedvt, gboolean use_interp, MonoError *error)
 {
@@ -3157,8 +3159,9 @@ mono_llvmonly_runtime_invoke (MonoMethod *method, RuntimeInvokeInfo *info, void
 {
        MonoMethodSignature *sig = info->sig;
        MonoObject *(*runtime_invoke) (MonoObject *this_obj, void **params, MonoObject **exc, void* compiled_method);
+       gboolean retval_malloc = FALSE;
        gpointer retval_ptr;
-       guint8 retval [256];
+       guint8 retval [MONO_SIZEOF_DYN_CALL_RET_BUF];
        int i, pindex;
 
        error_init (error);
@@ -3185,7 +3188,21 @@ mono_llvmonly_runtime_invoke (MonoMethod *method, RuntimeInvokeInfo *info, void
        if (sig->hasthis)
                args [pindex ++] = &obj;
        if (sig->ret->type != MONO_TYPE_VOID) {
-               retval_ptr = &retval;
+               if (info->ret_box_class && !sig->ret->byref &&
+                   (sig->ret->type == MONO_TYPE_VALUETYPE ||
+                    (sig->ret->type == MONO_TYPE_GENERICINST && !MONO_TYPE_IS_REFERENCE (sig->ret)))) {
+                       // if the return type is a struct and its too big for the stack buffer, malloc instead
+                       MonoClass *ret_klass = mono_class_from_mono_type_internal (sig->ret);
+                       g_assert (!mono_class_has_failure (ret_klass));
+                       int32_t inst_size = mono_class_instance_size (ret_klass);
+                       if (inst_size > MONO_SIZEOF_DYN_CALL_RET_BUF) {
+                               retval_malloc = TRUE;
+                               retval_ptr = g_new0 (guint8, inst_size);
+                               g_assert (retval_ptr);
+                       }
+               }
+               if (!retval_malloc)
+                       retval_ptr = &retval;
                args [pindex ++] = &retval_ptr;
        }
        for (i = 0; i < sig->param_count; ++i) {
@@ -3235,7 +3252,10 @@ mono_llvmonly_runtime_invoke (MonoMethod *method, RuntimeInvokeInfo *info, void
                        if (sig->ret->byref) {
                                return mono_value_box_checked (info->ret_box_class, *(gpointer*)retval, error);
                        } else {
-                               return mono_value_box_checked (info->ret_box_class, retval, error);
+                               MonoObject *ret = mono_value_box_checked (info->ret_box_class, retval_ptr, error);
+                               if (retval_malloc)
+                                       g_free (retval_ptr);
+                               return ret;
                        }
                } else {
                        if (sig->ret->byref)
@@ -3398,7 +3418,25 @@ mono_jit_runtime_invoke (MonoMethod *method, void *obj, void **params, MonoObjec
                gpointer *args;
                int i, pindex, buf_size;
                guint8 *buf;
-               guint8 retval [256];
+               guint8 retbuf [MONO_SIZEOF_DYN_CALL_RET_BUF];
+               guint8 *retval = &retbuf[0];
+               gboolean retval_malloc = FALSE;
+
+               /* if the return value is too big, put it in a dynamically allocated temporary */
+               if (info->ret_box_class && !sig->ret->byref &&
+                   (sig->ret->type == MONO_TYPE_VALUETYPE ||
+                    (sig->ret->type == MONO_TYPE_GENERICINST && !MONO_TYPE_IS_REFERENCE (sig->ret)))) {
+                       // if the return type is a struct and its too big for the stack buffer, malloc instead
+                       MonoClass *ret_klass = mono_class_from_mono_type_internal (sig->ret);
+                       g_assert (!mono_class_has_failure (ret_klass));
+                       int32_t inst_size = mono_class_instance_size (ret_klass);
+                       if (inst_size > MONO_SIZEOF_DYN_CALL_RET_BUF) {
+                               retval_malloc = TRUE;
+                               retval = g_new0 (guint8, inst_size);
+                               g_assert (retval);
+                       }
+               }
+
 
                /* Convert the arguments to the format expected by start_dyn_call () */
                args = (void **)g_alloca ((sig->param_count + sig->hasthis) * sizeof (gpointer));
@@ -3434,10 +3472,31 @@ mono_jit_runtime_invoke (MonoMethod *method, void *obj, void **params, MonoObjec
                        return NULL;
                }
 
-               if (info->ret_box_class)
-                       return mono_value_box_checked (info->ret_box_class, retval, error);
-               else
-                       return *(MonoObject**)retval;
+               if (sig->ret->byref) {
+                       if (*(gpointer*)retval == NULL) {
+                               MonoClass *klass = mono_class_get_nullbyrefreturn_ex_class ();
+                               MonoObject *ex = mono_object_new_checked (klass, error);
+                               mono_error_assert_ok (error);
+                               mono_error_set_exception_instance (error, (MonoException*)ex);
+                               return NULL;
+                       }
+               }
+
+               if (info->ret_box_class) {
+                       if (sig->ret->byref) {
+                               return mono_value_box_checked (info->ret_box_class, *(gpointer*)retval, error);
+                       } else {
+                               MonoObject *boxed_ret = mono_value_box_checked (info->ret_box_class, retval, error);
+                               if (retval_malloc)
+                                       g_free (retval);
+                               return boxed_ret;
+                       }
+               } else {
+                       if (sig->ret->byref)
+                               return **(MonoObject***)retval;
+                       else
+                               return *(MonoObject**)retval;
+               }
        }
 #endif