From 9c8962e087225da3d87e374afc3b308549174de2 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Wed, 1 Sep 2021 12:16:43 +0200
Subject: [PATCH] [release/6.0] [mini] Dynamically allocate a buffer for large
 runtime invoke results (#58364)

* [mini] Dynamically allocate a buffer for large runtime invoke results

If the return type is a struct that's bigger than our buffer, malloc a
buffer for it instead of using a fixed-size stack buffer

* Throw nullbyrefreturn exception for non-LLVM dyn invoke

Fixes various tests in InvokeRefReturnNetcoreTests

Co-authored-by: Aleksey Kliger <alklig@microsoft.com>
---
 src/mono/mono/mini/mini-runtime.c | 75 +++++++++++++++++++++++++++----
 1 file changed, 67 insertions(+), 8 deletions(-)

diff --git a/src/mono/mono/mini/mini-runtime.c b/src/mono/mono/mini/mini-runtime.c
index 2ca296b1fd6..2abfe01772b 100644
--- a/src/mono/mono/mini/mini-runtime.c
+++ b/src/mono/mono/mini/mini-runtime.c
@@ -2999,6 +2999,8 @@ typedef struct {
 	gpointer *wrapper_arg;
 } RuntimeInvokeInfo;
 
+#define MONO_SIZEOF_DYN_CALL_RET_BUF 256
+
 static RuntimeInvokeInfo*
 create_runtime_invoke_info (MonoMethod *method, gpointer compiled_method, gboolean callee_gsharedvt, gboolean use_interp, MonoError *error)
 {
@@ -3157,8 +3159,9 @@ mono_llvmonly_runtime_invoke (MonoMethod *method, RuntimeInvokeInfo *info, void
 {
 	MonoMethodSignature *sig = info->sig;
 	MonoObject *(*runtime_invoke) (MonoObject *this_obj, void **params, MonoObject **exc, void* compiled_method);
+	gboolean retval_malloc = FALSE;
 	gpointer retval_ptr;
-	guint8 retval [256];
+	guint8 retval [MONO_SIZEOF_DYN_CALL_RET_BUF];
 	int i, pindex;
 
 	error_init (error);
@@ -3185,7 +3188,21 @@ mono_llvmonly_runtime_invoke (MonoMethod *method, RuntimeInvokeInfo *info, void
 	if (sig->hasthis)
 		args [pindex ++] = &obj;
 	if (sig->ret->type != MONO_TYPE_VOID) {
-		retval_ptr = &retval;
+		if (info->ret_box_class && !sig->ret->byref &&
+		    (sig->ret->type == MONO_TYPE_VALUETYPE ||
+		     (sig->ret->type == MONO_TYPE_GENERICINST && !MONO_TYPE_IS_REFERENCE (sig->ret)))) {
+			// if the return type is a struct and its too big for the stack buffer, malloc instead
+			MonoClass *ret_klass = mono_class_from_mono_type_internal (sig->ret);
+			g_assert (!mono_class_has_failure (ret_klass));
+			int32_t inst_size = mono_class_instance_size (ret_klass);
+			if (inst_size > MONO_SIZEOF_DYN_CALL_RET_BUF) {
+				retval_malloc = TRUE;
+				retval_ptr = g_new0 (guint8, inst_size);
+				g_assert (retval_ptr);
+			}
+		}
+		if (!retval_malloc)
+			retval_ptr = &retval;
 		args [pindex ++] = &retval_ptr;
 	}
 	for (i = 0; i < sig->param_count; ++i) {
@@ -3235,7 +3252,10 @@ mono_llvmonly_runtime_invoke (MonoMethod *method, RuntimeInvokeInfo *info, void
 			if (sig->ret->byref) {
 				return mono_value_box_checked (info->ret_box_class, *(gpointer*)retval, error);
 			} else {
-				return mono_value_box_checked (info->ret_box_class, retval, error);
+				MonoObject *ret = mono_value_box_checked (info->ret_box_class, retval_ptr, error);
+				if (retval_malloc)
+					g_free (retval_ptr);
+				return ret;
 			}
 		} else {
 			if (sig->ret->byref)
@@ -3398,7 +3418,25 @@ mono_jit_runtime_invoke (MonoMethod *method, void *obj, void **params, MonoObjec
 		gpointer *args;
 		int i, pindex, buf_size;
 		guint8 *buf;
-		guint8 retval [256];
+		guint8 retbuf [MONO_SIZEOF_DYN_CALL_RET_BUF];
+		guint8 *retval = &retbuf[0];
+		gboolean retval_malloc = FALSE;
+
+		/* if the return value is too big, put it in a dynamically allocated temporary */
+		if (info->ret_box_class && !sig->ret->byref &&
+		    (sig->ret->type == MONO_TYPE_VALUETYPE ||
+		     (sig->ret->type == MONO_TYPE_GENERICINST && !MONO_TYPE_IS_REFERENCE (sig->ret)))) {
+			// if the return type is a struct and its too big for the stack buffer, malloc instead
+			MonoClass *ret_klass = mono_class_from_mono_type_internal (sig->ret);
+			g_assert (!mono_class_has_failure (ret_klass));
+			int32_t inst_size = mono_class_instance_size (ret_klass);
+			if (inst_size > MONO_SIZEOF_DYN_CALL_RET_BUF) {
+				retval_malloc = TRUE;
+				retval = g_new0 (guint8, inst_size);
+				g_assert (retval);
+			}
+		}
+
 
 		/* Convert the arguments to the format expected by start_dyn_call () */
 		args = (void **)g_alloca ((sig->param_count + sig->hasthis) * sizeof (gpointer));
@@ -3434,10 +3472,31 @@ mono_jit_runtime_invoke (MonoMethod *method, void *obj, void **params, MonoObjec
 			return NULL;
 		}
 
-		if (info->ret_box_class)
-			return mono_value_box_checked (info->ret_box_class, retval, error);
-		else
-			return *(MonoObject**)retval;
+		if (sig->ret->byref) {
+			if (*(gpointer*)retval == NULL) {
+				MonoClass *klass = mono_class_get_nullbyrefreturn_ex_class ();
+				MonoObject *ex = mono_object_new_checked (klass, error);
+				mono_error_assert_ok (error);
+				mono_error_set_exception_instance (error, (MonoException*)ex);
+				return NULL;
+			}
+		}
+
+		if (info->ret_box_class) {
+			if (sig->ret->byref) {
+				return mono_value_box_checked (info->ret_box_class, *(gpointer*)retval, error);
+			} else {
+				MonoObject *boxed_ret = mono_value_box_checked (info->ret_box_class, retval, error);
+				if (retval_malloc)
+					g_free (retval);
+				return boxed_ret;
+			}
+		} else {
+			if (sig->ret->byref)
+				return **(MonoObject***)retval;
+			else
+				return *(MonoObject**)retval;
+		}
 	}
 #endif
 
-- 
2.34.1