[runtime] Handle null byref returns in runtime invokes. (#40071)

* [runtime] Handle null byref returns in runtime invokes.

Fixes https://github.com/dotnet/runtime/issues/39380.

* Reenable fixed test

Co-authored-by: Alexander Köplinger <alex.koeplinger@outlook.com>

diff --git a/src/libraries/System.Runtime/tests/System/Reflection/InvokeRefReturn.cs b/src/libraries/System.Runtime/tests/System/Reflection/InvokeRefReturn.cs
index ee28dd5..885e179 100644 (file)
--- a/src/libraries/System.Runtime/tests/System/Reflection/InvokeRefReturn.cs
+++ b/src/libraries/System.Runtime/tests/System/Reflection/InvokeRefReturn.cs
@@ -59,7 +59,6 @@ namespace System.Reflection.Tests
         }
 
         [Fact]
-        [ActiveIssue("https://github.com/dotnet/runtime/issues/39380", TestPlatforms.Browser)]
         public static unsafe void TestNullRefReturnOfPointer()
         {
             TestClassIntPointer tc = new TestClassIntPointer(null);

diff --git a/src/mono/mono/metadata/object.c b/src/mono/mono/metadata/object.c
index 71e7377..5f0baa6 100644 (file)
--- a/src/mono/mono/metadata/object.c
+++ b/src/mono/mono/metadata/object.c
@@ -5737,8 +5737,13 @@ mono_runtime_try_invoke_array (MonoMethod *method, void *obj, MonoArray *params,
                                mono_error_assert_ok (error);
                        MONO_STATIC_POINTER_INIT_END (MonoMethod, box_method)
 
-                       g_assert (res->vtable->klass == mono_defaults.int_class);
-                       box_args [0] = ((MonoIntPtr*)res)->m_value;
+                       if (res) {
+                               g_assert (res->vtable->klass == mono_defaults.int_class);
+                               box_args [0] = ((MonoIntPtr*)res)->m_value;
+                       } else {
+                               g_assert (sig->ret->byref);
+                               box_args [0] = NULL;
+                       }
                        if (sig->ret->byref) {
                                // byref is already unboxed by the invoke code
                                MonoType *tmpret = mono_metadata_type_dup (NULL, sig->ret);