[x86/Linux] Fix IL_STUB_PInvoke with RetBuf (#10144)

* [x86/Linux] Fix IL_STUB_PInvoke with RetBuf

Fix calling convention and IL_STUB_PInvoke for native functions
which was problematic for native functions that has
a RetBuf argument(struct size <= 8) on x86/Linux.

Fix #10027

diff --git a/src/vm/callingconvention.h b/src/vm/callingconvention.h
index c9a27c2371f8063b76f6b64b84124cbb6885cd47..cde2ba465a1e78f2609988946d048bd30591e48e 100644 (file)
--- a/src/vm/callingconvention.h
+++ b/src/vm/callingconvention.h
@@ -1701,6 +1701,17 @@ inline BOOL HasRetBuffArg(MetaSig * pSig)
     return argit.HasRetBuffArg();
 }
 
+#ifdef UNIX_X86_ABI
+// For UNIX_X86_ABI and unmanaged function, we always need RetBuf if the return type is VALUETYPE
+inline BOOL HasRetBuffArgUnmanagedFixup(MetaSig * pSig)
+{
+    WRAPPER_NO_CONTRACT;
+    // We cannot just pSig->GetReturnType() here since it will return ELEMENT_TYPE_VALUETYPE for enums
+    CorElementType type = pSig->GetRetTypeHandleThrowing().GetVerifierCorElementType();
+    return type == ELEMENT_TYPE_VALUETYPE;
+}
+#endif
+
 inline BOOL IsRetBuffPassedAsFirstArg()
 {
     WRAPPER_NO_CONTRACT;

diff --git a/src/vm/dllimport.cpp b/src/vm/dllimport.cpp
index ba56831bd673074f6a9a9060ea1b34be8b34dd5a..c0a73e8b809ff09f1ff1f836114348401bece526 100644 (file)
--- a/src/vm/dllimport.cpp
+++ b/src/vm/dllimport.cpp
@@ -3684,8 +3684,15 @@ static void CreateNDirectStubWorker(StubState*         pss,
         // return buffer in correct register.
         // The return structure secret arg comes first, however byvalue return is processed at
         // the end because it could be the HRESULT-swapped argument which always comes last.
+
+#ifdef UNIX_X86_ABI
+        // For functions with value type class, managed and unmanaged calling convention differ
+        fMarshalReturnValueFirst = HasRetBuffArgUnmanagedFixup(&msig);
+#else // UNIX_X86_ABI
         fMarshalReturnValueFirst = HasRetBuffArg(&msig);
-#endif
+#endif // UNIX_X86_ABI
+
+#endif // defined(_TARGET_X86_) || defined(_TARGET_ARM_)
 
     }
     

diff --git a/src/vm/i386/cgencpu.h b/src/vm/i386/cgencpu.h
index ac239533f7e552e2e9f2199daea2bcaedd2b64c0..ff76d992fc2eef7333fcd5ecb880ee6351fb5dec 100644 (file)
--- a/src/vm/i386/cgencpu.h
+++ b/src/vm/i386/cgencpu.h
@@ -483,10 +483,15 @@ inline BOOL IsUnmanagedValueTypeReturnedByRef(UINT sizeofvaluetype)
 {
     LIMITED_METHOD_CONTRACT;
 
+#ifndef UNIX_X86_ABI
     // odd-sized small structures are not 
     //  enregistered e.g. struct { char a,b,c; }
     return (sizeofvaluetype > 8) ||
         (sizeofvaluetype & (sizeofvaluetype - 1)); // check that the size is power of two
+#else
+    // For UNIX_X86_ABI, we always return the value type by reference regardless of its size
+    return true;
+#endif
 }
 
 #include <pshpack1.h>

diff --git a/src/vm/ilmarshalers.h b/src/vm/ilmarshalers.h
index 7c7f9a65535ceb6d0943c5376465dda71b712aeb..5ac5e4124286d75592fe657d55922bdc58d83556 100644 (file)
--- a/src/vm/ilmarshalers.h
+++ b/src/vm/ilmarshalers.h
@@ -607,12 +607,15 @@ public:
 
             // for X86 and AMD64-Windows we bash the return type from struct to U1, U2, U4 or U8
             // and use byrefNativeReturn for all other structs.
+            // for UNIX_X86_ABI, we always need a return buffer argument for any size of structs.
             switch (nativeSize)
             {
+#ifndef UNIX_X86_ABI
                 case 1: typ = ELEMENT_TYPE_U1; break;
                 case 2: typ = ELEMENT_TYPE_U2; break;
                 case 4: typ = ELEMENT_TYPE_U4; break;
                 case 8: typ = ELEMENT_TYPE_U8; break;
+#endif
                 default: byrefNativeReturn = true; break;
             }
 #endif