Fix Pinvoke IL Stubs to emit correct return type in signature of calli instruction

Commit migrated from https://github.com/dotnet/coreclr/commit/6d5ef671e0eb35c813955a4532f95e7a03843d35

diff --git a/src/coreclr/src/vm/dllimport.cpp b/src/coreclr/src/vm/dllimport.cpp
index d33b817..7cd08a2 100644 (file)
--- a/src/coreclr/src/vm/dllimport.cpp
+++ b/src/coreclr/src/vm/dllimport.cpp
@@ -4194,9 +4194,17 @@ static void CreateNDirectStubWorker(StubState*         pss,
     bool fHasCopyCtorArgs = false;
     bool fStubNeedsCOM = SF_IsCOMStub(dwStubFlags);
 
+#if defined(_TARGET_X86_) || defined(_TARGET_ARM_)
+    // JIT32 has problems in generating code for pinvoke ILStubs which do a return in return buffer.
+    // Therefore instead we change the signature of calli to return void and make the return buffer as first
+    // argument. This matches the ABI i.e. return buffer is passed as first arg. So native target will get the
+    // return buffer in correct register.
     // The return structure secret arg comes first, however byvalue return is processed at
     // the end because it could be the HRESULT-swapped argument which always comes last.
     bool fMarshalReturnValueFirst = !SF_IsHRESULTSwapping(dwStubFlags) && HasRetBuffArg(&msig);
+#else
+    bool fMarshalReturnValueFirst = false;
+#endif
 
     if (fMarshalReturnValueFirst)
     {

diff --git a/src/coreclr/src/vm/ilmarshalers.h b/src/coreclr/src/vm/ilmarshalers.h
index 1730eab..8043cdb 100644 (file)
--- a/src/coreclr/src/vm/ilmarshalers.h
+++ b/src/coreclr/src/vm/ilmarshalers.h
@@ -600,7 +600,7 @@ public:
                 nativeSize = wNativeSize;
             }
 
-#if !defined(_TARGET_ARM_) && !(defined(UNIX_AMD64_ABI) && defined(FEATURE_UNIX_AMD64_STRUCT_PASSING))
+#if defined(_TARGET_X86_)
             switch (nativeSize)
             {
                 case 1: typ = ELEMENT_TYPE_U1; break;