[x86/Linux] Pass return buffer on reverse P/Invoke (dotnet/coreclr#10226)

* Pass return buffer on reverse P/Invoke

* Use PLATFORM_UNIX and track m_cbRetPop locally

* Declare cbRetPop UINT16 instead of int

Commit migrated from https://github.com/dotnet/coreclr/commit/bcc0a1d528b78f5e05bf7973090f33ef2403e685

diff --git a/src/coreclr/src/vm/dllimportcallback.cpp b/src/coreclr/src/vm/dllimportcallback.cpp
index b72bc16..c70d52d 100644 (file)
--- a/src/coreclr/src/vm/dllimportcallback.cpp
+++ b/src/coreclr/src/vm/dllimportcallback.cpp
@@ -1379,6 +1379,7 @@ VOID UMThunkMarshInfo::RunTimeInit()
 #if defined(_TARGET_X86_)
     MetaSig sig(pMD);
     int numRegistersUsed = 0;
+    UINT16 cbRetPop = 0;
 
     //
     // m_cbStackArgSize represents the number of arg bytes for the MANAGED signature
@@ -1386,6 +1387,17 @@ VOID UMThunkMarshInfo::RunTimeInit()
     m_cbStackArgSize = 0;
 
     int offs = 0;
+
+#ifdef UNIX_X86_ABI
+    if (HasRetBuffArgUnmanagedFixup(&sig))
+    {
+        // callee should pop retbuf
+        numRegistersUsed += 1;
+        offs += STACK_ELEM_SIZE;
+        cbRetPop += STACK_ELEM_SIZE;
+    }
+#endif // UNIX_X86_ABI
+
     for (UINT i = 0 ; i < sig.NumFixedArgs(); i++)
     {
         TypeHandle thValueType;
@@ -1408,16 +1420,14 @@ VOID UMThunkMarshInfo::RunTimeInit()
         new (&sigInfo) PInvokeStaticSigInfo(pMD);
     else
         new (&sigInfo) PInvokeStaticSigInfo(GetSignature(), GetModule());
-
     if (sigInfo.GetCallConv() == pmCallConvCdecl)
     {
-        // caller pop
-        m_cbRetPop = 0;
+        m_cbRetPop = cbRetPop;
     }
     else
     {
-        // callee pop
-        m_cbRetPop = static_cast<UINT16>(m_cbActualArgSize);
+        // For all the other calling convention except cdecl, callee pops the stack arguments
+        m_cbRetPop = cbRetPop + static_cast<UINT16>(m_cbActualArgSize);
     }
 #else // _TARGET_X86_
     //
@@ -1467,6 +1477,16 @@ VOID UMThunkMarshInfo::SetupArguments(char *pSrc, ArgumentRegisters *pArgRegs, c
 
     int numRegistersUsed = 0;
 
+#ifdef UNIX_X86_ABI
+    if (HasRetBuffArgUnmanagedFixup(&sig))
+    {
+        // Pass retbuf via Ecx
+        numRegistersUsed += 1;
+        pArgRegs->Ecx = *((UINT32 *)pCurSrc);
+        pCurSrc += STACK_ELEM_SIZE;
+    }
+#endif // UNIX_X86_ABI
+
     for (UINT i = 0 ; i < sig.NumFixedArgs(); i++)
     {
         TypeHandle thValueType;