* [x86/Linux] Fix IL_STUB_PInvoke with RetBuf
Fix calling convention and IL_STUB_PInvoke for native functions
which was problematic for native functions that has
a RetBuf argument(struct size <= 8) on x86/Linux.
Fix dotnet/coreclr#10027
Commit migrated from https://github.com/dotnet/coreclr/commit/
9422f47c5de802d877c08f9f5118471b375239b1
return argit.HasRetBuffArg();
}
+#ifdef UNIX_X86_ABI
+// For UNIX_X86_ABI and unmanaged function, we always need RetBuf if the return type is VALUETYPE
+inline BOOL HasRetBuffArgUnmanagedFixup(MetaSig * pSig)
+{
+ WRAPPER_NO_CONTRACT;
+ // We cannot just pSig->GetReturnType() here since it will return ELEMENT_TYPE_VALUETYPE for enums
+ CorElementType type = pSig->GetRetTypeHandleThrowing().GetVerifierCorElementType();
+ return type == ELEMENT_TYPE_VALUETYPE;
+}
+#endif
+
inline BOOL IsRetBuffPassedAsFirstArg()
{
WRAPPER_NO_CONTRACT;
// return buffer in correct register.
// The return structure secret arg comes first, however byvalue return is processed at
// the end because it could be the HRESULT-swapped argument which always comes last.
+
+#ifdef UNIX_X86_ABI
+ // For functions with value type class, managed and unmanaged calling convention differ
+ fMarshalReturnValueFirst = HasRetBuffArgUnmanagedFixup(&msig);
+#else // UNIX_X86_ABI
fMarshalReturnValueFirst = HasRetBuffArg(&msig);
-#endif
+#endif // UNIX_X86_ABI
+
+#endif // defined(_TARGET_X86_) || defined(_TARGET_ARM_)
}
{
LIMITED_METHOD_CONTRACT;
+#ifndef UNIX_X86_ABI
// odd-sized small structures are not
// enregistered e.g. struct { char a,b,c; }
return (sizeofvaluetype > 8) ||
(sizeofvaluetype & (sizeofvaluetype - 1)); // check that the size is power of two
+#else
+ // For UNIX_X86_ABI, we always return the value type by reference regardless of its size
+ return true;
+#endif
}
#include <pshpack1.h>
// for X86 and AMD64-Windows we bash the return type from struct to U1, U2, U4 or U8
// and use byrefNativeReturn for all other structs.
+ // for UNIX_X86_ABI, we always need a return buffer argument for any size of structs.
switch (nativeSize)
{
+#ifndef UNIX_X86_ABI
case 1: typ = ELEMENT_TYPE_U1; break;
case 2: typ = ELEMENT_TYPE_U2; break;
case 4: typ = ELEMENT_TYPE_U4; break;
case 8: typ = ELEMENT_TYPE_U8; break;
+#endif
default: byrefNativeReturn = true; break;
}
#endif