Ensure stack size is computed for stdcall name mangling (#46722)

Fixes #46667

diff --git a/src/coreclr/vm/dllimport.cpp b/src/coreclr/vm/dllimport.cpp
index 02e5e23..d92ac44 100644 (file)
--- a/src/coreclr/vm/dllimport.cpp
+++ b/src/coreclr/vm/dllimport.cpp
@@ -3697,22 +3697,6 @@ static void CreateNDirectStubWorker(StubState*         pss,
             _ASSERTE(nativeStackSize >= TARGET_POINTER_SIZE);
             nativeStackSize -= TARGET_POINTER_SIZE;
         }
-#else // TARGET_X86
-        //
-        // The algorithm to compute nativeStackSize on the fly is x86-specific.
-        // Recompute the correct size for other platforms from the stub signature.
-        //
-        if (SF_IsForwardStub(dwStubFlags))
-        {
-            // It would be nice to compute the correct value for forward stubs too.
-            // The value is only used in MarshalNative::NumParamBytes right now,
-            // and changing what MarshalNative::NumParamBytes returns is
-            // a potential breaking change.
-        }
-        else
-        {
-            // native stack size is updated in code:ILStubState.SwapStubSignatures
-        }
 #endif // TARGET_X86
 
         if (!FitsInU2(nativeStackSize))
@@ -5109,7 +5093,7 @@ MethodDesc* NDirect::GetILStubMethodDesc(NDirectMethodDesc* pNMD, PInvokeStaticS
 
     MethodDesc* pStubMD = NULL;
 
-    if (!pNMD->IsVarArgs())
+    if (!pNMD->IsVarArgs() || SF_IsForNumParamBytes(dwStubFlags))
     {
         if (pNMD->IsClassConstructorTriggeredByILStub())
         {
@@ -5118,7 +5102,7 @@ MethodDesc* NDirect::GetILStubMethodDesc(NDirectMethodDesc* pNMD, PInvokeStaticS
 
         pStubMD = CreateCLRToNativeILStub(
             pSigInfo,
-            dwStubFlags,
+            dwStubFlags & ~NDIRECTSTUB_FL_FOR_NUMPARAMBYTES,
             pNMD);
     }
 
@@ -5219,6 +5203,9 @@ PCODE NDirect::GetStubForILStub(NDirectMethodDesc* pNMD, MethodDesc** ppStubMD,
         *ppStubMD = NDirect::GetILStubMethodDesc(pNMD, &sigInfo, dwStubFlags);
     }
 
+    if (SF_IsForNumParamBytes(dwStubFlags))
+        return NULL;
+
     if (*ppStubMD)
     {
         pStub = JitILStub(*ppStubMD);

diff --git a/src/coreclr/vm/dllimport.h b/src/coreclr/vm/dllimport.h
index 0cb2fad..43a230b 100644 (file)
--- a/src/coreclr/vm/dllimport.h
+++ b/src/coreclr/vm/dllimport.h
@@ -162,7 +162,7 @@ enum NDirectStubFlags
     // unused                               = 0x00800000,
 
     // internal flags -- these won't ever show up in an NDirectStubHashBlob
-    // unused                               = 0x10000000,
+    NDIRECTSTUB_FL_FOR_NUMPARAMBYTES        = 0x10000000,   // do just enough to return the right value from Marshal.NumParamBytes
 
 #ifdef FEATURE_COMINTEROP
     NDIRECTSTUB_FL_COMLATEBOUND             = 0x20000000,   // we use a generic stub for late bound calls
@@ -211,6 +211,7 @@ inline bool SF_IsNGENedStubForProfiling(DWORD dwStubFlags) { LIMITED_METHOD_CONT
 inline bool SF_IsDebuggableStub        (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return (dwStubFlags < NDIRECTSTUB_FL_INVALID && 0 != (dwStubFlags & NDIRECTSTUB_FL_GENERATEDEBUGGABLEIL)); }
 inline bool SF_IsCALLIStub             (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return (dwStubFlags < NDIRECTSTUB_FL_INVALID && 0 != (dwStubFlags & NDIRECTSTUB_FL_UNMANAGED_CALLI)); }
 inline bool SF_IsStubWithCctorTrigger  (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return (dwStubFlags < NDIRECTSTUB_FL_INVALID && 0 != (dwStubFlags & NDIRECTSTUB_FL_TRIGGERCCTOR)); }
+inline bool SF_IsForNumParamBytes      (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return (dwStubFlags < NDIRECTSTUB_FL_INVALID && 0 != (dwStubFlags & NDIRECTSTUB_FL_FOR_NUMPARAMBYTES)); }
 inline bool SF_IsStructMarshalStub     (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return (dwStubFlags < NDIRECTSTUB_FL_INVALID && 0 != (dwStubFlags & NDIRECTSTUB_FL_STRUCT_MARSHAL)); }
 
 #ifdef FEATURE_ARRAYSTUB_AS_IL

diff --git a/src/coreclr/vm/method.cpp b/src/coreclr/vm/method.cpp
index 20191c2..1535a73 100644 (file)
--- a/src/coreclr/vm/method.cpp
+++ b/src/coreclr/vm/method.cpp
@@ -5277,7 +5277,7 @@ void NDirectMethodDesc::InterlockedSetNDirectFlags(WORD wFlags)
 #ifndef CROSSGEN_COMPILE
 
 #ifdef TARGET_WINDOWS
-FARPROC NDirectMethodDesc::FindEntryPointWithMangling(NATIVE_LIBRARY_HANDLE hMod, PTR_CUTF8 entryPointName) const
+FARPROC NDirectMethodDesc::FindEntryPointWithMangling(NATIVE_LIBRARY_HANDLE hMod, PTR_CUTF8 entryPointName)
 {
     CONTRACTL
     {
@@ -5298,12 +5298,7 @@ FARPROC NDirectMethodDesc::FindEntryPointWithMangling(NATIVE_LIBRARY_HANDLE hMod
 
     if (IsStdCall())
     {
-        if (GetModule()->IsReadyToRun())
-        {
-            // Computing if marshalling is required also computes the required stack size. We need the stack size to correctly form the
-            // name of the import pinvoke function on x86
-            ((NDirectMethodDesc*)this)->MarshalingRequired();
-        }
+        EnsureStackArgumentSize();
 
         DWORD probedEntrypointNameLength = (DWORD)(strlen(entryPointName) + 1); // 1 for null terminator
         int dstbufsize = (int)(sizeof(char) * (probedEntrypointNameLength + 10)); // 10 for stdcall mangling
@@ -5323,7 +5318,7 @@ FARPROC NDirectMethodDesc::FindEntryPointWithMangling(NATIVE_LIBRARY_HANDLE hMod
     return pFunc;
 }
 
-FARPROC NDirectMethodDesc::FindEntryPointWithSuffix(NATIVE_LIBRARY_HANDLE hMod, PTR_CUTF8 entryPointName, char suffix) const
+FARPROC NDirectMethodDesc::FindEntryPointWithSuffix(NATIVE_LIBRARY_HANDLE hMod, PTR_CUTF8 entryPointName, char suffix)
 {
     // Allocate space for a copy of the entry point name.
     DWORD entryPointWithSuffixLen = (DWORD)(strlen(entryPointName) + 1); // +1 for charset decorations
@@ -5342,7 +5337,7 @@ FARPROC NDirectMethodDesc::FindEntryPointWithSuffix(NATIVE_LIBRARY_HANDLE hMod,
 #endif
 
 //*******************************************************************************
-LPVOID NDirectMethodDesc::FindEntryPoint(NATIVE_LIBRARY_HANDLE hMod) const
+LPVOID NDirectMethodDesc::FindEntryPoint(NATIVE_LIBRARY_HANDLE hMod)
 {
     CONTRACTL
     {
@@ -5390,6 +5385,26 @@ LPVOID NDirectMethodDesc::FindEntryPoint(NATIVE_LIBRARY_HANDLE hMod) const
     return reinterpret_cast<LPVOID>(pFunc);
 #endif
 }
+
+#if defined(TARGET_X86)
+//*******************************************************************************
+void NDirectMethodDesc::EnsureStackArgumentSize()
+{
+    STANDARD_VM_CONTRACT;
+
+    if (ndirect.m_cbStackArgumentSize == 0xFFFF)
+    {
+        // Marshalling required check sets the stack size as side-effect when marshalling is not required.
+        if (MarshalingRequired())
+        {
+            // Generating interop stub sets the stack size as side-effect in all cases
+            MethodDesc* pStubMD;
+            GetStubForInteropMethod(this, NDIRECTSTUB_FL_FOR_NUMPARAMBYTES, &pStubMD);
+        }
+    }
+}
+#endif
+
 #endif // CROSSGEN_COMPILE
 
 #if !defined(CROSSGEN_COMPILE)

diff --git a/src/coreclr/vm/method.hpp b/src/coreclr/vm/method.hpp
index f34b9f1..330e9e9 100644 (file)
--- a/src/coreclr/vm/method.hpp
+++ b/src/coreclr/vm/method.hpp
@@ -3022,7 +3022,7 @@ public:
     };
 
     // Resolve the import to the NDirect target and set it on the NDirectMethodDesc.
-    static void *ResolveAndSetNDirectTarget(_In_ NDirectMethodDesc *pMD);
+    static void* ResolveAndSetNDirectTarget(_In_ NDirectMethodDesc* pMD);
 
     // Attempt to import the NDirect target if a GC transition is suppressed.
     static BOOL TryResolveNDirectTargetForNoGCTransition(_In_ MethodDesc* pMD, _Out_ void** ndirectTarget);
@@ -3148,7 +3148,7 @@ public:
     BOOL IsDefaultDllImportSearchPathsAttributeCached()
     {
         LIMITED_METHOD_CONTRACT;
-        return (ndirect.m_wFlags  & kDefaultDllImportSearchPathsIsCached) != 0;
+        return (ndirect.m_wFlags & kDefaultDllImportSearchPathsIsCached) != 0;
     }
 
     ULONG DefaultDllImportSearchPathsAttributeCachedValue()
@@ -3221,12 +3221,12 @@ public:
     //  Find the entry point name and function address
     //  based on the module and data from NDirectMethodDesc
     //
-    LPVOID FindEntryPoint(NATIVE_LIBRARY_HANDLE hMod) const;
+    LPVOID FindEntryPoint(NATIVE_LIBRARY_HANDLE hMod);
 
 #ifdef TARGET_WINDOWS
 private:
-    FARPROC FindEntryPointWithMangling(NATIVE_LIBRARY_HANDLE mod, PTR_CUTF8 entryPointName) const;
-    FARPROC FindEntryPointWithSuffix(NATIVE_LIBRARY_HANDLE mod, PTR_CUTF8 entryPointName, char suffix) const;
+    FARPROC FindEntryPointWithMangling(NATIVE_LIBRARY_HANDLE mod, PTR_CUTF8 entryPointName);
+    FARPROC FindEntryPointWithSuffix(NATIVE_LIBRARY_HANDLE mod, PTR_CUTF8 entryPointName, char suffix);
 #endif
 public:
 
@@ -3255,6 +3255,8 @@ public:
     }
 
 #if defined(TARGET_X86)
+    void EnsureStackArgumentSize();
+
     WORD GetStackArgumentSize() const
     {
         LIMITED_METHOD_DAC_CONTRACT;