[release/3.0] Opt COM methods out of the new Windows instance-method handling (#24012)

* Opt COM methods out of the new Windows instance-method handling.

* Remove calls to run the MarshalStructReturn tests since we're rolling back the calling convention support change.

diff --git a/src/vm/dllimport.cpp b/src/vm/dllimport.cpp
index 924a50f..93d0f79 100644 (file)
--- a/src/vm/dllimport.cpp
+++ b/src/vm/dllimport.cpp
@@ -3956,7 +3956,10 @@ static void CreateNDirectStubWorker(StubState*         pss,
     BOOL fMarshalReturnValueFirst = FALSE;
 
     BOOL fReverseWithReturnBufferArg = FALSE;
-    bool isInstanceMethod = fStubNeedsCOM || fThisCall;
+    // Only consider ThisCall methods to be instance methods.
+    // Techinically COM methods are also instance methods, but we don't want to change the behavior of the built-in
+    // COM abi work because there are many users that rely on the current behavior (for example WPF).
+    bool isInstanceMethod = fThisCall;
     
     // We can only change fMarshalReturnValueFirst to true when we are NOT doing HRESULT-swapping!
     // When we are HRESULT-swapping, the managed return type is actually the type of the last parameter and not the return type.

diff --git a/src/vm/ilmarshalers.h b/src/vm/ilmarshalers.h
index 866b3ec..750ac3d 100644 (file)
--- a/src/vm/ilmarshalers.h
+++ b/src/vm/ilmarshalers.h
@@ -583,9 +583,7 @@ public:
         bool byrefNativeReturn = false;
         CorElementType typ = ELEMENT_TYPE_VOID;
         UINT32 nativeSize = 0;
-        bool nativeMethodIsMemberFunction = (m_pslNDirect->TargetHasThis() && IsCLRToNative(m_dwMarshalFlags))
-            || (m_pslNDirect->HasThis() && !IsCLRToNative(m_dwMarshalFlags))
-            || ((CorInfoCallConv)m_pslNDirect->GetStubTargetCallingConv() == CORINFO_CALLCONV_THISCALL);
+        bool nativeMethodIsMemberFunction = (CorInfoCallConv)m_pslNDirect->GetStubTargetCallingConv() == CORINFO_CALLCONV_THISCALL;
             
         // we need to convert value type return types to primitives as
         // JIT does not inline P/Invoke calls that return structures

diff --git a/tests/src/Interop/COM/NETClients/Primitives/NumericTests.cs b/tests/src/Interop/COM/NETClients/Primitives/NumericTests.cs
index 1eaa2fd..c134ea2 100644 (file)
--- a/tests/src/Interop/COM/NETClients/Primitives/NumericTests.cs
+++ b/tests/src/Interop/COM/NETClients/Primitives/NumericTests.cs
@@ -38,7 +38,6 @@ namespace NetClient
             this.Marshal_Float(a / 100f, b / 100f);
             this.Marshal_Double(a / 100.0, b / 100.0);
             this.Marshal_ManyInts();
-            this.Marshal_Struct_Return();
         }
 
         static private bool EqualByBound(float expected, float actual)

diff --git a/tests/src/Interop/COM/NativeClients/Primitives/NumericTests.cpp b/tests/src/Interop/COM/NativeClients/Primitives/NumericTests.cpp
index a11bb77..c02e704 100644 (file)
--- a/tests/src/Interop/COM/NativeClients/Primitives/NumericTests.cpp
+++ b/tests/src/Interop/COM/NativeClients/Primitives/NumericTests.cpp
@@ -265,5 +265,4 @@ void Run_NumericTests()
     MarshalFloat(numericTesting, (float)a / 100.f, (float)b / 100.f);
     MarshalDouble(numericTesting, (double)a / 100.0, (double)b / 100.0);
     MarshalManyInts(numericTesting);
-    MarshalStructReturn(numericTesting);
 }