Fallback to old behavior when marshaling a non-blittable fixed buffer (dotnet/coreclr...

* Throw an exception only when attempting to marshal nonblittable fixed buffers to/from native instead of on all marshalling operations.

* Assert in debug and checked builds when trying to marshal a fixed buffer of a non-blittable type.

Commit migrated from https://github.com/dotnet/coreclr/commit/f7863cf3877a1ce17bde80b8e78ed5b9ba2cf8f5

diff --git a/src/coreclr/src/vm/fieldmarshaler.cpp b/src/coreclr/src/vm/fieldmarshaler.cpp
index ef7dc9d..d5c16b0 100644 (file)
--- a/src/coreclr/src/vm/fieldmarshaler.cpp
+++ b/src/coreclr/src/vm/fieldmarshaler.cpp
@@ -35,7 +35,9 @@
 BOOL CheckForPrimitiveType(CorElementType elemType, CQuickArray<WCHAR> *pStrPrimitiveType);
 TypeHandle ArraySubTypeLoadWorker(const SString &strUserDefTypeName, Assembly* pAssembly);
 TypeHandle GetFieldTypeHandleWorker(MetaSig *pFieldSig);
+#ifdef _DEBUG
 BOOL IsFixedBuffer(mdFieldDef field, IMDInternalImport *pInternalImport);
+#endif
 
 
 //=======================================================================
@@ -660,14 +662,11 @@ do                                                      \
                 {
                     if (IsStructMarshalable(thNestedType))
                     {
-                        if (IsFixedBuffer(pfwalk->m_MD, pInternalImport) && !thNestedType.GetMethodTable()->IsBlittable())
-                        {
-                            INITFIELDMARSHALER(NFT_ILLEGAL, FieldMarshaler_Illegal, (IDS_EE_BADMARSHAL_NOTMARSHALABLE));
-                        }
-                        else
-                        {
-                            INITFIELDMARSHALER(NFT_NESTEDVALUECLASS, FieldMarshaler_NestedValueClass, (thNestedType.GetMethodTable()));
-                        }
+#ifdef _DEBUG
+                        INITFIELDMARSHALER(NFT_NESTEDVALUECLASS, FieldMarshaler_NestedValueClass, (thNestedType.GetMethodTable(), IsFixedBuffer(pfwalk->m_MD, pInternalImport)));
+#else
+                        INITFIELDMARSHALER(NFT_NESTEDVALUECLASS, FieldMarshaler_NestedValueClass, (thNestedType.GetMethodTable()));
+#endif
                     }
                     else
                     {
@@ -1237,12 +1236,14 @@ BOOL IsStructMarshalable(TypeHandle th)
     return TRUE;
 }
 
+#ifdef _DEBUG
 BOOL IsFixedBuffer(mdFieldDef field, IMDInternalImport *pInternalImport)
 {
     HRESULT hr = pInternalImport->GetCustomAttributeByName(field, g_FixedBufferAttribute, NULL, NULL);
 
     return hr == S_OK ? TRUE : FALSE;
 }
+#endif
 
 
 //=======================================================================
@@ -2897,6 +2898,9 @@ VOID FieldMarshaler_NestedValueClass::NestedValueClassUpdateNativeImpl(const VOI
     }
     else
     {
+#ifdef _DEBUG
+        _ASSERTE_MSG(!IsFixedBuffer(), "Cannot correctly marshal fixed buffers of non-blittable types");
+#endif
         LayoutUpdateNative((LPVOID*)ppProtectedCLR, startoffset, pMT, (BYTE*)pNative, ppCleanupWorkListOnStack);
     }
 }
@@ -2931,6 +2935,9 @@ VOID FieldMarshaler_NestedValueClass::NestedValueClassUpdateCLRImpl(const VOID *
     }
     else
     {
+#ifdef _DEBUG
+        _ASSERTE_MSG(!IsFixedBuffer(), "Cannot correctly marshal fixed buffers of non-blittable types");
+#endif
         LayoutUpdateCLR((LPVOID*)ppProtectedCLR,
             startoffset,
             pMT,

diff --git a/src/coreclr/src/vm/fieldmarshaler.h b/src/coreclr/src/vm/fieldmarshaler.h
index e3390c7..1f7ac88 100644 (file)
--- a/src/coreclr/src/vm/fieldmarshaler.h
+++ b/src/coreclr/src/vm/fieldmarshaler.h
@@ -710,10 +710,17 @@ private:
 class FieldMarshaler_NestedValueClass : public FieldMarshaler
 {
 public:
+#ifndef _DEBUG
     FieldMarshaler_NestedValueClass(MethodTable *pMT)
+#else
+    FieldMarshaler_NestedValueClass(MethodTable *pMT, BOOL isFixedBuffer)
+#endif
     {
         WRAPPER_NO_CONTRACT;
         m_pNestedMethodTable.SetValueMaybeNull(pMT);
+#ifdef _DEBUG
+        m_isFixedBuffer = isFixedBuffer;
+#endif
     }
 
     BOOL IsNestedValueClassMarshalerImpl() const
@@ -761,6 +768,9 @@ public:
     START_COPY_TO_IMPL(FieldMarshaler_NestedValueClass)
     {
         pDestFieldMarshaller->m_pNestedMethodTable.SetValueMaybeNull(GetMethodTable());
+#ifdef _DEBUG
+        pDestFieldMarshaller->m_isFixedBuffer = m_isFixedBuffer;
+#endif
     }
     END_COPY_TO_IMPL(FieldMarshaler_NestedValueClass)
 
@@ -793,10 +803,20 @@ public:
         return m_pNestedMethodTable.GetValueMaybeNull();
     }
 
+#ifdef _DEBUG
+    BOOL IsFixedBuffer() const
+    {
+        return m_isFixedBuffer;
+    }
+#endif
+
 
 private:
     // MethodTable of nested NStruct.
     RelativeFixupPointer<PTR_MethodTable> m_pNestedMethodTable;
+#ifdef _DEBUG
+    BOOL m_isFixedBuffer;
+#endif
 };