Return null on allocation failure due to OOM instead of throwing a exception from...

Commit migrated from https://github.com/dotnet/coreclr/commit/2d9b2ab82c67344aa1d9fee422d7b7d5760cae92

diff --git a/src/coreclr/src/gc/env/gcenv.object.h b/src/coreclr/src/gc/env/gcenv.object.h
index c999e45..db8995a 100644 (file)
--- a/src/coreclr/src/gc/env/gcenv.object.h
+++ b/src/coreclr/src/gc/env/gcenv.object.h
@@ -31,6 +31,8 @@ public:
     void ClrGCBit() { m_uSyncBlockValue &= ~BIT_SBLK_GC_RESERVE; }
 };
 
+static_assert(sizeof(ObjHeader) == sizeof(uintptr_t), "this assumption is made by the VM!");
+
 #define MTFlag_ContainsPointers 1
 #define MTFlag_HasFinalizer 2
 #define MTFlag_IsArray 4

diff --git a/src/coreclr/src/gc/gc.cpp b/src/coreclr/src/gc/gc.cpp
index 11254cd..6cea472 100644 (file)
--- a/src/coreclr/src/gc/gc.cpp
+++ b/src/coreclr/src/gc/gc.cpp
@@ -30484,28 +30484,18 @@ CObjectHeader* gc_heap::allocate_large_object (size_t jsize, int64_t& alloc_byte
 #endif //BACKGROUND_GC
 #endif // MARK_ARRAY
 
-    size_t maxObjectSize = (INT32_MAX - 7 - Align(min_obj_size));
-
-#ifdef BIT64
-    if (g_pConfig->GetGCAllowVeryLargeObjects())
-    {
-        maxObjectSize = (INT64_MAX - 7 - Align(min_obj_size));
-    }
-#endif
-
-    if (jsize >= maxObjectSize)
-    {
-        if (g_pConfig->IsGCBreakOnOOMEnabled())
-        {
-            GCToOSInterface::DebugBreak();
-        }
-
-#ifndef FEATURE_REDHAWK
-        ThrowOutOfMemoryDimensionsExceeded();
-#else
-        return 0;
-#endif
-    }
+    // these next few lines are not strictly necessary anymore - they are here
+    // to sanity check that we didn't get asked to create an object
+    // that's too large.
+    #if BIT64
+    size_t maxObjectSize = (INT64_MAX - 7 - Align(min_obj_size)); 
+    #else
+    size_t maxObjectSize = (INT32_MAX - 7 - Align(min_obj_size)); 
+    #endif
+
+    // The VM should have thrown instead of passing us an allocation
+    // request that's too large.
+    assert(jsize < maxObjectSize);
 
     size_t size = AlignQword (jsize);
     int align_const = get_alignment_constant (FALSE);
@@ -34129,7 +34119,6 @@ BOOL GCHeap::StressHeap(gc_alloc_context * context)
 #define REGISTER_FOR_FINALIZATION(_object, _size) true
 #endif // FEATURE_PREMORTEM_FINALIZATION
 
-#ifdef FEATURE_REDHAWK
 #define CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(_object, _size, _register) do {  \
     if ((_object) == NULL || ((_register) && !REGISTER_FOR_FINALIZATION(_object, _size)))   \
     {                                                                                       \
@@ -34137,19 +34126,6 @@ BOOL GCHeap::StressHeap(gc_alloc_context * context)
         return NULL;                                                                        \
     }                                                                                       \
 } while (false)
-#else // FEATURE_REDHAWK
-#define CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(_object, _size, _register) do {  \
-    if ((_object) == NULL)                                                                  \
-    {                                                                                       \
-        STRESS_LOG_OOM_STACK(_size);                                                        \
-        ThrowOutOfMemory();                                                                 \
-    }                                                                                       \
-    if (_register)                                                                          \
-    {                                                                                       \
-        REGISTER_FOR_FINALIZATION(_object, _size);                                          \
-    }                                                                                       \
-} while (false)
-#endif // FEATURE_REDHAWK
 
 //
 // Small Object Allocator
@@ -34159,23 +34135,10 @@ Object *
 GCHeap::Alloc( size_t size, uint32_t flags REQD_ALIGN_DCL)
 {
     CONTRACTL {
-#ifdef FEATURE_REDHAWK
-        // Under Redhawk NULL is returned on failure.
         NOTHROW;
-#else
-        THROWS;
-#endif
         GC_TRIGGERS;
     } CONTRACTL_END;
 
-#if defined(_DEBUG) && !defined(FEATURE_REDHAWK)
-    if (g_pConfig->ShouldInjectFault(INJECTFAULT_GCHEAP))
-    {
-        char *a = new char;
-        delete a;
-    }
-#endif //_DEBUG && !FEATURE_REDHAWK
-
     TRIGGERSGC();
 
     Object* newAlloc = NULL;
@@ -34255,12 +34218,7 @@ GCHeap::AllocAlign8( size_t size, uint32_t flags)
 {
 #ifdef FEATURE_64BIT_ALIGNMENT
     CONTRACTL {
-#ifdef FEATURE_REDHAWK
-        // Under Redhawk NULL is returned on failure.
         NOTHROW;
-#else
-        THROWS;
-#endif
         GC_TRIGGERS;
     } CONTRACTL_END;
 
@@ -34294,12 +34252,7 @@ GCHeap::AllocAlign8(gc_alloc_context* ctx, size_t size, uint32_t flags )
 {
 #ifdef FEATURE_64BIT_ALIGNMENT
     CONTRACTL {
-#ifdef FEATURE_REDHAWK
-        // Under Redhawk NULL is returned on failure.
         NOTHROW;
-#else
-        THROWS;
-#endif
         GC_TRIGGERS;
     } CONTRACTL_END;
 
@@ -34333,25 +34286,12 @@ GCHeap::AllocAlign8Common(void* _hp, alloc_context* acontext, size_t size, uint3
 {
 #ifdef FEATURE_64BIT_ALIGNMENT
     CONTRACTL {
-#ifdef FEATURE_REDHAWK
-        // Under Redhawk NULL is returned on failure.
         NOTHROW;
-#else
-        THROWS;
-#endif
         GC_TRIGGERS;
     } CONTRACTL_END;
 
     gc_heap* hp = (gc_heap*)_hp;
 
-#if defined(_DEBUG) && !defined(FEATURE_REDHAWK)
-    if (g_pConfig->ShouldInjectFault(INJECTFAULT_GCHEAP))
-    {
-        char *a = new char;
-        delete a;
-    }
-#endif //_DEBUG && !FEATURE_REDHAWK
-
     TRIGGERSGC();
 
     Object* newAlloc = NULL;
@@ -34465,23 +34405,10 @@ Object *
 GCHeap::AllocLHeap( size_t size, uint32_t flags REQD_ALIGN_DCL)
 {
     CONTRACTL {
-#ifdef FEATURE_REDHAWK
-        // Under Redhawk NULL is returned on failure.
         NOTHROW;
-#else
-        THROWS;
-#endif
         GC_TRIGGERS;
     } CONTRACTL_END;
 
-#if defined(_DEBUG) && !defined(FEATURE_REDHAWK)
-    if (g_pConfig->ShouldInjectFault(INJECTFAULT_GCHEAP))
-    {
-        char *a = new char;
-        delete a;
-    }
-#endif //_DEBUG && !FEATURE_REDHAWK
-
     TRIGGERSGC();
 
     Object* newAlloc = NULL;
@@ -34536,23 +34463,10 @@ Object*
 GCHeap::Alloc(gc_alloc_context* context, size_t size, uint32_t flags REQD_ALIGN_DCL)
 {
     CONTRACTL {
-#ifdef FEATURE_REDHAWK
-        // Under Redhawk NULL is returned on failure.
         NOTHROW;
-#else
-        THROWS;
-#endif
         GC_TRIGGERS;
     } CONTRACTL_END;
 
-#if defined(_DEBUG) && !defined(FEATURE_REDHAWK)
-    if (g_pConfig->ShouldInjectFault(INJECTFAULT_GCHEAP))
-    {
-        char *a = new char;
-        delete a;
-    }
-#endif //_DEBUG && !FEATURE_REDHAWK
-
     TRIGGERSGC();
 
     Object* newAlloc = NULL;
@@ -36038,12 +35952,7 @@ bool
 CFinalize::RegisterForFinalization (int gen, Object* obj, size_t size)
 {
     CONTRACTL {
-#ifdef FEATURE_REDHAWK
-        // Under Redhawk false is returned on failure.
         NOTHROW;
-#else
-        THROWS;
-#endif
         GC_NOTRIGGER;
     } CONTRACTL_END;
 
@@ -36081,11 +35990,7 @@ CFinalize::RegisterForFinalization (int gen, Object* obj, size_t size)
             {
                 GCToOSInterface::DebugBreak();
             }
-#ifdef FEATURE_REDHAWK
             return false;
-#else
-            ThrowOutOfMemory();
-#endif
         }
     }
     Object*** end_si = &SegQueueLimit (dest);

diff --git a/src/coreclr/src/gc/gcinterface.h b/src/coreclr/src/gc/gcinterface.h
index f9ff098..d521dda 100644 (file)
--- a/src/coreclr/src/gc/gcinterface.h
+++ b/src/coreclr/src/gc/gcinterface.h
@@ -84,6 +84,10 @@ struct segment_info
 
 #define LARGE_OBJECT_SIZE ((size_t)(85000))
 
+// The minimum size of an object is three pointers wide: one for the syncblock,
+// one for the object header, and one for the first field in the object.
+#define min_obj_size ((sizeof(uint8_t*) + sizeof(uintptr_t) + sizeof(size_t)))
+
 class Object;
 class IGCHeap;
 
@@ -387,7 +391,16 @@ public:
     /*
     ===========================================================================
     Allocation routines. These all call into the GC's allocator and may trigger a garbage
-    collection.
+    collection. All allocation routines return NULL when the allocation request
+    couldn't be serviced due to being out of memory.
+
+    These allocation routines should not be called with allocation requests
+    larger than:
+       32-bit  -> 0x7FFFFFE0
+       64-bit  -> 0x7FFFFFFFFFFFFFE0
+
+    It is up to the caller of the API to raise appropriate errors if the amount
+    of requested memory is too large.
     ===========================================================================
     */
 

diff --git a/src/coreclr/src/gc/gcpriv.h b/src/coreclr/src/gc/gcpriv.h
index 5865e76..e0147c3 100644 (file)
--- a/src/coreclr/src/gc/gcpriv.h
+++ b/src/coreclr/src/gc/gcpriv.h
@@ -4073,8 +4073,6 @@ size_t generation_unusable_fragmentation (generation* inst)
 }
 
 #define plug_skew           sizeof(ObjHeader)
-#define min_obj_size        (sizeof(uint8_t*)+plug_skew+sizeof(size_t))//syncblock + vtable+ first field
-//Note that this encodes the fact that plug_skew is a multiple of uint8_t*.
 // We always use USE_PADDING_TAIL when fitting so items on the free list should be
 // twice the min_obj_size.
 #define min_free_list       (2*min_obj_size)

diff --git a/src/coreclr/src/vm/gchelpers.cpp b/src/coreclr/src/vm/gchelpers.cpp
index 9b0a17f..6b7a7a7 100644 (file)
--- a/src/coreclr/src/vm/gchelpers.cpp
+++ b/src/coreclr/src/vm/gchelpers.cpp
@@ -60,6 +60,41 @@ inline gc_alloc_context* GetThreadAllocContext()
     return & GetThread()->m_alloc_context;
 }
 
+// Checks to see if the given allocation size exceeds the
+// largest object size allowed - if it does, it throws
+// an OutOfMemoryException with a message indicating that
+// the OOM was not from memory pressure but from an object
+// being too large.
+inline void CheckObjectSize(size_t alloc_size)
+{
+    CONTRACTL {
+        THROWS;
+        GC_TRIGGERS;
+    } CONTRACTL_END;
+
+    size_t max_object_size;
+#ifdef BIT64
+    if (g_pConfig->GetGCAllowVeryLargeObjects())
+    {
+        max_object_size = (INT64_MAX - 7 - min_obj_size);
+    }
+    else
+#endif // BIT64
+    {
+        max_object_size = (INT32_MAX - 7 - min_obj_size);
+    }
+
+    if (alloc_size >= max_object_size)
+    {
+        if (g_pConfig->IsGCBreakOnOOMEnabled())
+        {
+            DebugBreak();
+        }
+
+        ThrowOutOfMemoryDimensionsExceeded();
+    }
+}
+
 
 // There are only three ways to get into allocate an object.
 //     * Call optimized helpers that were generated on the fly. This is how JIT compiled code does most
@@ -98,6 +133,7 @@ inline Object* Alloc(size_t size, BOOL bFinalize, BOOL bContainsPointers )
                    (bFinalize ? GC_ALLOC_FINALIZE : 0));
 
     Object *retVal = NULL;
+    CheckObjectSize(size);
 
     // We don't want to throw an SO during the GC, so make sure we have plenty
     // of stack before calling in.
@@ -106,6 +142,12 @@ inline Object* Alloc(size_t size, BOOL bFinalize, BOOL bContainsPointers )
         retVal = GCHeapUtilities::GetGCHeap()->Alloc(GetThreadAllocContext(), size, flags);
     else
         retVal = GCHeapUtilities::GetGCHeap()->Alloc(size, flags);
+
+    if (!retVal)
+    {
+        ThrowOutOfMemory();
+    }
+
     END_INTERIOR_STACK_PROBE;
     return retVal;
 }
@@ -126,6 +168,7 @@ inline Object* AllocAlign8(size_t size, BOOL bFinalize, BOOL bContainsPointers,
                    (bAlignBias ? GC_ALLOC_ALIGN8_BIAS : 0));
 
     Object *retVal = NULL;
+    CheckObjectSize(size);
 
     // We don't want to throw an SO during the GC, so make sure we have plenty
     // of stack before calling in.
@@ -135,6 +178,11 @@ inline Object* AllocAlign8(size_t size, BOOL bFinalize, BOOL bContainsPointers,
     else
         retVal = GCHeapUtilities::GetGCHeap()->AllocAlign8(size, flags);
 
+    if (!retVal)
+    {
+        ThrowOutOfMemory();
+    }
+
     END_INTERIOR_STACK_PROBE;
     return retVal;
 }
@@ -169,11 +217,18 @@ inline Object* AllocLHeap(size_t size, BOOL bFinalize, BOOL bContainsPointers )
                    (bFinalize ? GC_ALLOC_FINALIZE : 0));
 
     Object *retVal = NULL;
+    CheckObjectSize(size);
 
     // We don't want to throw an SO during the GC, so make sure we have plenty
     // of stack before calling in.
     INTERIOR_STACK_PROBE_FOR(GetThread(), static_cast<unsigned>(DEFAULT_ENTRY_PROBE_AMOUNT * 1.5));
     retVal = GCHeapUtilities::GetGCHeap()->AllocLHeap(size, flags);
+
+    if (!retVal)
+    {
+        ThrowOutOfMemory();
+    }
+
     END_INTERIOR_STACK_PROBE;
     return retVal;
 }