Ensure oom_fn callback executed on out-of-memory in calloc

(fix commits e10c1eb, 4e1a6f9)

* malloc.c [REDIRECT_MALLOC && !REDIRECT_MALLOC_IN_HEADER] (calloc):
Call oom_fn(SIZE_MAX) (instead of returning NULL) if n*lb overflows.
* typd_mlc.c (GC_calloc_explicitly_typed): Likewise.
* typd_mlc.c (GC_calloc_explicitly_typed): If register_disappearing_link
failed due to lack of memory then call oom_fn(lb) instead of
GC_malloc(lb); update comment.

diff --git a/malloc.c b/malloc.c
index c4a5430..5301518 100644 (file)
--- a/malloc.c
+++ b/malloc.c
@@ -431,7 +431,7 @@ GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_uncollectable(size_t lb)
   {
     if ((lb | n) > GC_SQRT_SIZE_MAX /* fast initial test */
         && lb && n > GC_SIZE_MAX / lb)
-      return NULL;
+      return (*GC_get_oom_fn())(GC_SIZE_MAX); /* n*lb overflow */
 #   if defined(GC_LINUX_THREADS)
       /* libpthread allocated some memory that is only pointed to by    */
       /* mmapped thread stacks.  Make sure it is not collectible.       */

diff --git a/typd_mlc.c b/typd_mlc.c
index d19e4c1..7e91fab 100644 (file)
--- a/typd_mlc.c
+++ b/typd_mlc.c
@@ -647,7 +647,7 @@ GC_API GC_ATTR_MALLOC void * GC_CALL GC_calloc_explicitly_typed(size_t n,
                                           &complex_descr, &leaf);
     if ((lb | n) > GC_SQRT_SIZE_MAX /* fast initial check */
         && lb > 0 && n > GC_SIZE_MAX / lb)
-      return NULL; /* n*lb overflow */
+      return (*GC_get_oom_fn())(GC_SIZE_MAX); /* n*lb overflow */
     lb *= n;
     switch(descr_type) {
         case NO_MEM: return(0);
@@ -690,9 +690,7 @@ GC_API GC_ATTR_MALLOC void * GC_CALL GC_calloc_explicitly_typed(size_t n,
 #     endif
         {
             /* Couldn't register it due to lack of memory.  Punt.       */
-            /* This will probably fail too, but gives the recovery code */
-            /* a chance.                                                */
-            return GC_malloc(lb);
+            return (*GC_get_oom_fn())(lb);
         }
     }
     return op;