From 4e1a6f9d8f2a49403bbd00b8c8e5324048fb84d4 Mon Sep 17 00:00:00 2001 From: Ivan Maidanski Date: Thu, 15 Sep 2016 18:40:21 +0300 Subject: [PATCH] Fix calloc_explicitly_typed in case of lb*n overflow * typd_mlc.c: Include limits.h (for SIZE_MAX). * typd_mlc.c (GC_SIZE_MAX, GC_SQRT_SIZE_MAX): New macro (same as in malloc.c). * typd_mlc.c (GC_calloc_explicitly_typed): Return NULL if lb * n overflows (same algorithm as in calloc defined in malloc.c); eliminate lb *= n code duplication. --- typd_mlc.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/typd_mlc.c b/typd_mlc.c index 104e7c4..fe0220c 100644 --- a/typd_mlc.c +++ b/typd_mlc.c @@ -627,6 +627,15 @@ GC_API GC_ATTR_MALLOC void * GC_CALL return((void *) op); } +#include +#ifdef SIZE_MAX +# define GC_SIZE_MAX SIZE_MAX +#else +# define GC_SIZE_MAX (~(size_t)0) +#endif + +#define GC_SQRT_SIZE_MAX ((1U << (WORDSZ / 2)) - 1) + GC_API GC_ATTR_MALLOC void * GC_CALL GC_calloc_explicitly_typed(size_t n, size_t lb, GC_descr d) { @@ -640,15 +649,18 @@ GC_API GC_ATTR_MALLOC void * GC_CALL GC_calloc_explicitly_typed(size_t n, GC_ASSERT(GC_explicit_typing_initialized); descr_type = GC_make_array_descriptor((word)n, (word)lb, d, &simple_descr, &complex_descr, &leaf); + if ((lb | n) > GC_SQRT_SIZE_MAX /* fast initial check */ + && lb > 0 && n > GC_SIZE_MAX / lb) + return NULL; /* n*lb overflow */ + lb *= n; switch(descr_type) { case NO_MEM: return(0); - case SIMPLE: return(GC_malloc_explicitly_typed(n*lb, simple_descr)); + case SIMPLE: + return GC_malloc_explicitly_typed(lb, simple_descr); case LEAF: - lb *= n; lb += sizeof(struct LeafDescriptor) + TYPD_EXTRA_BYTES; break; case COMPLEX: - lb *= n; lb += TYPD_EXTRA_BYTES; break; } -- 2.7.4