[sanitizer] fix calloc overflow in asan/tsan/msan

llvm-svn: 173441

diff --git a/compiler-rt/lib/asan/asan_allocator.cc b/compiler-rt/lib/asan/asan_allocator.cc
index 30dd4ce..370307a 100644 (file)
--- a/compiler-rt/lib/asan/asan_allocator.cc
+++ b/compiler-rt/lib/asan/asan_allocator.cc
@@ -35,6 +35,7 @@
 #include "asan_thread.h"
 #include "asan_thread_registry.h"
 #include "sanitizer/asan_interface.h"
+#include "sanitizer_common/sanitizer_allocator.h"
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_mutex.h"
 
@@ -712,6 +713,7 @@ void *asan_malloc(uptr size, StackTrace *stack) {
 }
 
 void *asan_calloc(uptr nmemb, uptr size, StackTrace *stack) {
+  if (__sanitizer::CallocShouldReturnNullDueToOverflow(size, nmemb)) return 0;
   void *ptr = (void*)Allocate(0, nmemb * size, stack, FROM_MALLOC);
   if (ptr)
     REAL(memset)(ptr, 0, nmemb * size);

diff --git a/compiler-rt/lib/asan/asan_allocator2.cc b/compiler-rt/lib/asan/asan_allocator2.cc
index 7bfa5fd..184627a 100644 (file)
--- a/compiler-rt/lib/asan/asan_allocator2.cc
+++ b/compiler-rt/lib/asan/asan_allocator2.cc
@@ -607,6 +607,7 @@ void *asan_malloc(uptr size, StackTrace *stack) {
 }
 
 void *asan_calloc(uptr nmemb, uptr size, StackTrace *stack) {
+  if (CallocShouldReturnNullDueToOverflow(size, nmemb)) return 0;
   void *ptr = Allocate(nmemb * size, 8, stack, FROM_MALLOC);
   if (ptr)
     REAL(memset)(ptr, 0, nmemb * size);

diff --git a/compiler-rt/lib/asan/tests/asan_noinst_test.cc b/compiler-rt/lib/asan/tests/asan_noinst_test.cc
index b7b4a4b..95ad15e 100644 (file)
--- a/compiler-rt/lib/asan/tests/asan_noinst_test.cc
+++ b/compiler-rt/lib/asan/tests/asan_noinst_test.cc
@@ -25,6 +25,7 @@
 #include <string.h>  // for memset()
 #include <algorithm>
 #include <vector>
+#include <limits>
 
 
 TEST(AddressSanitizer, InternalSimpleDeathTest) {
@@ -831,3 +832,11 @@ TEST(AddressSanitizerInterface, GetOwnershipStressTest) {
   for (size_t i = 0, n = pointers.size(); i < n; i++)
     free(pointers[i]);
 }
+
+TEST(AddressSanitizerInterface, CallocOverflow) {
+  size_t kArraySize = 4096;
+  volatile size_t kMaxSizeT = std::numeric_limits<size_t>::max();
+  volatile size_t kArraySize2 = kMaxSizeT / kArraySize + 10;
+  void *p = calloc(kArraySize, kArraySize2);  // Should return 0.
+  EXPECT_EQ(0L, Ident(p));
+}

diff --git a/compiler-rt/lib/msan/msan_interceptors.cc b/compiler-rt/lib/msan/msan_interceptors.cc
index 67c872f..0672bcf 100644 (file)
--- a/compiler-rt/lib/msan/msan_interceptors.cc
+++ b/compiler-rt/lib/msan/msan_interceptors.cc
@@ -18,6 +18,7 @@
 #include "interception/interception.h"
 #include "msan.h"
 #include "msan_platform_limits_posix.h"
+#include "sanitizer_common/sanitizer_allocator.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_libc.h"
 
@@ -683,6 +684,7 @@ INTERCEPTOR(SSIZE_T, recvmsg, int fd, struct msghdr *msg, int flags) {
 }
 
 INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) {
+  if (CallocShouldReturnNullDueToOverflow(size, nmemb)) return 0;
   GET_MALLOC_STACK_TRACE;
   if (!msan_inited) {
     // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.

diff --git a/compiler-rt/lib/msan/tests/msan_test.cc b/compiler-rt/lib/msan/tests/msan_test.cc
index f65d6f9..09bdf50 100644 (file)
--- a/compiler-rt/lib/msan/tests/msan_test.cc
+++ b/compiler-rt/lib/msan/tests/msan_test.cc
@@ -1685,6 +1685,14 @@ NOINLINE void RecursiveMalloc(int depth) {
   delete x2;
 }
 
+TEST(MemorySanitizer, CallocOverflow) {
+  size_t kArraySize = 4096;
+  volatile size_t kMaxSizeT = std::numeric_limits<size_t>::max();
+  volatile size_t kArraySize2 = kMaxSizeT / kArraySize + 10;
+  void *p = calloc(kArraySize, kArraySize2);  // Should return 0.
+  EXPECT_EQ(0L, Ident(p));
+}
+
 TEST(MemorySanitizerStress, DISABLED_MallocStackTrace) {
   RecursiveMalloc(22);
 }

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc
index b13a7c6..26baf73 100644 (file)
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc
@@ -75,4 +75,9 @@ void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback) {
   low_level_alloc_callback = callback;
 }
 
+bool CallocShouldReturnNullDueToOverflow(uptr size, uptr n) {
+  uptr mul = size * n;
+  return mul < size || mul < n;
+}
+
 }  // namespace __sanitizer

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h
index 1ca7c9a..39c20b9 100644 (file)
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h
@@ -1098,6 +1098,9 @@ class CombinedAllocator {
   AllocatorGlobalStats stats_;
 };
 
+// Returns true if calloc(size, n) should return 0 due to overflow in size*n.
+bool CallocShouldReturnNullDueToOverflow(uptr size, uptr n);
+
 }  // namespace __sanitizer
 
 #endif  // SANITIZER_ALLOCATOR_H

diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc b/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc
index be58ca9..3fc3845 100644 (file)
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc
@@ -326,6 +326,7 @@ TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
 }
 
 TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
+  if (__sanitizer::CallocShouldReturnNullDueToOverflow(size, n)) return 0;
   void *p = 0;
   {
     SCOPED_INTERCEPTOR_RAW(calloc, size, n);

diff --git a/compiler-rt/lib/tsan/tests/unit/tsan_mman_test.cc b/compiler-rt/lib/tsan/tests/unit/tsan_mman_test.cc
index 988918f..af813a2 100644 (file)
--- a/compiler-rt/lib/tsan/tests/unit/tsan_mman_test.cc
+++ b/compiler-rt/lib/tsan/tests/unit/tsan_mman_test.cc
@@ -10,6 +10,7 @@
 // This file is a part of ThreadSanitizer (TSan), a race detector.
 //
 //===----------------------------------------------------------------------===//
+#include <limits>
 #include "tsan_mman.h"
 #include "tsan_rtl.h"
 #include "gtest/gtest.h"
@@ -145,4 +146,13 @@ TEST(Mman, Stats) {
   EXPECT_EQ(__tsan_get_free_bytes(), free0);
   EXPECT_EQ(__tsan_get_unmapped_bytes(), unmapped0);
 }
+
+TEST(Mman, CallocOverflow) {
+  size_t kArraySize = 4096;
+  volatile size_t kMaxSizeT = std::numeric_limits<size_t>::max();
+  volatile size_t kArraySize2 = kMaxSizeT / kArraySize + 10;
+  volatile void *p = calloc(kArraySize, kArraySize2);  // Should return 0.
+  EXPECT_EQ(0L, p);
+}
+
 }  // namespace __tsan