[sanitizer] Change Mmap*NoAccess to return nullptr on error

Summary:
`MmapNoAccess` & `MmapFixedNoAccess` return directly the result of
`internal_mmap`, as opposed to other Mmap functions that return nullptr.

This inconsistency leads to some confusion for the callers, as some check for
`~(uptr)0` (`MAP_FAILED`) for failure (while it can fail with `-ENOMEM` for
example).

Two potential solutions: change the callers, or make the functions return
`nullptr` on failure to follow the precedent set by the other functions.
The second option looked more appropriate to me.

Correct the callers that were wrongly checking for `~(uptr)0` or
`MAP_FAILED`.

TODO for follow up CLs:
- There are a couple of `internal_mmap` calls in XRay that check for
  MMAP_FAILED as a result as well (cc: @dberris); they should use
  `internal_iserror`;

Reviewers: eugenis, alekseyshl, dberris, kubamracek

Reviewed By: alekseyshl

Subscribers: kristina, kubamracek, delcypher, #sanitizers, dberris, llvm-commits

Differential Revision: https://reviews.llvm.org/D50940

llvm-svn: 340576

diff --git a/compiler-rt/lib/asan/asan_linux.cc b/compiler-rt/lib/asan/asan_linux.cc
index 625f32d..cb0a613 100644 (file)
--- a/compiler-rt/lib/asan/asan_linux.cc
+++ b/compiler-rt/lib/asan/asan_linux.cc
@@ -126,7 +126,7 @@ uptr FindDynamicShadowStart() {
   uptr map_size = shadow_size + left_padding + alignment;
 
   uptr map_start = (uptr)MmapNoAccess(map_size);
-  CHECK_NE(map_start, ~(uptr)0);
+  CHECK(map_start);
 
   uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
   UnmapFromTo(map_start, shadow_start - left_padding);

diff --git a/compiler-rt/lib/asan/asan_premap_shadow.cc b/compiler-rt/lib/asan/asan_premap_shadow.cc
index 229eba9..9002fd6 100644 (file)
--- a/compiler-rt/lib/asan/asan_premap_shadow.cc
+++ b/compiler-rt/lib/asan/asan_premap_shadow.cc
@@ -40,7 +40,7 @@ uptr PremapShadow() {
   uptr map_size = shadow_size + left_padding + alignment;
 
   uptr map_start = (uptr)MmapNoAccess(map_size);
-  CHECK_NE(map_start, ~(uptr)0);
+  CHECK(map_start);
 
   uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
   uptr shadow_end = shadow_start + shadow_size;

diff --git a/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cc b/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cc
index 1733800..0eb4956 100644 (file)
--- a/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cc
+++ b/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cc
@@ -46,7 +46,7 @@ static uptr MapDynamicShadow(uptr shadow_size_bytes) {
   const uptr map_size = shadow_size + left_padding + alignment;
 
   const uptr map_start = (uptr)MmapNoAccess(map_size);
-  CHECK_NE(map_start, ~(uptr)0);
+  CHECK(map_start);
 
   const uptr shadow_start = RoundUpTo(map_start + left_padding, alignment);
 

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
index 6acb4f8..88134a6 100644 (file)
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
@@ -77,7 +77,7 @@ class SizeClassAllocator64 {
     } else {
       NonConstSpaceBeg = address_range.Init(TotalSpaceSize,
                                             PrimaryAllocatorName);
-      CHECK_NE(NonConstSpaceBeg, ~(uptr)0);
+      CHECK(NonConstSpaceBeg);
     }
     SetReleaseToOSIntervalMs(release_to_os_interval_ms);
     MapWithCallbackOrDie(SpaceEnd(), AdditionalSize());

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc b/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc
index 090f17c..43fde2b 100644 (file)
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac_libcdep.cc
@@ -15,14 +15,12 @@
 #if SANITIZER_MAC
 #include "sanitizer_mac.h"
 
-#include <sys/mman.h>
-
 namespace __sanitizer {
 
 void RestrictMemoryToMaxAddress(uptr max_address) {
   uptr size_to_mmap = GetMaxUserVirtualAddress() + 1 - max_address;
   void *res = MmapFixedNoAccess(max_address, size_to_mmap, "high gap");
-  CHECK(res != MAP_FAILED);
+  CHECK(res);
 }
 
 }  // namespace __sanitizer

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc
index 266e9bd..d30c306 100644 (file)
--- a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc
@@ -387,13 +387,18 @@ void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name) {
   unsigned flags = MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE;
   if (fd == -1) flags |= MAP_ANON;
 
-  return (void *)internal_mmap((void *)fixed_addr, size, PROT_NONE, flags, fd,
-                               0);
+  uptr p = internal_mmap((void *)fixed_addr, size, PROT_NONE, flags, fd, 0);
+  if (internal_iserror(p))
+    return nullptr;
+  return (void *)p;
 }
 
 void *MmapNoAccess(uptr size) {
   unsigned flags = MAP_PRIVATE | MAP_ANON | MAP_NORESERVE;
-  return (void *)internal_mmap(nullptr, size, PROT_NONE, flags, -1, 0);
+  uptr p = internal_mmap(nullptr, size, PROT_NONE, flags, -1, 0);
+  if (internal_iserror(p))
+    return nullptr;
+  return (void *)p;
 }
 
 // This function is defined elsewhere if we intercepted pthread_attr_getstack.

diff --git a/compiler-rt/lib/scudo/scudo_allocator_secondary.h b/compiler-rt/lib/scudo/scudo_allocator_secondary.h
index ff6246e..882236b 100644 (file)
--- a/compiler-rt/lib/scudo/scudo_allocator_secondary.h
+++ b/compiler-rt/lib/scudo/scudo_allocator_secondary.h
@@ -86,7 +86,7 @@ class LargeMmapAllocator {
 
     ReservedAddressRange AddressRange;
     uptr ReservedBeg = AddressRange.Init(ReservedSize, SecondaryAllocatorName);
-    if (UNLIKELY(ReservedBeg == ~static_cast<uptr>(0)))
+    if (UNLIKELY(!ReservedBeg))
       return nullptr;
     // A page-aligned pointer is assumed after that, so check it now.
     DCHECK(IsAligned(ReservedBeg, PageSize));