From 90678f65ae47523586bd34392ed3cd1369cf5e9b Mon Sep 17 00:00:00 2001 From: Kostya Kortchinsky Date: Mon, 26 Oct 2020 14:54:22 -0700 Subject: [PATCH] [GWP-ASan] Abstract the thread local variables access In a similar fashion to D87420 for Scudo, this CL introduces a way to get thread local variables via a platform-specific reserved TLS slot, since Fuchsia doesn't support ELF TLS from the libc itself. If needing to use this, a platform will have to define `GWP_ASAN_HAS_PLATFORM_TLS_SLOT` and provide `gwp_asan_platform_tls_slot.h` which will define a `uint64_t *getPlatformGwpAsanTlsSlot()` function that will return the TLS word of storage. I snuck in a couple of cleanup items as well, moving some static functions to anonymous namespace for consistency. Differential Revision: https://reviews.llvm.org/D90195 --- compiler-rt/lib/gwp_asan/CMakeLists.txt | 1 + .../lib/gwp_asan/guarded_pool_allocator.cpp | 34 ++++++-------- compiler-rt/lib/gwp_asan/guarded_pool_allocator.h | 40 +++------------- .../guarded_pool_allocator_posix.cpp | 2 +- .../platform_specific/guarded_pool_allocator_tls.h | 53 ++++++++++++++++++++++ .../gwp_asan/platform_specific/utilities_posix.cpp | 2 +- compiler-rt/lib/gwp_asan/utilities.cpp | 2 +- 7 files changed, 78 insertions(+), 56 deletions(-) create mode 100644 compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_tls.h diff --git a/compiler-rt/lib/gwp_asan/CMakeLists.txt b/compiler-rt/lib/gwp_asan/CMakeLists.txt index 6bfb3ca..889a15e 100644 --- a/compiler-rt/lib/gwp_asan/CMakeLists.txt +++ b/compiler-rt/lib/gwp_asan/CMakeLists.txt @@ -22,6 +22,7 @@ set(GWP_ASAN_HEADERS mutex.h options.h options.inc + platform_specific/guarded_pool_allocator_tls.h stack_trace_compressor.h utilities.h ) diff --git a/compiler-rt/lib/gwp_asan/guarded_pool_allocator.cpp b/compiler-rt/lib/gwp_asan/guarded_pool_allocator.cpp index 6c4f2b8..a895032 100644 --- a/compiler-rt/lib/gwp_asan/guarded_pool_allocator.cpp +++ b/compiler-rt/lib/gwp_asan/guarded_pool_allocator.cpp @@ -37,6 +37,14 @@ namespace { // referenced by users outside this translation unit, in order to avoid // init-order-fiasco. GuardedPoolAllocator *SingletonPtr = nullptr; + +size_t roundUpTo(size_t Size, size_t Boundary) { + return (Size + Boundary - 1) & ~(Boundary - 1); +} + +uintptr_t getPageAddr(uintptr_t Ptr, uintptr_t PageSize) { + return Ptr & ~(PageSize - 1); +} } // anonymous namespace // Gets the singleton implementation of this class. Thread-compatible until @@ -45,10 +53,6 @@ GuardedPoolAllocator *GuardedPoolAllocator::getSingleton() { return SingletonPtr; } -static size_t roundUpTo(size_t Size, size_t Boundary) { - return (Size + Boundary - 1) & ~(Boundary - 1); -} - void GuardedPoolAllocator::init(const options::Options &Opts) { // Note: We return from the constructor here if GWP-ASan is not available. // This will stop heap-allocation of class members, as well as mmap() of the @@ -99,7 +103,7 @@ void GuardedPoolAllocator::init(const options::Options &Opts) { AdjustedSampleRatePlusOne = 2; initPRNG(); - ThreadLocals.NextSampleCounter = + getThreadLocals()->NextSampleCounter = ((getRandomUnsigned32() % (AdjustedSampleRatePlusOne - 1)) + 1) & ThreadLocalPackedVariables::NextSampleCounterMask; @@ -146,22 +150,18 @@ void GuardedPoolAllocator::uninitTestOnly() { } } -static uintptr_t getPageAddr(uintptr_t Ptr, uintptr_t PageSize) { - return Ptr & ~(PageSize - 1); -} - void *GuardedPoolAllocator::allocate(size_t Size) { // GuardedPagePoolEnd == 0 when GWP-ASan is disabled. If we are disabled, fall // back to the supporting allocator. if (State.GuardedPagePoolEnd == 0) { - ThreadLocals.NextSampleCounter = + getThreadLocals()->NextSampleCounter = (AdjustedSampleRatePlusOne - 1) & ThreadLocalPackedVariables::NextSampleCounterMask; return nullptr; } // Protect against recursivity. - if (ThreadLocals.RecursiveGuard) + if (getThreadLocals()->RecursiveGuard) return nullptr; ScopedRecursiveGuard SRG; @@ -212,7 +212,7 @@ void GuardedPoolAllocator::trapOnAddress(uintptr_t Address, Error E) { } void GuardedPoolAllocator::stop() { - ThreadLocals.RecursiveGuard = true; + getThreadLocals()->RecursiveGuard = true; PoolMutex.tryLock(); } @@ -243,7 +243,7 @@ void GuardedPoolAllocator::deallocate(void *Ptr) { // Ensure that the unwinder is not called if the recursive flag is set, // otherwise non-reentrant unwinders may deadlock. - if (!ThreadLocals.RecursiveGuard) { + if (!getThreadLocals()->RecursiveGuard) { ScopedRecursiveGuard SRG; Meta->DeallocationTrace.RecordBacktrace(Backtrace); } @@ -290,15 +290,11 @@ void GuardedPoolAllocator::freeSlot(size_t SlotIndex) { } uint32_t GuardedPoolAllocator::getRandomUnsigned32() { - uint32_t RandomState = ThreadLocals.RandomState; + uint32_t RandomState = getThreadLocals()->RandomState; RandomState ^= RandomState << 13; RandomState ^= RandomState >> 17; RandomState ^= RandomState << 5; - ThreadLocals.RandomState = RandomState; + getThreadLocals()->RandomState = RandomState; return RandomState; } - -GWP_ASAN_TLS_INITIAL_EXEC -GuardedPoolAllocator::ThreadLocalPackedVariables - GuardedPoolAllocator::ThreadLocals; } // namespace gwp_asan diff --git a/compiler-rt/lib/gwp_asan/guarded_pool_allocator.h b/compiler-rt/lib/gwp_asan/guarded_pool_allocator.h index 294a5b4..5ce70e4 100644 --- a/compiler-rt/lib/gwp_asan/guarded_pool_allocator.h +++ b/compiler-rt/lib/gwp_asan/guarded_pool_allocator.h @@ -13,6 +13,7 @@ #include "gwp_asan/definitions.h" #include "gwp_asan/mutex.h" #include "gwp_asan/options.h" +#include "gwp_asan/platform_specific/guarded_pool_allocator_tls.h" #include "gwp_asan/stack_trace_compressor.h" #include @@ -77,12 +78,12 @@ public: // class must be valid when zero-initialised, and we wish to sample as // infrequently as possible when this is the case, hence we underflow to // UINT32_MAX. - if (GWP_ASAN_UNLIKELY(ThreadLocals.NextSampleCounter == 0)) - ThreadLocals.NextSampleCounter = + if (GWP_ASAN_UNLIKELY(getThreadLocals()->NextSampleCounter == 0)) + getThreadLocals()->NextSampleCounter = ((getRandomUnsigned32() % (AdjustedSampleRatePlusOne - 1)) + 1) & ThreadLocalPackedVariables::NextSampleCounterMask; - return GWP_ASAN_UNLIKELY(--ThreadLocals.NextSampleCounter == 0); + return GWP_ASAN_UNLIKELY(--getThreadLocals()->NextSampleCounter == 0); } // Returns whether the provided pointer is a current sampled allocation that @@ -206,37 +207,10 @@ private: // the sample rate. uint32_t AdjustedSampleRatePlusOne = 0; - // Pack the thread local variables into a struct to ensure that they're in - // the same cache line for performance reasons. These are the most touched - // variables in GWP-ASan. - struct alignas(8) ThreadLocalPackedVariables { - constexpr ThreadLocalPackedVariables() - : RandomState(0xff82eb50), NextSampleCounter(0), RecursiveGuard(false) { - } - // Initialised to a magic constant so that an uninitialised GWP-ASan won't - // regenerate its sample counter for as long as possible. The xorshift32() - // algorithm used below results in getRandomUnsigned32(0xff82eb50) == - // 0xfffffea4. - uint32_t RandomState; - // Thread-local decrementing counter that indicates that a given allocation - // should be sampled when it reaches zero. - uint32_t NextSampleCounter : 31; - // The mask is needed to silence conversion errors. - static const uint32_t NextSampleCounterMask = (1U << 31) - 1; - // Guard against recursivity. Unwinders often contain complex behaviour that - // may not be safe for the allocator (i.e. the unwinder calls dlopen(), - // which calls malloc()). When recursive behaviour is detected, we will - // automatically fall back to the supporting allocator to supply the - // allocation. - bool RecursiveGuard : 1; - }; - static_assert(sizeof(ThreadLocalPackedVariables) == sizeof(uint64_t), - "thread local data does not fit in a uint64_t"); - class ScopedRecursiveGuard { public: - ScopedRecursiveGuard() { ThreadLocals.RecursiveGuard = true; } - ~ScopedRecursiveGuard() { ThreadLocals.RecursiveGuard = false; } + ScopedRecursiveGuard() { getThreadLocals()->RecursiveGuard = true; } + ~ScopedRecursiveGuard() { getThreadLocals()->RecursiveGuard = false; } }; // Initialise the PRNG, platform-specific. @@ -245,8 +219,6 @@ private: // xorshift (32-bit output), extremely fast PRNG that uses arithmetic // operations only. Seeded using platform-specific mechanisms by initPRNG(). uint32_t getRandomUnsigned32(); - - static GWP_ASAN_TLS_INITIAL_EXEC ThreadLocalPackedVariables ThreadLocals; }; } // namespace gwp_asan diff --git a/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp b/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp index 21b622c..dad749b 100644 --- a/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp +++ b/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp @@ -38,7 +38,7 @@ void MaybeSetMappingName(void *Mapping, size_t Size, const char *Name) { namespace gwp_asan { void GuardedPoolAllocator::initPRNG() { - ThreadLocals.RandomState = + getThreadLocals()->RandomState = static_cast(time(nullptr) + getThreadID()); } diff --git a/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_tls.h b/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_tls.h new file mode 100644 index 0000000..28b37e7 --- /dev/null +++ b/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_tls.h @@ -0,0 +1,53 @@ +//===-- guarded_pool_allocator_tls.h ----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef GWP_ASAN_GUARDED_POOL_ALLOCATOR_TLS_H_ +#define GWP_ASAN_GUARDED_POOL_ALLOCATOR_TLS_H_ + +#include "gwp_asan/definitions.h" + +#include + +namespace gwp_asan { +// Pack the thread local variables into a struct to ensure that they're in +// the same cache line for performance reasons. These are the most touched +// variables in GWP-ASan. +struct ThreadLocalPackedVariables { + constexpr ThreadLocalPackedVariables() + : RandomState(0xacd979ce), NextSampleCounter(0), RecursiveGuard(false) {} + // Initialised to a magic constant so that an uninitialised GWP-ASan won't + // regenerate its sample counter for as long as possible. The xorshift32() + // algorithm used below results in getRandomUnsigned32(0xacd979ce) == + // 0xfffffffe. + uint32_t RandomState; + // Thread-local decrementing counter that indicates that a given allocation + // should be sampled when it reaches zero. + uint32_t NextSampleCounter : 31; + // The mask is needed to silence conversion errors. + static const uint32_t NextSampleCounterMask = (1U << 31) - 1; + // Guard against recursivity. Unwinders often contain complex behaviour that + // may not be safe for the allocator (i.e. the unwinder calls dlopen(), + // which calls malloc()). When recursive behaviour is detected, we will + // automatically fall back to the supporting allocator to supply the + // allocation. + bool RecursiveGuard : 1; +}; +static_assert(sizeof(ThreadLocalPackedVariables) == sizeof(uint64_t), + "thread local data does not fit in a uint64_t"); + +#ifdef GWP_ASAN_PLATFORM_TLS_HEADER +#include GWP_ASAN_PLATFORM_TLS_HEADER +#else +inline ThreadLocalPackedVariables *getThreadLocals() { + alignas(8) static GWP_ASAN_TLS_INITIAL_EXEC ThreadLocalPackedVariables Locals; + return &Locals; +} +#endif +} // namespace gwp_asan + +#endif // GWP_ASAN_GUARDED_POOL_ALLOCATOR_TLS_H_ diff --git a/compiler-rt/lib/gwp_asan/platform_specific/utilities_posix.cpp b/compiler-rt/lib/gwp_asan/platform_specific/utilities_posix.cpp index 477a7ff..7ee7659 100644 --- a/compiler-rt/lib/gwp_asan/platform_specific/utilities_posix.cpp +++ b/compiler-rt/lib/gwp_asan/platform_specific/utilities_posix.cpp @@ -24,7 +24,7 @@ void die(const char *Message) { if (&android_set_abort_message != nullptr) android_set_abort_message(Message); abort(); -#else // __BIONIC__ +#else // __BIONIC__ fprintf(stderr, "%s", Message); __builtin_trap(); #endif // __BIONIC__ diff --git a/compiler-rt/lib/gwp_asan/utilities.cpp b/compiler-rt/lib/gwp_asan/utilities.cpp index 902e145..287630f 100644 --- a/compiler-rt/lib/gwp_asan/utilities.cpp +++ b/compiler-rt/lib/gwp_asan/utilities.cpp @@ -37,7 +37,7 @@ static size_t alignPowerOfTwo(size_t RealAllocationSize) { #ifdef __BIONIC__ static constexpr AlignmentStrategy PlatformDefaultAlignment = AlignmentStrategy::BIONIC; -#else // __BIONIC__ +#else // __BIONIC__ static constexpr AlignmentStrategy PlatformDefaultAlignment = AlignmentStrategy::POWER_OF_TWO; #endif // __BIONIC__ -- 2.7.4