hwasan: Read shadow address from ifunc if we don't need a frame record.

This saves a cbz+cold call in the interceptor ABI, as well as a realign
in both ABIs, trading off a dcache entry against some branch predictor
entries and some code size.

Unfortunately the functionality is hidden behind a flag because ifunc is
known to be broken on static binaries on Android.

Differential Revision: https://reviews.llvm.org/D57084

llvm-svn: 351989

diff --git a/compiler-rt/test/hwasan/lit.cfg b/compiler-rt/test/hwasan/lit.cfg
index 66008a632bcfbd619cd933b5c9e0b3508d605b45..0f1fdbd1ee47002df80fa03fc487eefc58e07a11 100644 (file)
--- a/compiler-rt/test/hwasan/lit.cfg
+++ b/compiler-rt/test/hwasan/lit.cfg
@@ -11,7 +11,8 @@ config.test_source_root = os.path.dirname(__file__)
 # Setup default compiler flags used with -fsanitize=memory option.
 clang_cflags = [config.target_cflags] + config.debug_info_flags
 clang_cxxflags = config.cxx_mode_flags + clang_cflags
-clang_hwasan_cflags = ["-fsanitize=hwaddress", "-mllvm", "-hwasan-generate-tags-with-calls"] + clang_cflags
+clang_hwasan_cflags = ["-fsanitize=hwaddress", "-mllvm", "-hwasan-generate-tags-with-calls",
+                       "-mllvm", "-hwasan-allow-ifunc"] + clang_cflags
 clang_hwasan_cxxflags = config.cxx_mode_flags + clang_hwasan_cflags
 
 def build_invocation(compile_flags):

diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index a00070e34aa7c228f8b5b38eb01c1228d53c138f..6170b792a68999f0b0b707dc57a25f4354605909 100644 (file)
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -160,6 +160,10 @@ static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
                                        cl::desc("inline all checks"),
                                        cl::Hidden, cl::init(false));
 
+static cl::opt<bool> ClAllowIfunc("hwasan-allow-ifunc",
+                                  cl::desc("allow the use of ifunc"),
+                                  cl::Hidden, cl::init(false));
+
 namespace {
 
 /// An instrumentation pass implementing detection of addressability bugs
@@ -183,6 +187,7 @@ public:
 
   void initializeCallbacks(Module &M);
 
+  Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
   Value *getDynamicShadowNonTls(IRBuilder<> &IRB);
 
   void untagPointerOperand(Instruction *I, Value *Addr);
@@ -384,9 +389,8 @@ void HWAddressSanitizer::initializeCallbacks(Module &M) {
   HwasanGenerateTagFunc = checkSanitizerInterfaceFunction(
       M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty));
 
-  if (Mapping.InGlobal)
-    ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
-                                       ArrayType::get(IRB.getInt8Ty(), 0));
+  ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
+                                     ArrayType::get(IRB.getInt8Ty(), 0));
 
   const std::string MemIntrinCallbackPrefix =
       CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix;
@@ -404,19 +408,23 @@ void HWAddressSanitizer::initializeCallbacks(Module &M) {
       M.getOrInsertFunction("__hwasan_thread_enter", IRB.getVoidTy()));
 }
 
+Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
+  // An empty inline asm with input reg == output reg.
+  // An opaque no-op cast, basically.
+  InlineAsm *Asm = InlineAsm::get(
+      FunctionType::get(Int8PtrTy, {ShadowGlobal->getType()}, false),
+      StringRef(""), StringRef("=r,0"),
+      /*hasSideEffects=*/false);
+  return IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow");
+}
+
 Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) {
   // Generate code only when dynamic addressing is needed.
   if (Mapping.Offset != kDynamicShadowSentinel)
     return nullptr;
 
   if (Mapping.InGlobal) {
-    // An empty inline asm with input reg == output reg.
-    // An opaque no-op cast, basically.
-    InlineAsm *Asm = InlineAsm::get(
-        FunctionType::get(Int8PtrTy, {ShadowGlobal->getType()}, false),
-        StringRef(""), StringRef("=r,0"),
-        /*hasSideEffects=*/false);
-    return IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow");
+    return getDynamicShadowIfunc(IRB);
   } else {
     Value *GlobalDynamicAddress =
         IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
@@ -828,6 +836,9 @@ Value *HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB,
   if (!Mapping.InTls)
     return getDynamicShadowNonTls(IRB);
 
+  if (ClAllowIfunc && !WithFrameRecord && TargetTriple.isAndroid())
+    return getDynamicShadowIfunc(IRB);
+
   Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);
   assert(SlotPtr);
 

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/lazy-thread-init.ll b/llvm/test/Instrumentation/HWAddressSanitizer/lazy-thread-init.ll
index dab62d6f6b24b758151ea29c80af272513c4d6bd..b02d0f88a0f8c6c943a94aa1d4a4f19869c6d8ab 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/lazy-thread-init.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/lazy-thread-init.ll
@@ -1,17 +1,20 @@
-; RUN: opt -S -hwasan < %s | FileCheck %s
+; RUN: opt -S -hwasan -hwasan-allow-ifunc < %s | FileCheck %s
 
-target triple = "x86_64-unknown-linux-gnu"
+target triple = "aarch64--linux-android"
 
 declare void @bar([16 x i32]* %p)
 
-define void @foo() sanitize_hwaddress "hwasan-abi"="interceptor" {
-  ; CHECK: [[LOAD:%[^ ]*]] = load i64, i64* @__hwasan_tls
+define void @alloca() sanitize_hwaddress "hwasan-abi"="interceptor" {
+  ; CHECK: [[A:%[^ ]*]] = call i8* @llvm.thread.pointer()
+  ; CHECK: [[B:%[^ ]*]] = getelementptr i8, i8* [[A]], i32 48
+  ; CHECK: [[C:%[^ ]*]] = bitcast i8* [[B]] to i64*
+  ; CHECK: [[LOAD:%[^ ]*]] = load i64, i64* [[C]]
   ; CHECK: [[ICMP:%[^ ]*]] = icmp eq i64 [[LOAD]], 0
   ; CHECK: br i1 [[ICMP]], label %[[INIT:[^,]*]], label %[[CONT:[^,]*]], !prof [[PROF:![0-9]+]]
 
   ; CHECK: [[INIT]]:
   ; CHECK: call void @__hwasan_thread_enter()
-  ; CHECK: [[RELOAD:%[^ ]*]] = load i64, i64* @__hwasan_tls
+  ; CHECK: [[RELOAD:%[^ ]*]] = load i64, i64* [[C]]
   ; CHECK: br label %[[CONT]]
 
   ; CHECK: [[CONT]]:
@@ -22,4 +25,12 @@ define void @foo() sanitize_hwaddress "hwasan-abi"="interceptor" {
   ret void
 }
 
+define i32 @load(i32* %p) sanitize_hwaddress "hwasan-abi"="interceptor" {
+  ; CHECK: [[SHADOW:%[^ ]*]] = call i8* asm "", "=r,0"([0 x i8]* @__hwasan_shadow)
+  ; CHECK-NOT: icmp
+  ; CHECK: call void @llvm.hwasan.check.memaccess(i8* [[SHADOW]],
+  %v = load i32, i32* %p
+  ret i32 %v
+}
+
 ; CHECK: [[PROF]] = !{!"branch_weights", i32 1, i32 100000}

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/prologue.ll b/llvm/test/Instrumentation/HWAddressSanitizer/prologue.ll
index 5cbfa67420602985157ebfb27eb4b840d31e1901..1163dbe72debad7b8384bd019d9ce37d907ead87 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/prologue.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/prologue.ll
@@ -1,15 +1,15 @@
 ; Test -hwasan-with-ifunc flag.
 ;
-; RUN: opt -hwasan -S < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-TLS,CHECK-HISTORY
-; RUN: opt -hwasan -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=1 < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-TLS,CHECK-HISTORY
-; RUN: opt -hwasan -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=0 < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-TLS,CHECK-NOHISTORY
-; RUN: opt -hwasan -S -hwasan-with-ifunc=0 -hwasan-with-tls=0 < %s | \
+; RUN: opt -hwasan -hwasan-allow-ifunc -S < %s | \
+; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-NOGLOBAL,CHECK-TLS,CHECK-HISTORY
+; RUN: opt -hwasan -hwasan-allow-ifunc -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=1 < %s | \
+; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-NOGLOBAL,CHECK-TLS,CHECK-HISTORY
+; RUN: opt -hwasan -hwasan-allow-ifunc -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=0 < %s | \
+; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-NOGLOBAL,CHECK-IFUNC,CHECK-NOHISTORY
+; RUN: opt -hwasan -hwasan-allow-ifunc -S -hwasan-with-ifunc=0 -hwasan-with-tls=0 < %s | \
 ; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-GLOBAL,CHECK-NOHISTORY
-; RUN: opt -hwasan -S -hwasan-with-ifunc=1  -hwasan-with-tls=0 < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-IFUNC,CHECK-NOHISTORY
+; RUN: opt -hwasan -hwasan-allow-ifunc -S -hwasan-with-ifunc=1  -hwasan-with-tls=0 < %s | \
+; RUN:     FileCheck %s --check-prefixes=CHECK,CHECk-NOGLOBAL,CHECK-IFUNC,CHECK-NOHISTORY
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android22"
@@ -23,18 +23,11 @@ define i32 @test_load(i32* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load
 ; CHECK: entry:
 
-; CHECK-IFUNC:   %[[A:[^ ]*]] = call i8* asm "", "=r,0"([0 x i8]* @__hwasan_shadow)
-; CHECK-IFUNC:   @llvm.hwasan.check.memaccess(i8* %[[A]]
+; CHECK-NOGLOBAL:   %[[A:[^ ]*]] = call i8* asm "", "=r,0"([0 x i8]* @__hwasan_shadow)
+; CHECK-NOGLOBAL:   @llvm.hwasan.check.memaccess(i8* %[[A]]
 
 ; CHECK-GLOBAL: load i8*, i8** @__hwasan_shadow_memory_dynamic_address
 
-; CHECK-TLS:   %[[A:[^ ]*]] = call i8* @llvm.thread.pointer()
-; CHECK-TLS:   %[[B:[^ ]*]] = getelementptr i8, i8* %[[A]], i32 48
-; CHECK-TLS:   %[[C:[^ ]*]] = bitcast i8* %[[B]] to i64*
-; CHECK-TLS:   %[[D:[^ ]*]] = load i64, i64* %[[C]]
-; CHECK-TLS:   %[[E:[^ ]*]] = or i64 %[[D]], 4294967295
-; CHECK-TLS:   = add i64 %[[E]], 1
-
 ; "store i64" is only used to update stack history (this input IR intentionally does not use any i64)
 ; W/o any allocas, the history is not updated, even if it is enabled explicitly with -hwasan-record-stack-history=1
 ; CHECK-NOT: store i64