[test][HWASAN] Regenerate some HWASAN tests

Example of the command used to update tests:
```
ninja -C <build_dir> opt && llvm/utils/update_test_checks.py \
  --opt-binary <build_dir>/bin/opt llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca.ll
```

Reviewed By: kstoimenov

Differential Revision: https://reviews.llvm.org/D149219

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/alloca-with-calls.ll b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/alloca-with-calls.ll
index e8f1ca4..5338d16 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/alloca-with-calls.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/alloca-with-calls.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test alloca instrumentation when tags are generated by HWASan function.
 ;
 ; RUN: opt < %s -passes=hwasan -hwasan-generate-tags-with-calls -S | FileCheck %s
@@ -8,12 +9,49 @@ target triple = "riscv64-unknown-linux"
 declare void @use32(ptr)
 
 define void @test_alloca() sanitize_hwaddress {
-; CHECK-LABEL: @test_alloca(
-; CHECK: %[[T1:[^ ]*]] = call i8 @__hwasan_generate_tag()
-; CHECK: %[[A:[^ ]*]] = zext i8 %[[T1]] to i64
-; CHECK: %[[B:[^ ]*]] = ptrtoint ptr %x to i64
-; CHECK: %[[C:[^ ]*]] = shl i64 %[[A]], 56
-; CHECK: or i64 %[[B]], %[[C]]
+; CHECK-LABEL: define void @test_alloca
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 44
+; CHECK-NEXT:    [[TMP6:%.*]] = or i64 ptrtoint (ptr @test_alloca to i64), [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    store i64 [[TMP6]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = ashr i64 [[TMP0]], 56
+; CHECK-NEXT:    [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 12
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], -1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP12:%.*]] = and i64 [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    store i64 [[TMP12]], ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP13]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP15:%.*]] = call i8 @__hwasan_generate_tag()
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i8 [[TMP15]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = shl i64 [[TMP16]], 56
+; CHECK-NEXT:    [[TMP19:%.*]] = or i64 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP19]] to ptr
+; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP16]] to i8
+; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = lshr i64 [[TMP21]], 4
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP23]], i32 0
+; CHECK-NEXT:    store i8 4, ptr [[TMP24]], align 1
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[X]], i32 15
+; CHECK-NEXT:    store i8 [[TMP20]], ptr [[TMP25]], align 1
+; CHECK-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]])
+; CHECK-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP27:%.*]] = lshr i64 [[TMP26]], 4
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP27]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP28]], i8 0, i64 1, i1 false)
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %x = alloca i32, align 4

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/alloca.ll b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/alloca.ll
index 488b8bd..42effd4 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/alloca.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/alloca.ll
@@ -1,8 +1,9 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test alloca instrumentation.
 ;
-; RUN: opt < %s -passes=hwasan -hwasan-with-ifunc=1 -S | FileCheck %s --check-prefixes=CHECK,DYNAMIC-SHADOW,NO-UAR-TAGS
-; RUN: opt < %s -passes=hwasan -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,ZERO-BASED-SHADOW,NO-UAR-TAGS
-; RUN: opt < %s -passes=hwasan -hwasan-with-ifunc=1 -hwasan-uar-retag-to-zero=0 -S | FileCheck %s --check-prefixes=CHECK,DYNAMIC-SHADOW,UAR-TAGS
+; RUN: opt < %s -passes=hwasan -hwasan-with-ifunc=1 -S | FileCheck %s --check-prefixes=DYNAMIC-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=ZERO-BASED-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-with-ifunc=1 -hwasan-uar-retag-to-zero=0 -S | FileCheck %s --check-prefixes=DYNAMIC-SHADOW-UAR-TAGS
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "riscv64-unknown-linux"
@@ -10,44 +11,98 @@ target triple = "riscv64-unknown-linux"
 declare void @use32(ptr)
 
 define void @test_alloca() sanitize_hwaddress !dbg !15 {
-; CHECK-LABEL: @test_alloca(
-; CHECK: %[[FP:[^ ]*]] = call ptr @llvm.frameaddress.p0(i32 0)
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %[[FP]] to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 20
-; CHECK: %[[BASE_TAG:[^ ]*]] = xor i64 %[[A]], %[[B]]
-
-; CHECK: %[[X:[^ ]*]] = alloca { i32, [12 x i8] }, align 16
-; CHECK: %[[X_TAG:[^ ]*]] = xor i64 %[[BASE_TAG]], 0
-; CHECK: %[[X1:[^ ]*]] = ptrtoint ptr %[[X]] to i64
-; CHECK: %[[C:[^ ]*]] = shl i64 %[[X_TAG]], 56
-; CHECK: %[[D:[^ ]*]] = or i64 %[[X1]], %[[C]]
-; CHECK: %[[X_HWASAN:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-
-; CHECK: %[[X_TAG2:[^ ]*]] = trunc i64 %[[X_TAG]] to i8
-; CHECK: %[[E:[^ ]*]] = ptrtoint ptr %[[X]] to i64
-; CHECK: %[[F:[^ ]*]] = lshr i64 %[[E]], 4
-; DYNAMIC-SHADOW: %[[X_SHADOW:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %[[F]]
-; ZERO-BASED-SHADOW: %[[X_SHADOW:[^ ]*]] = inttoptr i64 %[[F]] to ptr
-; CHECK: %[[X_SHADOW_GEP:[^ ]*]] = getelementptr i8, ptr %[[X_SHADOW]], i32 0
-; CHECK: store i8 4, ptr %[[X_SHADOW_GEP]]
-; CHECK: %[[X_I8_GEP:[^ ]*]] = getelementptr i8, ptr %[[X]], i32 15
-; CHECK: store i8 %[[X_TAG2]], ptr %[[X_I8_GEP]]
-; CHECK: call void @llvm.dbg.value(
-; CHECK-SAME: metadata !DIArgList(ptr %[[X]], ptr %[[X]])
-; CHECK-SAME: metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_tag_offset, 0, DW_OP_LLVM_arg, 1, DW_OP_LLVM_tag_offset, 0,
-; CHECK: call void @use32(ptr nonnull %[[X_HWASAN]])
-
-; UAR-TAGS: %[[BASE_TAG_COMPL:[^ ]*]] = xor i64 %[[BASE_TAG]], 255
-; UAR-TAGS: %[[X_TAG_UAR:[^ ]*]] = trunc i64 %[[BASE_TAG_COMPL]] to i8
-; CHECK: %[[E2:[^ ]*]] = ptrtoint ptr %[[X]] to i64
-; CHECK: %[[F2:[^ ]*]] = lshr i64 %[[E2]], 4
-; DYNAMIC-SHADOW: %[[X_SHADOW2:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %[[F2]]
-; ZERO-BASED-SHADOW: %[[X_SHADOW2:[^ ]*]] = inttoptr i64 %[[F2]] to ptr
-; NO-UAR-TAGS: call void @llvm.memset.p0.i64(ptr align 1 %[[X_SHADOW2]], i8 0, i64 1, i1 false)
-; UAR-TAGS: call void @llvm.memset.p0.i64(ptr align 1 %[[X_SHADOW2]], i8 %[[X_TAG_UAR]], i64 1, i1 false)
-; CHECK: ret void
-
-
+; DYNAMIC-SHADOW-LABEL: define void @test_alloca
+; DYNAMIC-SHADOW-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk !dbg [[DBG7:![0-9]+]] {
+; DYNAMIC-SHADOW-NEXT:  entry:
+; DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; DYNAMIC-SHADOW-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; DYNAMIC-SHADOW-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0, !dbg [[DBG10:![0-9]+]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP3]], 56, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = or i64 [[TMP4]], [[TMP5]], !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP6]] to ptr, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i8, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = lshr i64 [[TMP8]], 4, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP9]], !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i32 0, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    store i8 4, ptr [[TMP11]], align 1, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[X]], i32 15, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    store i8 [[TMP7]], ptr [[TMP12]], align 1, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    call void @llvm.dbg.value(metadata !DIArgList(ptr [[X]], ptr [[X]]), metadata [[META11:![0-9]+]], metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_tag_offset, 0, DW_OP_LLVM_arg, 1, DW_OP_LLVM_tag_offset, 0, DW_OP_plus, DW_OP_deref)), !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]]), !dbg [[DBG13:![0-9]+]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG14:![0-9]+]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = lshr i64 [[TMP13]], 4, !dbg [[DBG14]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP14]], !dbg [[DBG14]]
+; DYNAMIC-SHADOW-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP15]], i8 0, i64 1, i1 false), !dbg [[DBG14]]
+; DYNAMIC-SHADOW-NEXT:    ret void, !dbg [[DBG14]]
+;
+; ZERO-BASED-SHADOW-LABEL: define void @test_alloca
+; ZERO-BASED-SHADOW-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk !dbg [[DBG7:![0-9]+]] {
+; ZERO-BASED-SHADOW-NEXT:  entry:
+; ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; ZERO-BASED-SHADOW-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; ZERO-BASED-SHADOW-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0, !dbg [[DBG10:![0-9]+]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP3]], 56, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = or i64 [[TMP4]], [[TMP5]], !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP6]] to ptr, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i8, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = lshr i64 [[TMP8]], 4, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i32 0, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    store i8 4, ptr [[TMP11]], align 1, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[X]], i32 15, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    store i8 [[TMP7]], ptr [[TMP12]], align 1, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    call void @llvm.dbg.value(metadata !DIArgList(ptr [[X]], ptr [[X]]), metadata [[META11:![0-9]+]], metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_tag_offset, 0, DW_OP_LLVM_arg, 1, DW_OP_LLVM_tag_offset, 0, DW_OP_plus, DW_OP_deref)), !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]]), !dbg [[DBG13:![0-9]+]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG14:![0-9]+]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = lshr i64 [[TMP13]], 4, !dbg [[DBG14]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr, !dbg [[DBG14]]
+; ZERO-BASED-SHADOW-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP15]], i8 0, i64 1, i1 false), !dbg [[DBG14]]
+; ZERO-BASED-SHADOW-NEXT:    ret void, !dbg [[DBG14]]
+;
+; DYNAMIC-SHADOW-UAR-TAGS-LABEL: define void @test_alloca
+; DYNAMIC-SHADOW-UAR-TAGS-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk !dbg [[DBG7:![0-9]+]] {
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:  entry:
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP3:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0, !dbg [[DBG10:![0-9]+]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP3]], 56, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP6:%.*]] = or i64 [[TMP4]], [[TMP5]], !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP6]] to ptr, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i8, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP9:%.*]] = lshr i64 [[TMP8]], 4, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP9]], !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i32 0, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    store i8 4, ptr [[TMP11]], align 1, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[X]], i32 15, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    store i8 [[TMP7]], ptr [[TMP12]], align 1, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    call void @llvm.dbg.value(metadata !DIArgList(ptr [[X]], ptr [[X]]), metadata [[META11:![0-9]+]], metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_tag_offset, 0, DW_OP_LLVM_arg, 1, DW_OP_LLVM_tag_offset, 0, DW_OP_plus, DW_OP_deref)), !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]]), !dbg [[DBG13:![0-9]+]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP13:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 255, !dbg [[DBG14:![0-9]+]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP13]] to i8, !dbg [[DBG14]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG14]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP16:%.*]] = lshr i64 [[TMP15]], 4, !dbg [[DBG14]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP16]], !dbg [[DBG14]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP17]], i8 [[TMP14]], i64 1, i1 false), !dbg [[DBG14]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    ret void, !dbg [[DBG14]]
+;
 entry:
   %x = alloca i32, align 4
   call void @llvm.dbg.value(metadata !DIArgList(ptr %x, ptr %x), metadata !22, metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_plus, DW_OP_deref)), !dbg !21

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/atomic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/atomic.ll
index 47ad906..0e83b42 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/atomic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/atomic.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test basic address sanitizer instrumentation.
 ;
 ; RUN: opt < %s -passes=hwasan -S | FileCheck %s
@@ -6,10 +7,18 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "riscv64-unknown-linux"
 
 define void @atomicrmw(ptr %ptr) sanitize_hwaddress {
-; CHECK-LABEL: @atomicrmw(
-; CHECK: call void @llvm.hwasan.check.memaccess.shortgranules({{.*}}, ptr %ptr, i32 19)
-; CHECK: atomicrmw add ptr %ptr, i64 1 seq_cst
-; CHECK: ret void
+; CHECK-LABEL: define void @atomicrmw
+; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[PTR]], i32 19)
+; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw add ptr [[PTR]], i64 1 seq_cst, align 8
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %0 = atomicrmw add ptr %ptr, i64 1 seq_cst
@@ -17,10 +26,18 @@ entry:
 }
 
 define void @cmpxchg(ptr %ptr, i64 %compare_to, i64 %new_value) sanitize_hwaddress {
-; CHECK-LABEL: @cmpxchg(
-; CHECK: call void @llvm.hwasan.check.memaccess.shortgranules({{.*}}, ptr %ptr, i32 19)
-; CHECK: cmpxchg ptr %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst
-; CHECK: ret void
+; CHECK-LABEL: define void @cmpxchg
+; CHECK-SAME: (ptr [[PTR:%.*]], i64 [[COMPARE_TO:%.*]], i64 [[NEW_VALUE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP3]], ptr [[PTR]], i32 19)
+; CHECK-NEXT:    [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[COMPARE_TO]], i64 [[NEW_VALUE]] seq_cst seq_cst, align 8
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %0 = cmpxchg ptr %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/basic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/basic.ll
index cb3667b..12822d8 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/basic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/basic.ll
@@ -1,9 +1,10 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test basic address sanitizer instrumentation.
 ;
-; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=CHECK,ABORT
-; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,RECOVER-DYNAMIC-SHADOW
-; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,ABORT
-; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,RECOVER-ZERO-BASED-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=ABORT-DYNAMIC-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=RECOVER-DYNAMIC-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=ABORT-ZERO-BASED-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=RECOVER-ZERO-BASED-SHADOW
 
 ; CHECK: @llvm.used = appending global [1 x ptr] [ptr @hwasan.module_ctor]
 ; CHECK: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 0, ptr @hwasan.module_ctor, ptr @hwasan.module_ctor }]
@@ -12,49 +13,104 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "riscv64-unknown-linux"
 
 define i8 @test_load8(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load8(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label %[[MISMATCH:[0-9]*]], label %[[CONT:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[MISMATCH]]:
-; RECOVER: %[[NOTSHORT:[^ ]*]] = icmp ugt i8 %[[MEMTAG]], 15
-; RECOVER: br i1 %[[NOTSHORT]], label %[[FAIL:[0-9]*]], label %[[SHORT:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[FAIL]]:
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 96", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; RECOVER: [[SHORT]]:
-; RECOVER: %[[LOWBITS:[^ ]*]] = and i64 %[[A]], 15
-; RECOVER: %[[LOWBITS_I8:[^ ]*]] = trunc i64 %[[LOWBITS]] to i8
-; RECOVER: %[[LAST:[^ ]*]] = add i8 %[[LOWBITS_I8]], 0
-; RECOVER: %[[OOB:[^ ]*]] = icmp uge i8 %[[LAST]], %[[MEMTAG]]
-; RECOVER: br i1 %[[OOB]], label %[[FAIL]], label %[[INBOUNDS:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[INBOUNDS]]:
-; RECOVER: %[[EOG_ADDR:[^ ]*]] = or i64 %[[C]], 15
-; RECOVER: %[[EOG_PTR:[^ ]*]] = inttoptr i64 %[[EOG_ADDR]] to ptr
-; RECOVER: %[[EOGTAG:[^ ]*]] = load i8, ptr %[[EOG_PTR]]
-; RECOVER: %[[EOG_MISMATCH:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[EOGTAG]]
-; RECOVER: br i1 %[[EOG_MISMATCH]], label %[[FAIL]], label %[[CONT1:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[CONT1]]:
-; RECOVER: br label %[[CONT]]
-
-; RECOVER: [[CONT]]:
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 0)
-
-; CHECK: %[[G:[^ ]*]] = load i8, ptr %a, align 4
-; CHECK: ret i8 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load8
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load8
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1:![0-9]+]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 96", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load8
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load8
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1:![0-9]+]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 96", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+
+
+
+
+
+
+
+
 
 entry:
   %b = load i8, ptr %a, align 4
@@ -62,49 +118,104 @@ entry:
 }
 
 define i16 @test_load16(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load16(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label %[[MISMATCH:[0-9]*]], label %[[CONT:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[MISMATCH]]:
-; RECOVER: %[[NOTSHORT:[^ ]*]] = icmp ugt i8 %[[MEMTAG]], 15
-; RECOVER: br i1 %[[NOTSHORT]], label %[[FAIL:[0-9]*]], label %[[SHORT:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[FAIL]]:
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 97", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; RECOVER: [[SHORT]]:
-; RECOVER: %[[LOWBITS:[^ ]*]] = and i64 %[[A]], 15
-; RECOVER: %[[LOWBITS_I8:[^ ]*]] = trunc i64 %[[LOWBITS]] to i8
-; RECOVER: %[[LAST:[^ ]*]] = add i8 %[[LOWBITS_I8]], 1
-; RECOVER: %[[OOB:[^ ]*]] = icmp uge i8 %[[LAST]], %[[MEMTAG]]
-; RECOVER: br i1 %[[OOB]], label %[[FAIL]], label %[[INBOUNDS:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[INBOUNDS]]:
-; RECOVER: %[[EOG_ADDR:[^ ]*]] = or i64 %[[C]], 15
-; RECOVER: %[[EOG_PTR:[^ ]*]] = inttoptr i64 %[[EOG_ADDR]] to ptr
-; RECOVER: %[[EOGTAG:[^ ]*]] = load i8, ptr %[[EOG_PTR]]
-; RECOVER: %[[EOG_MISMATCH:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[EOGTAG]]
-; RECOVER: br i1 %[[EOG_MISMATCH]], label %[[FAIL]], label %[[CONT1:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[CONT1]]:
-; RECOVER: br label %[[CONT]]
-
-; RECOVER: [[CONT]]:
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 1)
-
-; CHECK: %[[G:[^ ]*]] = load i16, ptr %a, align 4
-; CHECK: ret i16 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i16 @test_load16
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i16 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i16 @test_load16
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 97", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i16 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i16 @test_load16
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i16 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i16 @test_load16
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 97", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i16 [[B]]
+;
+
+
+
+
+
+
+
+
 
 entry:
   %b = load i16, ptr %a, align 4
@@ -112,25 +223,99 @@ entry:
 }
 
 define i32 @test_load32(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load32(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 98", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 2)
-
-; CHECK: %[[G:[^ ]*]] = load i32, ptr %a, align 4
-; CHECK: ret i32 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i32 @test_load32
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i32 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i32 @test_load32
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 98", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i32 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i32 @test_load32
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i32 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i32 @test_load32
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 98", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i32 [[B]]
+;
+
+
+
 
 entry:
   %b = load i32, ptr %a, align 4
@@ -138,25 +323,99 @@ entry:
 }
 
 define i64 @test_load64(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load64(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 99", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 3)
-
-; CHECK: %[[G:[^ ]*]] = load i64, ptr %a, align 8
-; CHECK: ret i64 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i64 @test_load64
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i64 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i64 @test_load64
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 99", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i64 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i64 @test_load64
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i64 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i64 @test_load64
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 99", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i64 [[B]]
+;
+
+
+
 
 entry:
   %b = load i64, ptr %a, align 8
@@ -164,25 +423,99 @@ entry:
 }
 
 define i128 @test_load128(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load128(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 100", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 4)
-
-; CHECK: %[[G:[^ ]*]] = load i128, ptr %a, align 16
-; CHECK: ret i128 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i128 @test_load128
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i128 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i128 @test_load128
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 100", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i128 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i128 @test_load128
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i128 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i128 @test_load128
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 100", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i128 [[B]]
+;
+
+
+
 
 entry:
   %b = load i128, ptr %a, align 16
@@ -190,12 +523,42 @@ entry:
 }
 
 define i40 @test_load40(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load40(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; ABORT: call void @__hwasan_loadN(i64 %[[A]], i64 5)
-; RECOVER: call void @__hwasan_loadN_noabort(i64 %[[A]], i64 5)
-; CHECK: %[[B:[^ ]*]] = load i40, ptr %a
-; CHECK: ret i40 %[[B]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i40 @test_load40
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i40 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i40 @test_load40
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_loadN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i40 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i40 @test_load40
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i40 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i40 @test_load40
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_loadN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i40 [[B]]
+;
 
 entry:
   %b = load i40, ptr %a, align 4
@@ -203,25 +566,99 @@ entry:
 }
 
 define void @test_store8(ptr %a, i8 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store8(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 112", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 16)
-
-; CHECK: store i8 %b, ptr %a, align 4
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store8
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store8
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 112", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store8
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store8
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 112", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i8 %b, ptr %a, align 4
@@ -229,25 +666,99 @@ entry:
 }
 
 define void @test_store16(ptr %a, i16 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store16(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 113", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 17)
-
-; CHECK: store i16 %b, ptr %a, align 4
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store16
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store16
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 113", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store16
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store16
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 113", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i16 %b, ptr %a, align 4
@@ -255,25 +766,99 @@ entry:
 }
 
 define void @test_store32(ptr %a, i32 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store32(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 114", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 18)
-
-; CHECK: store i32 %b, ptr %a, align 4
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store32
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store32
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 114", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store32
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store32
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 114", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i32 %b, ptr %a, align 4
@@ -281,25 +866,99 @@ entry:
 }
 
 define void @test_store64(ptr %a, i64 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store64(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 115", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 19)
-
-; CHECK: store i64 %b, ptr %a, align 8
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store64
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store64
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 115", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store64
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store64
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 115", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i64 %b, ptr %a, align 8
@@ -307,25 +966,99 @@ entry:
 }
 
 define void @test_store128(ptr %a, i128 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store128(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "ebreak\0Aaddiw x0, x11, 116", "{x10}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 20)
-
-; CHECK: store i128 %b, ptr %a, align 16
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store128
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store128
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 116", "{x10}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store128
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store128
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "ebreak\0Aaddiw x0, x11, 116", "{x10}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i128 %b, ptr %a, align 16
@@ -333,12 +1066,42 @@ entry:
 }
 
 define void @test_store40(ptr %a, i40 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store40(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 5)
-; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 5)
-; CHECK: store i40 %b, ptr %a
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store40
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store40
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store40
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store40
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
 
 entry:
   store i40 %b, ptr %a, align 4
@@ -346,12 +1109,42 @@ entry:
 }
 
 define void @test_store_unaligned(ptr %a, i64 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store_unaligned(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 8)
-; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 8)
-; CHECK: store i64 %b, ptr %a, align 4
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store_unaligned
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store_unaligned
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 8)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store_unaligned
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store_unaligned
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 8)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
 
 entry:
   store i64 %b, ptr %a, align 4
@@ -359,33 +1152,108 @@ entry:
 }
 
 define i8 @test_load_noattr(ptr %a) {
-; CHECK-LABEL: @test_load_noattr(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: %[[B:[^ ]*]] = load i8, ptr %a
-; CHECK-NEXT: ret i8 %[[B]]
-
+; CHECK-LABEL: define i8 @test_load_noattr
+; CHECK-SAME: (ptr [[A:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; CHECK-NEXT:    ret i8 [[B]]
+;
+; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_noattr
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_noattr
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_noattr
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_noattr
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
 entry:
   %b = load i8, ptr %a, align 4
   ret i8 %b
 }
 
 define i8 @test_load_notmyattr(ptr %a) sanitize_address {
-; CHECK-LABEL: @test_load_notmyattr(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: %[[B:[^ ]*]] = load i8, ptr %a
-; CHECK-NEXT: ret i8 %[[B]]
-
+; CHECK-LABEL: define i8 @test_load_notmyattr
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; CHECK-NEXT:    ret i8 [[B]]
+;
+; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_notmyattr
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_notmyattr
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_notmyattr
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_notmyattr
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
 entry:
   %b = load i8, ptr %a, align 4
   ret i8 %b
 }
 
 define i8 @test_load_addrspace(ptr addrspace(256) %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load_addrspace(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: %[[B:[^ ]*]] = load i8, ptr addrspace(256) %a
-; CHECK-NEXT: ret i8 %[[B]]
-
+; CHECK-LABEL: define i8 @test_load_addrspace
+; CHECK-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; CHECK-NEXT:    ret i8 [[B]]
+;
+; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_addrspace
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_addrspace
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_addrspace
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_addrspace
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
 entry:
   %b = load i8, ptr addrspace(256) %a, align 4
   ret i8 %b
@@ -393,9 +1261,7 @@ entry:
 
 ; CHECK: declare void @__hwasan_init()
 
-; CHECK:      define internal void @hwasan.module_ctor() #[[#ATTR:]] comdat {
 ; CHECK-NEXT:   call void @__hwasan_init()
 ; CHECK-NEXT:   ret void
 ; CHECK-NEXT: }
 
-; CHECK:      attributes #[[#ATTR]] = { nounwind }

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/exception-lifetime.ll b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/exception-lifetime.ll
index 7b51acd..c5d4ddf 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/exception-lifetime.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/exception-lifetime.ll
@@ -1,7 +1,8 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test allocas with multiple lifetime ends, as frequently seen for exception
 ; handling.
 ;
-; RUN: opt -passes=hwasan -hwasan-use-after-scope -S -o - %s | FileCheck %s --check-prefix=CHECK
+; RUN: opt -passes=hwasan -hwasan-use-after-scope -S -o - %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "riscv64-unknown-linux"
@@ -14,6 +15,78 @@ declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
 declare i32 @__gxx_personality_v0(...)
 
 define void @test() sanitize_hwaddress personality ptr @__gxx_personality_v0 {
+; CHECK-LABEL: define void @test
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk.__gxx_personality_v0 {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 44
+; CHECK-NEXT:    [[TMP6:%.*]] = or i64 ptrtoint (ptr @test to i64), [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    store i64 [[TMP6]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = ashr i64 [[TMP0]], 56
+; CHECK-NEXT:    [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 12
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], -1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP12:%.*]] = and i64 [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    store i64 [[TMP12]], ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP13]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = shl i64 [[TMP15]], 56
+; CHECK-NEXT:    [[TMP18:%.*]] = or i64 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-NEXT:    [[EXN_SLOT:%.*]] = alloca ptr, align 8
+; CHECK-NEXT:    [[EHSELECTOR_SLOT:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP15]] to i8
+; CHECK-NEXT:    [[TMP20:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP21:%.*]] = lshr i64 [[TMP20]], 4
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr i8, ptr [[TMP22]], i32 0
+; CHECK-NEXT:    store i8 4, ptr [[TMP23]], align 1
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[X]], i32 15
+; CHECK-NEXT:    store i8 [[TMP19]], ptr [[TMP24]], align 1
+; CHECK-NEXT:    invoke void @mayFail(ptr [[X_HWASAN]])
+; CHECK-NEXT:    to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]]
+; CHECK:       invoke.cont:
+; CHECK-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP26]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP27]], i8 0, i64 1, i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    ret void
+; CHECK:       lpad:
+; CHECK-NEXT:    [[TMP28:%.*]] = landingpad { ptr, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    [[TMP29:%.*]] = extractvalue { ptr, i32 } [[TMP28]], 0
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP14]], ptr [[EXN_SLOT]], i32 19)
+; CHECK-NEXT:    store ptr [[TMP29]], ptr [[EXN_SLOT]], align 8
+; CHECK-NEXT:    [[TMP30:%.*]] = extractvalue { ptr, i32 } [[TMP28]], 1
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP14]], ptr [[EHSELECTOR_SLOT]], i32 18)
+; CHECK-NEXT:    store i32 [[TMP30]], ptr [[EHSELECTOR_SLOT]], align 4
+; CHECK-NEXT:    call void @onExcept(ptr [[X_HWASAN]])
+; CHECK-NEXT:    [[TMP31:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP32:%.*]] = lshr i64 [[TMP31]], 4
+; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP32]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP33]], i8 0, i64 1, i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    br label [[EH_RESUME:%.*]]
+; CHECK:       eh.resume:
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP14]], ptr [[EXN_SLOT]], i32 3)
+; CHECK-NEXT:    [[EXN:%.*]] = load ptr, ptr [[EXN_SLOT]], align 8
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP14]], ptr [[EHSELECTOR_SLOT]], i32 2)
+; CHECK-NEXT:    [[SEL:%.*]] = load i32, ptr [[EHSELECTOR_SLOT]], align 4
+; CHECK-NEXT:    [[LPAD_VAL:%.*]] = insertvalue { ptr, i32 } undef, ptr [[EXN]], 0
+; CHECK-NEXT:    [[LPAD_VAL1:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1
+; CHECK-NEXT:    resume { ptr, i32 } [[LPAD_VAL1]]
+;
 entry:
   %x = alloca i32, align 8
   %exn.slot = alloca ptr, align 8
@@ -22,19 +95,11 @@ entry:
   invoke void @mayFail(ptr %x) to label %invoke.cont unwind label %lpad
 
 invoke.cont:                                      ; preds = %entry
-; CHECK: invoke.cont:
-; CHECK:  call void @llvm.memset.p0.i64(ptr align 1 %{{.*}}, i8 0, i64 1, i1 false)
-; CHECK:  call void @llvm.lifetime.end.p0(i64 16, ptr {{.*}}{{.*}}%x)
-; CHECK:  ret void
 
   call void @llvm.lifetime.end.p0(i64 8, ptr %x)
   ret void
 
 lpad:                                             ; preds = %entry
-; CHECK: lpad
-; CHECK:  call void @llvm.memset.p0.i64(ptr align 1 %{{.*}}, i8 0, i64 1, i1 false)
-; CHECK:  call void @llvm.lifetime.end.p0(i64 16, ptr {{.*}}{{.*}}%x)
-; CHECK:  br label %eh.resume
 
   %0 = landingpad { ptr, i32 }
   cleanup

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/use-after-scope-setjmp.ll b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/use-after-scope-setjmp.ll
index 0f50ee7..b30a42f 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/use-after-scope-setjmp.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/use-after-scope-setjmp.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; RUN: opt -passes=hwasan -hwasan-use-stack-safety=0 -hwasan-use-after-scope -S < %s | FileCheck %s
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "riscv64-unknown-linux"
@@ -8,12 +9,64 @@ target triple = "riscv64-unknown-linux"
 declare void @may_jump()
 
 define dso_local noundef i1 @_Z6targetv() sanitize_hwaddress {
+; CHECK-LABEL: define dso_local noundef i1 @_Z6targetv
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[TMP0]], 72057594037927935
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 44
+; CHECK-NEXT:    [[TMP6:%.*]] = or i64 ptrtoint (ptr @_Z6targetv to i64), [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP1]] to ptr
+; CHECK-NEXT:    store i64 [[TMP6]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = ashr i64 [[TMP0]], 56
+; CHECK-NEXT:    [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 12
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], -1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP12:%.*]] = and i64 [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    store i64 [[TMP12]], ptr @__hwasan_tls, align 8
+; CHECK-NEXT:    [[TMP13:%.*]] = or i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP13]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[BUF:%.*]] = alloca [4096 x i8], align 16
+; CHECK-NEXT:    [[TMP15:%.*]] = xor i64 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = ptrtoint ptr [[BUF]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = shl i64 [[TMP15]], 56
+; CHECK-NEXT:    [[TMP18:%.*]] = or i64 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[BUF_HWASAN:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP15]] to i8
+; CHECK-NEXT:    [[TMP20:%.*]] = ptrtoint ptr [[BUF]] to i64
+; CHECK-NEXT:    [[TMP21:%.*]] = lshr i64 [[TMP20]], 4
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP21]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP22]], i8 [[TMP19]], i64 256, i1 false)
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @setjmp(ptr noundef @jbuf)
+; CHECK-NEXT:    switch i32 [[CALL]], label [[WHILE_BODY:%.*]] [
+; CHECK-NEXT:    i32 1, label [[RETURN:%.*]]
+; CHECK-NEXT:    i32 2, label [[SW_BB1:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb1:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       while.body:
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[TMP14]], ptr @stackbuf, i32 19)
+; CHECK-NEXT:    store ptr [[BUF_HWASAN]], ptr @stackbuf, align 8
+; CHECK-NEXT:    call void @may_jump()
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i1 [ true, [[WHILE_BODY]] ], [ true, [[SW_BB1]] ], [ false, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP23:%.*]] = ptrtoint ptr [[BUF]] to i64
+; CHECK-NEXT:    [[TMP24:%.*]] = lshr i64 [[TMP23]], 4
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP24]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP25]], i8 0, i64 256, i1 false)
+; CHECK-NEXT:    ret i1 [[RETVAL_0]]
+;
 entry:
   %buf = alloca [4096 x i8], align 1
   %call = call i32 @setjmp(ptr noundef @jbuf)
   switch i32 %call, label %while.body [
-    i32 1, label %return
-    i32 2, label %sw.bb1
+  i32 1, label %return
+  i32 2, label %sw.bb1
   ]
 
 sw.bb1:                                           ; preds = %entry
@@ -29,8 +82,6 @@ while.body:                                       ; preds = %entry
   call void @llvm.lifetime.end.p0(i64 4096, ptr nonnull %buf) #10
   br label %return
 
-; CHECK-LABEL: return:
-; CHECK: void @llvm.memset.p0.i64({{.*}}, i8 0, i64 256, i1 false)
 return:                                           ; preds = %entry, %while.body, %sw.bb1
   %retval.0 = phi i1 [ true, %while.body ], [ true, %sw.bb1 ], [ false, %entry ]
   ret i1 %retval.0

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca-array.ll b/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca-array.ll
index d7155ed..4f4cb2b 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca-array.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca-array.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; RUN: opt < %s -passes=hwasan -S | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
@@ -6,9 +7,36 @@ target triple = "x86_64-unknown-linux-gnu"
 declare void @use(ptr, ptr)
 
 define void @test_alloca() sanitize_hwaddress {
-  ; CHECK: alloca { [4 x i8], [12 x i8] }, align 16
+; CHECK-LABEL: define void @test_alloca
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk {
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; CHECK-NEXT:    [[TMP1:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 20
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = and i64 [[TMP4]], 63
+; CHECK-NEXT:    [[X:%.*]] = alloca { [4 x i8], [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP5]], 57
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP5]] to i8
+; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[X]], i8 [[TMP9]], i64 16)
+; CHECK-NEXT:    [[Y:%.*]] = alloca i8, i64 16, align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[Y]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = shl i64 [[TMP10]], 57
+; CHECK-NEXT:    [[TMP13:%.*]] = or i64 [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[Y_HWASAN:%.*]] = inttoptr i64 [[TMP13]] to ptr
+; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP10]] to i8
+; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[Y]], i8 [[TMP14]], i64 16)
+; CHECK-NEXT:    call void @use(ptr [[X_HWASAN]], ptr [[Y_HWASAN]])
+; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[X]], i8 0, i64 16)
+; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[Y]], i8 0, i64 16)
+; CHECK-NEXT:    ret void
+;
   %x = alloca i8, i64 4
-  ; CHECK: alloca i8, i64 16, align 16
   %y = alloca i8, i64 16
   call void @use(ptr %x, ptr %y)
   ret void

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca-with-calls.ll b/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca-with-calls.ll
index aaa70c5..2f59c68 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca-with-calls.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca-with-calls.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test alloca instrumentation when tags are generated by HWASan function.
 ;
 ; RUN: opt < %s -passes=hwasan -hwasan-generate-tags-with-calls -S | FileCheck %s
@@ -8,12 +9,23 @@ target triple = "x86_64-unknown-linux-gnu"
 declare void @use32(ptr)
 
 define void @test_alloca() sanitize_hwaddress {
-; CHECK-LABEL: @test_alloca(
-; CHECK: %[[T1:[^ ]*]] = call i8 @__hwasan_generate_tag()
-; CHECK: %[[A:[^ ]*]] = zext i8 %[[T1]] to i64
-; CHECK: %[[B:[^ ]*]] = ptrtoint ptr %x to i64
-; CHECK: %[[C:[^ ]*]] = shl i64 %[[A]], 57
-; CHECK: or i64 %[[B]], %[[C]]
+; CHECK-LABEL: define void @test_alloca
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP0:%.*]] = call i8 @__hwasan_generate_tag()
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP1]], 57
+; CHECK-NEXT:    [[TMP4:%.*]] = or i64 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP1]] to i8
+; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[X]], i8 [[TMP5]], i64 16)
+; CHECK-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]])
+; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[X]], i8 0, i64 16)
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %x = alloca i32, align 4

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca.ll b/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca.ll
index 6e2d535..1075d0c 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca.ll
@@ -1,7 +1,8 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test alloca instrumentation.
 ;
-; RUN: opt < %s -passes=hwasan -S | FileCheck %s --check-prefixes=CHECK,NO-UAR-TAGS
-; RUN: opt < %s -passes=hwasan -hwasan-uar-retag-to-zero=0 -S | FileCheck %s --check-prefixes=CHECK,UAR-TAGS
+; RUN: opt < %s -passes=hwasan -S | FileCheck %s --check-prefixes=NO-UAR-TAGS
+; RUN: opt < %s -passes=hwasan -hwasan-uar-retag-to-zero=0 -S | FileCheck %s --check-prefixes=UAR-TAGS
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -9,32 +10,50 @@ target triple = "x86_64-unknown-linux-gnu"
 declare void @use32(ptr)
 
 define void @test_alloca() sanitize_hwaddress {
-; CHECK-LABEL: @test_alloca(
-; CHECK: %[[FP:[^ ]*]] = call ptr @llvm.frameaddress.p0(i32 0)
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %[[FP]] to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 20
-; CHECK: %[[A_XOR_B:[^ ]*]] = xor i64 %[[A]], %[[B]]
-; CHECK: %[[BASE_TAG:[^ ]*]] = and i64 %[[A_XOR_B]], 63
-
-; CHECK: %[[X:[^ ]*]] = alloca { i32, [12 x i8] }, align 16
-; CHECK: %[[X_TAG:[^ ]*]] = xor i64 %[[BASE_TAG]], 0
-; CHECK: %[[X1:[^ ]*]] = ptrtoint ptr %[[X]] to i64
-; CHECK: %[[C:[^ ]*]] = shl i64 %[[X_TAG]], 57
-; CHECK: %[[D:[^ ]*]] = or i64 %[[X1]], %[[C]]
-; CHECK: %[[X_HWASAN:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-
-; CHECK: %[[X_TAG2:[^ ]*]] = trunc i64 %[[X_TAG]] to i8
-; CHECK: call void @__hwasan_tag_memory(ptr %[[X]], i8 %[[X_TAG2]], i64 16)
-
-; CHECK: call void @use32(ptr nonnull %[[X_HWASAN]])
-
-; UAR-TAGS: %[[BASE_TAG_COMPL:[^ ]*]] = xor i64 %[[BASE_TAG]], 63
-; UAR-TAGS: %[[X_TAG_UAR:[^ ]*]] = trunc i64 %[[BASE_TAG_COMPL]] to i8
-; NO-UAR-TAGS: call void @__hwasan_tag_memory(ptr %[[X]], i8 0, i64 16)
-; UAR-TAGS: call void @__hwasan_tag_memory(ptr %[[X]], i8 %[[X_TAG_UAR]], i64 16)
-; CHECK: ret void
-
-
+; NO-UAR-TAGS-LABEL: define void @test_alloca
+; NO-UAR-TAGS-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk {
+; NO-UAR-TAGS-NEXT:  entry:
+; NO-UAR-TAGS-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; NO-UAR-TAGS-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; NO-UAR-TAGS-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; NO-UAR-TAGS-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; NO-UAR-TAGS-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; NO-UAR-TAGS-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = and i64 [[TMP3]], 63
+; NO-UAR-TAGS-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; NO-UAR-TAGS-NEXT:    [[TMP4:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0
+; NO-UAR-TAGS-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[X]] to i64
+; NO-UAR-TAGS-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP4]], 57
+; NO-UAR-TAGS-NEXT:    [[TMP7:%.*]] = or i64 [[TMP5]], [[TMP6]]
+; NO-UAR-TAGS-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; NO-UAR-TAGS-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP4]] to i8
+; NO-UAR-TAGS-NEXT:    call void @__hwasan_tag_memory(ptr [[X]], i8 [[TMP8]], i64 16)
+; NO-UAR-TAGS-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]])
+; NO-UAR-TAGS-NEXT:    call void @__hwasan_tag_memory(ptr [[X]], i8 0, i64 16)
+; NO-UAR-TAGS-NEXT:    ret void
+;
+; UAR-TAGS-LABEL: define void @test_alloca
+; UAR-TAGS-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk {
+; UAR-TAGS-NEXT:  entry:
+; UAR-TAGS-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; UAR-TAGS-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; UAR-TAGS-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; UAR-TAGS-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; UAR-TAGS-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; UAR-TAGS-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = and i64 [[TMP3]], 63
+; UAR-TAGS-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; UAR-TAGS-NEXT:    [[TMP4:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0
+; UAR-TAGS-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[X]] to i64
+; UAR-TAGS-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP4]], 57
+; UAR-TAGS-NEXT:    [[TMP7:%.*]] = or i64 [[TMP5]], [[TMP6]]
+; UAR-TAGS-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; UAR-TAGS-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP4]] to i8
+; UAR-TAGS-NEXT:    call void @__hwasan_tag_memory(ptr [[X]], i8 [[TMP8]], i64 16)
+; UAR-TAGS-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]])
+; UAR-TAGS-NEXT:    [[TMP9:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 63
+; UAR-TAGS-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i8
+; UAR-TAGS-NEXT:    call void @__hwasan_tag_memory(ptr [[X]], i8 [[TMP10]], i64 16)
+; UAR-TAGS-NEXT:    ret void
+;
 entry:
   %x = alloca i32, align 4
   call void @use32(ptr nonnull %x)

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/X86/atomic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/X86/atomic.ll
index 8949f62..49b5513 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/X86/atomic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/X86/atomic.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test basic address sanitizer instrumentation.
 ;
 ; RUN: opt < %s -passes=hwasan -S | FileCheck %s
@@ -6,13 +7,17 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @atomicrmw(ptr %ptr) sanitize_hwaddress {
-; CHECK-LABEL: @atomicrmw(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %ptr to i64
+; CHECK-LABEL: define void @atomicrmw
+; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    call void @__hwasan_store8(i64 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = atomicrmw add ptr [[PTR]], i64 1 seq_cst, align 8
+; CHECK-NEXT:    ret void
+;
 
-; CHECK: call void @__hwasan_store8(i64 %[[A]])
 
-; CHECK: atomicrmw add ptr %ptr, i64 1 seq_cst
-; CHECK: ret void
 
 entry:
   %0 = atomicrmw add ptr %ptr, i64 1 seq_cst
@@ -20,13 +25,17 @@ entry:
 }
 
 define void @cmpxchg(ptr %ptr, i64 %compare_to, i64 %new_value) sanitize_hwaddress {
-; CHECK-LABEL: @cmpxchg(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %ptr to i64
+; CHECK-LABEL: define void @cmpxchg
+; CHECK-SAME: (ptr [[PTR:%.*]], i64 [[COMPARE_TO:%.*]], i64 [[NEW_VALUE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[PTR]] to i64
+; CHECK-NEXT:    call void @__hwasan_store8(i64 [[TMP0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg ptr [[PTR]], i64 [[COMPARE_TO]], i64 [[NEW_VALUE]] seq_cst seq_cst, align 8
+; CHECK-NEXT:    ret void
+;
 
-; CHECK: call void @__hwasan_store8(i64 %[[A]])
 
-; CHECK: cmpxchg ptr %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst
-; CHECK: ret void
 
 entry:
   %0 = cmpxchg ptr %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/X86/basic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/X86/basic.ll
index 0c4b1a8..e1bc36c 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/X86/basic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/X86/basic.ll
@@ -1,22 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test basic address sanitizer instrumentation.
 ; Generic code is covered by ../basic.ll, only the x86_64 specific code is
 ; tested here.
 ;
-; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -S | FileCheck %s  --check-prefixes=CHECK,ABORT
-; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -S | FileCheck %s  --check-prefixes=CHECK,RECOVER
+; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -S | FileCheck %s  --check-prefixes=ABORT
+; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -S | FileCheck %s  --check-prefixes=RECOVER
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define i8 @test_load8(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load8(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; ABORT-LABEL: define i8 @test_load8
+; ABORT-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; ABORT-NEXT:  entry:
+; ABORT-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-NEXT:    call void @__hwasan_load1(i64 [[TMP0]])
+; ABORT-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-NEXT:    ret i8 [[B]]
+;
+; RECOVER-LABEL: define i8 @test_load8
+; RECOVER-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; RECOVER-NEXT:  entry:
+; RECOVER-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-NEXT:    call void @__hwasan_load1_noabort(i64 [[TMP0]])
+; RECOVER-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-NEXT:    ret i8 [[B]]
+;
 
-; ABORT: call void @__hwasan_load1(i64 %[[A]])
-; RECOVER: call void @__hwasan_load1_noabort(i64 %[[A]])
 
-; CHECK: %[[G:[^ ]*]] = load i8, ptr %a, align 4
-; CHECK: ret i8 %[[G]]
 
 entry:
   %b = load i8, ptr %a, align 4
@@ -24,14 +37,26 @@ entry:
 }
 
 define i40 @test_load40(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load40(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; ABORT-LABEL: define i40 @test_load40
+; ABORT-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-NEXT:  entry:
+; ABORT-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
+; ABORT-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; ABORT-NEXT:    ret i40 [[B]]
+;
+; RECOVER-LABEL: define i40 @test_load40
+; RECOVER-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-NEXT:  entry:
+; RECOVER-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-NEXT:    call void @__hwasan_loadN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; RECOVER-NEXT:    ret i40 [[B]]
+;
 
-; ABORT: call void @__hwasan_loadN(i64 %[[A]], i64 5)
-; RECOVER: call void @__hwasan_loadN_noabort(i64 %[[A]], i64 5)
 
-; CHECK: %[[B:[^ ]*]] = load i40, ptr %a
-; CHECK: ret i40 %[[B]]
 
 entry:
   %b = load i40, ptr %a, align 4
@@ -39,14 +64,26 @@ entry:
 }
 
 define void @test_store8(ptr %a, i8 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store8(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; ABORT-LABEL: define void @test_store8
+; ABORT-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-NEXT:  entry:
+; ABORT-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-NEXT:    call void @__hwasan_store1(i64 [[TMP0]])
+; ABORT-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; ABORT-NEXT:    ret void
+;
+; RECOVER-LABEL: define void @test_store8
+; RECOVER-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-NEXT:  entry:
+; RECOVER-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-NEXT:    call void @__hwasan_store1_noabort(i64 [[TMP0]])
+; RECOVER-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; RECOVER-NEXT:    ret void
+;
 
-; ABORT: call void @__hwasan_store1(i64 %[[A]])
-; RECOVER: call void @__hwasan_store1_noabort(i64 %[[A]])
 
-; CHECK: store i8 %b, ptr %a, align 4
-; CHECK: ret void
 
 entry:
   store i8 %b, ptr %a, align 4
@@ -54,14 +91,26 @@ entry:
 }
 
 define void @test_store40(ptr %a, i40 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store40(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; ABORT-LABEL: define void @test_store40
+; ABORT-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-NEXT:  entry:
+; ABORT-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
+; ABORT-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; ABORT-NEXT:    ret void
+;
+; RECOVER-LABEL: define void @test_store40
+; RECOVER-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-NEXT:  entry:
+; RECOVER-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; RECOVER-NEXT:    ret void
+;
 
-; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 5)
-; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 5)
 
-; CHECK: store i40 %b, ptr %a
-; CHECK: ret void
 
 entry:
   store i40 %b, ptr %a, align 4
@@ -69,14 +118,26 @@ entry:
 }
 
 define void @test_store_unaligned(ptr %a, i64 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store_unaligned(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
+; ABORT-LABEL: define void @test_store_unaligned
+; ABORT-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-NEXT:  entry:
+; ABORT-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
+; ABORT-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; ABORT-NEXT:    ret void
+;
+; RECOVER-LABEL: define void @test_store_unaligned
+; RECOVER-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-NEXT:  entry:
+; RECOVER-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 8)
+; RECOVER-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; RECOVER-NEXT:    ret void
+;
 
-; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 8)
-; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 8)
 
-; CHECK: store i64 %b, ptr %a, align 4
-; CHECK: ret void
 
 entry:
   store i64 %b, ptr %a, align 4

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/alloca-array.ll b/llvm/test/Instrumentation/HWAddressSanitizer/alloca-array.ll
index fe2f9e1..41382cf 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/alloca-array.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/alloca-array.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; RUN: opt < %s -passes=hwasan -S | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
@@ -6,9 +7,62 @@ target triple = "aarch64--linux-android"
 declare void @use(ptr, ptr)
 
 define void @test_alloca() sanitize_hwaddress {
-  ; CHECK: alloca { [4 x i8], [12 x i8] }, align 16
+; CHECK-LABEL: define void @test_alloca
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call ptr @llvm.thread.pointer()
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 48
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i64 [[TMP3]], 3
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.read_register.i64(metadata [[META1:![0-9]+]])
+; CHECK-NEXT:    [[TMP6:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[TMP7]], 44
+; CHECK-NEXT:    [[TMP9:%.*]] = or i64 [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP3]] to ptr
+; CHECK-NEXT:    store i64 [[TMP9]], ptr [[TMP10]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = ashr i64 [[TMP3]], 56
+; CHECK-NEXT:    [[TMP12:%.*]] = shl nuw nsw i64 [[TMP11]], 12
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i64 [[TMP12]], -1
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP3]], 8
+; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[TMP14]], [[TMP13]]
+; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = or i64 [[TMP3]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP16]], 1
+; CHECK-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[X:%.*]] = alloca { [4 x i8], [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP18:%.*]] = xor i64 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP19:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP20:%.*]] = shl i64 [[TMP18]], 56
+; CHECK-NEXT:    [[TMP21:%.*]] = or i64 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP18]] to i8
+; CHECK-NEXT:    [[TMP23:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP24:%.*]] = lshr i64 [[TMP23]], 4
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP24]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP25]], i8 [[TMP22]], i64 1, i1 false)
+; CHECK-NEXT:    [[Y:%.*]] = alloca i8, i64 16, align 16
+; CHECK-NEXT:    [[TMP26:%.*]] = xor i64 [[TMP4]], 128
+; CHECK-NEXT:    [[TMP27:%.*]] = ptrtoint ptr [[Y]] to i64
+; CHECK-NEXT:    [[TMP28:%.*]] = shl i64 [[TMP26]], 56
+; CHECK-NEXT:    [[TMP29:%.*]] = or i64 [[TMP27]], [[TMP28]]
+; CHECK-NEXT:    [[Y_HWASAN:%.*]] = inttoptr i64 [[TMP29]] to ptr
+; CHECK-NEXT:    [[TMP30:%.*]] = trunc i64 [[TMP26]] to i8
+; CHECK-NEXT:    [[TMP31:%.*]] = ptrtoint ptr [[Y]] to i64
+; CHECK-NEXT:    [[TMP32:%.*]] = lshr i64 [[TMP31]], 4
+; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP32]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP33]], i8 [[TMP30]], i64 1, i1 false)
+; CHECK-NEXT:    call void @use(ptr [[X_HWASAN]], ptr [[Y_HWASAN]])
+; CHECK-NEXT:    [[TMP34:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP35:%.*]] = lshr i64 [[TMP34]], 4
+; CHECK-NEXT:    [[TMP36:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP35]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP36]], i8 0, i64 1, i1 false)
+; CHECK-NEXT:    [[TMP37:%.*]] = ptrtoint ptr [[Y]] to i64
+; CHECK-NEXT:    [[TMP38:%.*]] = lshr i64 [[TMP37]], 4
+; CHECK-NEXT:    [[TMP39:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP38]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP39]], i8 0, i64 1, i1 false)
+; CHECK-NEXT:    ret void
+;
   %x = alloca i8, i64 4
-  ; CHECK: alloca i8, i64 16, align 16
   %y = alloca i8, i64 16
   call void @use(ptr %x, ptr %y)
   ret void

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/alloca-with-calls.ll b/llvm/test/Instrumentation/HWAddressSanitizer/alloca-with-calls.ll
index 8c386f2..018ade9 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/alloca-with-calls.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/alloca-with-calls.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test alloca instrumentation when tags are generated by HWASan function.
 ;
 ; RUN: opt < %s -passes=hwasan -hwasan-generate-tags-with-calls -S | FileCheck %s
@@ -8,12 +9,48 @@ target triple = "aarch64--linux-android"
 declare void @use32(ptr)
 
 define void @test_alloca() sanitize_hwaddress {
-; CHECK-LABEL: @test_alloca(
-; CHECK: %[[T1:[^ ]*]] = call i8 @__hwasan_generate_tag()
-; CHECK: %[[A:[^ ]*]] = zext i8 %[[T1]] to i64
-; CHECK: %[[B:[^ ]*]] = ptrtoint ptr %x to i64
-; CHECK: %[[C:[^ ]*]] = shl i64 %[[A]], 56
-; CHECK: or i64 %[[B]], %[[C]]
+; CHECK-LABEL: define void @test_alloca
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call ptr @llvm.thread.pointer()
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 48
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = ashr i64 [[TMP2]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.read_register.i64(metadata [[META1:![0-9]+]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 44
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP4]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    store i64 [[TMP8]], ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ashr i64 [[TMP2]], 56
+; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 12
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], -1
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP14:%.*]] = and i64 [[TMP13]], [[TMP12]]
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = or i64 [[TMP2]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP15]], 1
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP17:%.*]] = call i8 @__hwasan_generate_tag()
+; CHECK-NEXT:    [[TMP18:%.*]] = zext i8 [[TMP17]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP20:%.*]] = shl i64 [[TMP18]], 56
+; CHECK-NEXT:    [[TMP21:%.*]] = or i64 [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP21]] to ptr
+; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP18]] to i8
+; CHECK-NEXT:    [[TMP23:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP24:%.*]] = lshr i64 [[TMP23]], 4
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP24]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP25]], i8 [[TMP22]], i64 1, i1 false)
+; CHECK-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]])
+; CHECK-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP27:%.*]] = lshr i64 [[TMP26]], 4
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP27]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP28]], i8 0, i64 1, i1 false)
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %x = alloca i32, align 4

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/alloca.ll b/llvm/test/Instrumentation/HWAddressSanitizer/alloca.ll
index 7a26ae8..8963e5c 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/alloca.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/alloca.ll
@@ -1,8 +1,9 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test alloca instrumentation.
 ;
-; RUN: opt < %s -passes=hwasan -hwasan-with-ifunc=1 -S | FileCheck %s --check-prefixes=CHECK,DYNAMIC-SHADOW,NO-UAR-TAGS
-; RUN: opt < %s -passes=hwasan -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,ZERO-BASED-SHADOW,NO-UAR-TAGS
-; RUN: opt < %s -passes=hwasan -hwasan-with-ifunc=1 -hwasan-uar-retag-to-zero=0 -S | FileCheck %s --check-prefixes=CHECK,DYNAMIC-SHADOW,UAR-TAGS
+; RUN: opt < %s -passes=hwasan -hwasan-with-ifunc=1 -S | FileCheck %s --check-prefixes=DYNAMIC-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=ZERO-BASED-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-with-ifunc=1 -hwasan-uar-retag-to-zero=0 -S | FileCheck %s --check-prefixes=DYNAMIC-SHADOW-UAR-TAGS
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android10000"
@@ -10,44 +11,98 @@ target triple = "aarch64--linux-android10000"
 declare void @use32(ptr)
 
 define void @test_alloca() sanitize_hwaddress !dbg !15 {
-; CHECK-LABEL: @test_alloca(
-; CHECK: %[[FP:[^ ]*]] = call ptr @llvm.frameaddress.p0(i32 0)
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %[[FP]] to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 20
-; CHECK: %[[BASE_TAG:[^ ]*]] = xor i64 %[[A]], %[[B]]
-
-; CHECK: %[[X:[^ ]*]] = alloca { i32, [12 x i8] }, align 16
-; CHECK: %[[X_TAG:[^ ]*]] = xor i64 %[[BASE_TAG]], 0
-; CHECK: %[[X1:[^ ]*]] = ptrtoint ptr %[[X]] to i64
-; CHECK: %[[C:[^ ]*]] = shl i64 %[[X_TAG]], 56
-; CHECK: %[[D:[^ ]*]] = or i64 %[[X1]], %[[C]]
-; CHECK: %[[X_HWASAN:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-
-; CHECK: %[[X_TAG2:[^ ]*]] = trunc i64 %[[X_TAG]] to i8
-; CHECK: %[[E:[^ ]*]] = ptrtoint ptr %[[X]] to i64
-; CHECK: %[[F:[^ ]*]] = lshr i64 %[[E]], 4
-; DYNAMIC-SHADOW: %[[X_SHADOW:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %[[F]]
-; ZERO-BASED-SHADOW: %[[X_SHADOW:[^ ]*]] = inttoptr i64 %[[F]] to ptr
-; CHECK: %[[X_SHADOW_GEP:[^ ]*]] = getelementptr i8, ptr %[[X_SHADOW]], i32 0
-; CHECK: store i8 4, ptr %[[X_SHADOW_GEP]]
-; CHECK: %[[X_I8_GEP:[^ ]*]] = getelementptr i8, ptr %[[X]], i32 15
-; CHECK: store i8 %[[X_TAG2]], ptr %[[X_I8_GEP]]
-; CHECK: call void @llvm.dbg.value(
-; CHECK-SAME: metadata !DIArgList(ptr %[[X]], ptr %[[X]])
-; CHECK-SAME: metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_tag_offset, 0, DW_OP_LLVM_arg, 1, DW_OP_LLVM_tag_offset, 0,
-; CHECK: call void @use32(ptr nonnull %[[X_HWASAN]])
-
-; UAR-TAGS: %[[BASE_TAG_COMPL:[^ ]*]] = xor i64 %[[BASE_TAG]], 255
-; UAR-TAGS: %[[X_TAG_UAR:[^ ]*]] = trunc i64 %[[BASE_TAG_COMPL]] to i8
-; CHECK: %[[E2:[^ ]*]] = ptrtoint ptr %[[X]] to i64
-; CHECK: %[[F2:[^ ]*]] = lshr i64 %[[E2]], 4
-; DYNAMIC-SHADOW: %[[X_SHADOW2:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %[[F2]]
-; ZERO-BASED-SHADOW: %[[X_SHADOW2:[^ ]*]] = inttoptr i64 %[[F2]] to ptr
-; NO-UAR-TAGS: call void @llvm.memset.p0.i64(ptr align 1 %[[X_SHADOW2]], i8 0, i64 1, i1 false)
-; UAR-TAGS: call void @llvm.memset.p0.i64(ptr align 1 %[[X_SHADOW2]], i8 %[[X_TAG_UAR]], i64 1, i1 false)
-; CHECK: ret void
-
-
+; DYNAMIC-SHADOW-LABEL: define void @test_alloca
+; DYNAMIC-SHADOW-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk !dbg [[DBG7:![0-9]+]] {
+; DYNAMIC-SHADOW-NEXT:  entry:
+; DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; DYNAMIC-SHADOW-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; DYNAMIC-SHADOW-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0, !dbg [[DBG10:![0-9]+]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP3]], 56, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = or i64 [[TMP4]], [[TMP5]], !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP6]] to ptr, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i8, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = lshr i64 [[TMP8]], 4, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP9]], !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i32 0, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    store i8 4, ptr [[TMP11]], align 1, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[X]], i32 15, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    store i8 [[TMP7]], ptr [[TMP12]], align 1, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    call void @llvm.dbg.value(metadata !DIArgList(ptr [[X]], ptr [[X]]), metadata [[META11:![0-9]+]], metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_tag_offset, 0, DW_OP_LLVM_arg, 1, DW_OP_LLVM_tag_offset, 0, DW_OP_plus, DW_OP_deref)), !dbg [[DBG10]]
+; DYNAMIC-SHADOW-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]]), !dbg [[DBG13:![0-9]+]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG14:![0-9]+]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = lshr i64 [[TMP13]], 4, !dbg [[DBG14]]
+; DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP14]], !dbg [[DBG14]]
+; DYNAMIC-SHADOW-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP15]], i8 0, i64 1, i1 false), !dbg [[DBG14]]
+; DYNAMIC-SHADOW-NEXT:    ret void, !dbg [[DBG14]]
+;
+; ZERO-BASED-SHADOW-LABEL: define void @test_alloca
+; ZERO-BASED-SHADOW-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk !dbg [[DBG7:![0-9]+]] {
+; ZERO-BASED-SHADOW-NEXT:  entry:
+; ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; ZERO-BASED-SHADOW-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; ZERO-BASED-SHADOW-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0, !dbg [[DBG10:![0-9]+]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP3]], 56, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = or i64 [[TMP4]], [[TMP5]], !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP6]] to ptr, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i8, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = lshr i64 [[TMP8]], 4, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i32 0, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    store i8 4, ptr [[TMP11]], align 1, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[X]], i32 15, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    store i8 [[TMP7]], ptr [[TMP12]], align 1, !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    call void @llvm.dbg.value(metadata !DIArgList(ptr [[X]], ptr [[X]]), metadata [[META11:![0-9]+]], metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_tag_offset, 0, DW_OP_LLVM_arg, 1, DW_OP_LLVM_tag_offset, 0, DW_OP_plus, DW_OP_deref)), !dbg [[DBG10]]
+; ZERO-BASED-SHADOW-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]]), !dbg [[DBG13:![0-9]+]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG14:![0-9]+]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = lshr i64 [[TMP13]], 4, !dbg [[DBG14]]
+; ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr, !dbg [[DBG14]]
+; ZERO-BASED-SHADOW-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP15]], i8 0, i64 1, i1 false), !dbg [[DBG14]]
+; ZERO-BASED-SHADOW-NEXT:    ret void, !dbg [[DBG14]]
+;
+; DYNAMIC-SHADOW-UAR-TAGS-LABEL: define void @test_alloca
+; DYNAMIC-SHADOW-UAR-TAGS-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__hwasan_personality_thunk !dbg [[DBG7:![0-9]+]] {
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:  entry:
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP3:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0, !dbg [[DBG10:![0-9]+]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP3]], 56, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP6:%.*]] = or i64 [[TMP4]], [[TMP5]], !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP6]] to ptr, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP3]] to i8, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP9:%.*]] = lshr i64 [[TMP8]], 4, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP9]], !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i32 0, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    store i8 4, ptr [[TMP11]], align 1, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[X]], i32 15, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    store i8 [[TMP7]], ptr [[TMP12]], align 1, !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    call void @llvm.dbg.value(metadata !DIArgList(ptr [[X]], ptr [[X]]), metadata [[META11:![0-9]+]], metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_tag_offset, 0, DW_OP_LLVM_arg, 1, DW_OP_LLVM_tag_offset, 0, DW_OP_plus, DW_OP_deref)), !dbg [[DBG10]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]]), !dbg [[DBG13:![0-9]+]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP13:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 255, !dbg [[DBG14:![0-9]+]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP14:%.*]] = trunc i64 [[TMP13]] to i8, !dbg [[DBG14]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG14]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP16:%.*]] = lshr i64 [[TMP15]], 4, !dbg [[DBG14]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP16]], !dbg [[DBG14]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP17]], i8 [[TMP14]], i64 1, i1 false), !dbg [[DBG14]]
+; DYNAMIC-SHADOW-UAR-TAGS-NEXT:    ret void, !dbg [[DBG14]]
+;
 entry:
   %x = alloca i32, align 4
   call void @llvm.dbg.value(metadata !DIArgList(ptr %x, ptr %x), metadata !22, metadata !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_plus, DW_OP_deref)), !dbg !21

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/atomic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/atomic.ll
index 37d4cb7..93b9678 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/atomic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/atomic.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test basic address sanitizer instrumentation.
 ;
 ; RUN: opt < %s -passes=hwasan -S | FileCheck %s
@@ -6,10 +7,14 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 define void @atomicrmw(ptr %ptr) sanitize_hwaddress {
-; CHECK-LABEL: @atomicrmw(
-; CHECK: call void @llvm.hwasan.check.memaccess({{.*}}, ptr %ptr, i32 19)
-; CHECK: atomicrmw add ptr %ptr, i64 1 seq_cst
-; CHECK: ret void
+; CHECK-LABEL: define void @atomicrmw
+; CHECK-SAME: (ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[DOTHWASAN_SHADOW]], ptr [[PTR]], i32 19)
+; CHECK-NEXT:    [[TMP0:%.*]] = atomicrmw add ptr [[PTR]], i64 1 seq_cst, align 8
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %0 = atomicrmw add ptr %ptr, i64 1 seq_cst
@@ -17,10 +22,14 @@ entry:
 }
 
 define void @cmpxchg(ptr %ptr, i64 %compare_to, i64 %new_value) sanitize_hwaddress {
-; CHECK-LABEL: @cmpxchg(
-; CHECK: call void @llvm.hwasan.check.memaccess({{.*}}, ptr %ptr, i32 19)
-; CHECK: cmpxchg ptr %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst
-; CHECK: ret void
+; CHECK-LABEL: define void @cmpxchg
+; CHECK-SAME: (ptr [[PTR:%.*]], i64 [[COMPARE_TO:%.*]], i64 [[NEW_VALUE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[DOTHWASAN_SHADOW]], ptr [[PTR]], i32 19)
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg ptr [[PTR]], i64 [[COMPARE_TO]], i64 [[NEW_VALUE]] seq_cst seq_cst, align 8
+; CHECK-NEXT:    ret void
+;
 
 entry:
   %0 = cmpxchg ptr %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/basic-compat.ll b/llvm/test/Instrumentation/HWAddressSanitizer/basic-compat.ll
index 7b0097e..4831c0c 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/basic-compat.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/basic-compat.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test that the old outlined check is used with old API levels.
 
 ; RUN: opt < %s -passes=hwasan -S | FileCheck %s
@@ -6,8 +7,13 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 define i8 @test_load8(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load8(
-; CHECK: call void @llvm.hwasan.check.memaccess(ptr {{.*}}, ptr {{.*}}, i32 0)
+; CHECK-LABEL: define i8 @test_load8
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
+; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; CHECK-NEXT:    ret i8 [[B]]
+;
   %b = load i8, ptr %a, align 4
   ret i8 %b
 }

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll
index 30bbe64..3b0ec58 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll
@@ -1,9 +1,10 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test basic address sanitizer instrumentation.
 ;
-; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=CHECK,ABORT
-; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,RECOVER-DYNAMIC-SHADOW
-; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,ABORT
-; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,RECOVER-ZERO-BASED-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=ABORT-DYNAMIC-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=RECOVER-DYNAMIC-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=ABORT-ZERO-BASED-SHADOW
+; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=RECOVER-ZERO-BASED-SHADOW
 
 ; CHECK: @llvm.used = appending global [1 x ptr] [ptr @hwasan.module_ctor]
 ; CHECK: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 0, ptr @hwasan.module_ctor, ptr @hwasan.module_ctor }]
@@ -12,49 +13,104 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android10000"
 
 define i8 @test_load8(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load8(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label %[[MISMATCH:[0-9]*]], label %[[CONT:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[MISMATCH]]:
-; RECOVER: %[[NOTSHORT:[^ ]*]] = icmp ugt i8 %[[MEMTAG]], 15
-; RECOVER: br i1 %[[NOTSHORT]], label %[[FAIL:[0-9]*]], label %[[SHORT:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[FAIL]]:
-; RECOVER: call void asm sideeffect "brk #2336", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; RECOVER: [[SHORT]]:
-; RECOVER: %[[LOWBITS:[^ ]*]] = and i64 %[[A]], 15
-; RECOVER: %[[LOWBITS_I8:[^ ]*]] = trunc i64 %[[LOWBITS]] to i8
-; RECOVER: %[[LAST:[^ ]*]] = add i8 %[[LOWBITS_I8]], 0
-; RECOVER: %[[OOB:[^ ]*]] = icmp uge i8 %[[LAST]], %[[MEMTAG]]
-; RECOVER: br i1 %[[OOB]], label %[[FAIL]], label %[[INBOUNDS:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[INBOUNDS]]:
-; RECOVER: %[[EOG_ADDR:[^ ]*]] = or i64 %[[C]], 15
-; RECOVER: %[[EOG_PTR:[^ ]*]] = inttoptr i64 %[[EOG_ADDR]] to ptr
-; RECOVER: %[[EOGTAG:[^ ]*]] = load i8, ptr %[[EOG_PTR]]
-; RECOVER: %[[EOG_MISMATCH:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[EOGTAG]]
-; RECOVER: br i1 %[[EOG_MISMATCH]], label %[[FAIL]], label %[[CONT1:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[CONT1]]:
-; RECOVER: br label %[[CONT]]
-
-; RECOVER: [[CONT]]:
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 0)
-
-; CHECK: %[[G:[^ ]*]] = load i8, ptr %a, align 4
-; CHECK: ret i8 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load8
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load8
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1:![0-9]+]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2336", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load8
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load8
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1:![0-9]+]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2336", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+
+
+
+
+
+
+
+
 
 entry:
   %b = load i8, ptr %a, align 4
@@ -62,49 +118,104 @@ entry:
 }
 
 define i16 @test_load16(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load16(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label %[[MISMATCH:[0-9]*]], label %[[CONT:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[MISMATCH]]:
-; RECOVER: %[[NOTSHORT:[^ ]*]] = icmp ugt i8 %[[MEMTAG]], 15
-; RECOVER: br i1 %[[NOTSHORT]], label %[[FAIL:[0-9]*]], label %[[SHORT:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[FAIL]]:
-; RECOVER: call void asm sideeffect "brk #2337", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; RECOVER: [[SHORT]]:
-; RECOVER: %[[LOWBITS:[^ ]*]] = and i64 %[[A]], 15
-; RECOVER: %[[LOWBITS_I8:[^ ]*]] = trunc i64 %[[LOWBITS]] to i8
-; RECOVER: %[[LAST:[^ ]*]] = add i8 %[[LOWBITS_I8]], 1
-; RECOVER: %[[OOB:[^ ]*]] = icmp uge i8 %[[LAST]], %[[MEMTAG]]
-; RECOVER: br i1 %[[OOB]], label %[[FAIL]], label %[[INBOUNDS:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[INBOUNDS]]:
-; RECOVER: %[[EOG_ADDR:[^ ]*]] = or i64 %[[C]], 15
-; RECOVER: %[[EOG_PTR:[^ ]*]] = inttoptr i64 %[[EOG_ADDR]] to ptr
-; RECOVER: %[[EOGTAG:[^ ]*]] = load i8, ptr %[[EOG_PTR]]
-; RECOVER: %[[EOG_MISMATCH:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[EOGTAG]]
-; RECOVER: br i1 %[[EOG_MISMATCH]], label %[[FAIL]], label %[[CONT1:[0-9]*]], !prof {{.*}}
-
-; RECOVER: [[CONT1]]:
-; RECOVER: br label %[[CONT]]
-
-; RECOVER: [[CONT]]:
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 1)
-
-; CHECK: %[[G:[^ ]*]] = load i16, ptr %a, align 4
-; CHECK: ret i16 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i16 @test_load16
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i16 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i16 @test_load16
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2337", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i16 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i16 @test_load16
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i16 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i16 @test_load16
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2337", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i16 [[B]]
+;
+
+
+
+
+
+
+
+
 
 entry:
   %b = load i16, ptr %a, align 4
@@ -112,25 +223,99 @@ entry:
 }
 
 define i32 @test_load32(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load32(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "brk #2338", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 2)
-
-; CHECK: %[[G:[^ ]*]] = load i32, ptr %a, align 4
-; CHECK: ret i32 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i32 @test_load32
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i32 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i32 @test_load32
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2338", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i32 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i32 @test_load32
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i32 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i32 @test_load32
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2338", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i32 [[B]]
+;
+
+
+
 
 entry:
   %b = load i32, ptr %a, align 4
@@ -138,25 +323,99 @@ entry:
 }
 
 define i64 @test_load64(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load64(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "brk #2339", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 3)
-
-; CHECK: %[[G:[^ ]*]] = load i64, ptr %a, align 8
-; CHECK: ret i64 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i64 @test_load64
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i64 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i64 @test_load64
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2339", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i64 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i64 @test_load64
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i64 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i64 @test_load64
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2339", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i64 [[B]]
+;
+
+
+
 
 entry:
   %b = load i64, ptr %a, align 8
@@ -164,25 +423,99 @@ entry:
 }
 
 define i128 @test_load128(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load128(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "brk #2340", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 4)
-
-; CHECK: %[[G:[^ ]*]] = load i128, ptr %a, align 16
-; CHECK: ret i128 %[[G]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i128 @test_load128
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i128 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i128 @test_load128
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2340", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i128 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i128 @test_load128
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i128 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i128 @test_load128
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2340", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i128 [[B]]
+;
+
+
+
 
 entry:
   %b = load i128, ptr %a, align 16
@@ -190,12 +523,42 @@ entry:
 }
 
 define i40 @test_load40(ptr %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load40(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; ABORT: call void @__hwasan_loadN(i64 %[[A]], i64 5)
-; RECOVER: call void @__hwasan_loadN_noabort(i64 %[[A]], i64 5)
-; CHECK: %[[B:[^ ]*]] = load i40, ptr %a
-; CHECK: ret i40 %[[B]]
+; ABORT-DYNAMIC-SHADOW-LABEL: define i40 @test_load40
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i40 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i40 @test_load40
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_loadN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i40 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i40 @test_load40
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i40 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i40 @test_load40
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_loadN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i40 [[B]]
+;
 
 entry:
   %b = load i40, ptr %a, align 4
@@ -203,25 +566,99 @@ entry:
 }
 
 define void @test_store8(ptr %a, i8 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store8(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "brk #2352", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 16)
-
-; CHECK: store i8 %b, ptr %a, align 4
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store8
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store8
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2352", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store8
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store8
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2352", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i8 %b, ptr %a, align 4
@@ -229,25 +666,99 @@ entry:
 }
 
 define void @test_store16(ptr %a, i16 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store16(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "brk #2353", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 17)
-
-; CHECK: store i16 %b, ptr %a, align 4
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store16
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store16
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2353", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store16
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store16
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2353", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i16 %b, ptr %a, align 4
@@ -255,25 +766,99 @@ entry:
 }
 
 define void @test_store32(ptr %a, i32 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store32(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "brk #2354", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 18)
-
-; CHECK: store i32 %b, ptr %a, align 4
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store32
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store32
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2354", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store32
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store32
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2354", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i32 %b, ptr %a, align 4
@@ -281,25 +866,99 @@ entry:
 }
 
 define void @test_store64(ptr %a, i64 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store64(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "brk #2355", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 19)
-
-; CHECK: store i64 %b, ptr %a, align 8
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store64
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store64
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2355", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store64
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store64
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2355", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i64 %b, ptr %a, align 8
@@ -307,25 +966,99 @@ entry:
 }
 
 define void @test_store128(ptr %a, i128 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store128(
-; RECOVER: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, ptr %.hwasan.shadow, i64 %4
-; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to ptr
-; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, ptr %[[E]]
-; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; RECOVER: call void asm sideeffect "brk #2356", "{x0}"(i64 %[[A]])
-; RECOVER: br label
-
-; ABORT: call void @llvm.hwasan.check.memaccess.shortgranules(ptr %.hwasan.shadow, ptr %a, i32 20)
-
-; CHECK: store i128 %b, ptr %a, align 16
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store128
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store128
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       8:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       10:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2356", "{x0}"(i64 [[TMP0]])
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-DYNAMIC-SHADOW:       11:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       16:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-DYNAMIC-SHADOW:       21:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-DYNAMIC-SHADOW:       22:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store128
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store128
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       8:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       10:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2356", "{x0}"(i64 [[TMP0]])
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
+; RECOVER-ZERO-BASED-SHADOW:       11:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       16:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF1]]
+; RECOVER-ZERO-BASED-SHADOW:       21:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
+; RECOVER-ZERO-BASED-SHADOW:       22:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+
+
+
 
 entry:
   store i128 %b, ptr %a, align 16
@@ -333,12 +1066,42 @@ entry:
 }
 
 define void @test_store40(ptr %a, i40 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store40(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 5)
-; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 5)
-; CHECK: store i40 %b, ptr %a
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store40
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store40
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store40
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store40
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 5)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
 
 entry:
   store i40 %b, ptr %a, align 4
@@ -346,12 +1109,42 @@ entry:
 }
 
 define void @test_store_unaligned(ptr %a, i64 %b) sanitize_hwaddress {
-; CHECK-LABEL: @test_store_unaligned(
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
-; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 8)
-; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 8)
-; CHECK: store i64 %b, ptr %a, align 4
-; CHECK: ret void
+; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store_unaligned
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
+; ABORT-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store_unaligned
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 8)
+; RECOVER-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store_unaligned
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
+; ABORT-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store_unaligned
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 8)
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
+;
 
 entry:
   store i64 %b, ptr %a, align 4
@@ -359,33 +1152,108 @@ entry:
 }
 
 define i8 @test_load_noattr(ptr %a) {
-; CHECK-LABEL: @test_load_noattr(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: %[[B:[^ ]*]] = load i8, ptr %a
-; CHECK-NEXT: ret i8 %[[B]]
-
+; CHECK-LABEL: define i8 @test_load_noattr
+; CHECK-SAME: (ptr [[A:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; CHECK-NEXT:    ret i8 [[B]]
+;
+; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_noattr
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_noattr
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_noattr
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_noattr
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
 entry:
   %b = load i8, ptr %a, align 4
   ret i8 %b
 }
 
 define i8 @test_load_notmyattr(ptr %a) sanitize_address {
-; CHECK-LABEL: @test_load_notmyattr(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: %[[B:[^ ]*]] = load i8, ptr %a
-; CHECK-NEXT: ret i8 %[[B]]
-
+; CHECK-LABEL: define i8 @test_load_notmyattr
+; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; CHECK-NEXT:    ret i8 [[B]]
+;
+; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_notmyattr
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_notmyattr
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_notmyattr
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_notmyattr
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
 entry:
   %b = load i8, ptr %a, align 4
   ret i8 %b
 }
 
 define i8 @test_load_addrspace(ptr addrspace(256) %a) sanitize_hwaddress {
-; CHECK-LABEL: @test_load_addrspace(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: %[[B:[^ ]*]] = load i8, ptr addrspace(256) %a
-; CHECK-NEXT: ret i8 %[[B]]
-
+; CHECK-LABEL: define i8 @test_load_addrspace
+; CHECK-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; CHECK-NEXT:    ret i8 [[B]]
+;
+; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_addrspace
+; ABORT-DYNAMIC-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
+; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_addrspace
+; RECOVER-DYNAMIC-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
+; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
+;
+; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_addrspace
+; ABORT-ZERO-BASED-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
+; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
+; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_addrspace
+; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
+; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
+; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
+;
 entry:
   %b = load i8, ptr addrspace(256) %a, align 4
   ret i8 %b
@@ -393,9 +1261,7 @@ entry:
 
 ; CHECK: declare void @__hwasan_init()
 
-; CHECK:      define internal void @hwasan.module_ctor() #[[#ATTR:]] comdat {
 ; CHECK-NEXT:   call void @__hwasan_init()
 ; CHECK-NEXT:   ret void
 ; CHECK-NEXT: }
 
-; CHECK:      attributes #[[#ATTR]] = { nounwind }

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/exception-lifetime.ll b/llvm/test/Instrumentation/HWAddressSanitizer/exception-lifetime.ll
index ab52691..1940eb9 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/exception-lifetime.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/exception-lifetime.ll
@@ -1,7 +1,8 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test allocas with multiple lifetime ends, as frequently seen for exception
 ; handling.
 ;
-; RUN: opt -passes=hwasan -hwasan-use-after-scope -S -o - %s | FileCheck %s --check-prefix=CHECK
+; RUN: opt -passes=hwasan -hwasan-use-after-scope -S -o - %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
@@ -14,6 +15,79 @@ declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
 declare i32 @__gxx_personality_v0(...)
 
 define void @test() sanitize_hwaddress personality ptr @__gxx_personality_v0 {
+; CHECK-LABEL: define void @test
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] personality ptr @__gxx_personality_v0 {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call ptr @llvm.thread.pointer()
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 48
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = ashr i64 [[TMP2]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.read_register.i64(metadata [[META1:![0-9]+]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 44
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP4]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    store i64 [[TMP8]], ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ashr i64 [[TMP2]], 56
+; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 12
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], -1
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP14:%.*]] = and i64 [[TMP13]], [[TMP12]]
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = or i64 [[TMP2]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP15]], 1
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP18:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = shl i64 [[TMP17]], 56
+; CHECK-NEXT:    [[TMP20:%.*]] = or i64 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP20]] to ptr
+; CHECK-NEXT:    [[EXN_SLOT:%.*]] = alloca ptr, align 8
+; CHECK-NEXT:    [[EHSELECTOR_SLOT:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP17]] to i8
+; CHECK-NEXT:    [[TMP22:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP23:%.*]] = lshr i64 [[TMP22]], 4
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP23]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP24]], i8 [[TMP21]], i64 1, i1 false)
+; CHECK-NEXT:    invoke void @mayFail(ptr [[X_HWASAN]])
+; CHECK-NEXT:    to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]]
+; CHECK:       invoke.cont:
+; CHECK-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP26]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP27]], i8 0, i64 1, i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    ret void
+; CHECK:       lpad:
+; CHECK-NEXT:    [[TMP28:%.*]] = landingpad { ptr, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    [[TMP29:%.*]] = call i64 @llvm.read_register.i64(metadata [[META2:![0-9]+]])
+; CHECK-NEXT:    call void @__hwasan_handle_vfork(i64 [[TMP29]])
+; CHECK-NEXT:    [[TMP30:%.*]] = extractvalue { ptr, i32 } [[TMP28]], 0
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[TMP16]], ptr [[EXN_SLOT]], i32 19)
+; CHECK-NEXT:    store ptr [[TMP30]], ptr [[EXN_SLOT]], align 8
+; CHECK-NEXT:    [[TMP31:%.*]] = extractvalue { ptr, i32 } [[TMP28]], 1
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[TMP16]], ptr [[EHSELECTOR_SLOT]], i32 18)
+; CHECK-NEXT:    store i32 [[TMP31]], ptr [[EHSELECTOR_SLOT]], align 4
+; CHECK-NEXT:    call void @onExcept(ptr [[X_HWASAN]])
+; CHECK-NEXT:    [[TMP32:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP33:%.*]] = lshr i64 [[TMP32]], 4
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP33]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP34]], i8 0, i64 1, i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    br label [[EH_RESUME:%.*]]
+; CHECK:       eh.resume:
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[TMP16]], ptr [[EXN_SLOT]], i32 3)
+; CHECK-NEXT:    [[EXN:%.*]] = load ptr, ptr [[EXN_SLOT]], align 8
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[TMP16]], ptr [[EHSELECTOR_SLOT]], i32 2)
+; CHECK-NEXT:    [[SEL:%.*]] = load i32, ptr [[EHSELECTOR_SLOT]], align 4
+; CHECK-NEXT:    [[LPAD_VAL:%.*]] = insertvalue { ptr, i32 } undef, ptr [[EXN]], 0
+; CHECK-NEXT:    [[LPAD_VAL1:%.*]] = insertvalue { ptr, i32 } [[LPAD_VAL]], i32 [[SEL]], 1
+; CHECK-NEXT:    resume { ptr, i32 } [[LPAD_VAL1]]
+;
 entry:
   %x = alloca i32, align 8
   %exn.slot = alloca ptr, align 8
@@ -22,19 +96,11 @@ entry:
   invoke void @mayFail(ptr %x) to label %invoke.cont unwind label %lpad
 
 invoke.cont:                                      ; preds = %entry
-; CHECK: invoke.cont:
-; CHECK:  call void @llvm.memset.p0.i64(ptr align 1 %{{.*}}, i8 0, i64 1, i1 false)
-; CHECK:  call void @llvm.lifetime.end.p0(i64 16, ptr {{.*}}{{.*}}%x)
-; CHECK:  ret void
 
   call void @llvm.lifetime.end.p0(i64 8, ptr %x)
   ret void
 
 lpad:                                             ; preds = %entry
-; CHECK: lpad
-; CHECK:  call void @llvm.memset.p0.i64(ptr align 1 %{{.*}}, i8 0, i64 1, i1 false)
-; CHECK:  call void @llvm.lifetime.end.p0(i64 16, ptr {{.*}}{{.*}}%x)
-; CHECK:  br label %eh.resume
 
   %0 = landingpad { ptr, i32 }
   cleanup

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/kernel-alloca.ll b/llvm/test/Instrumentation/HWAddressSanitizer/kernel-alloca.ll
index 92ff51a..315cd4d 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/kernel-alloca.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/kernel-alloca.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; Test kernel hwasan instrumentation for alloca.
 ;
 ; RUN: opt < %s -passes=hwasan -hwasan-kernel=1 -S | FileCheck %s
@@ -8,19 +9,34 @@ target triple = "aarch64--linux-android"
 declare void @use32(ptr)
 
 define void @test_alloca() sanitize_hwaddress {
-; CHECK-LABEL: @test_alloca(
-; CHECK: %[[FP:[^ ]*]] = call ptr @llvm.frameaddress.p0(i32 0)
-; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %[[FP]] to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 20
-; CHECK: %[[BASE_TAG:[^ ]*]] = xor i64 %[[A]], %[[B]]
+; CHECK-LABEL: define void @test_alloca
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
+; CHECK-NEXT:    [[TMP0:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 20
+; CHECK-NEXT:    [[HWASAN_STACK_BASE_TAG:%.*]] = xor i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[HWASAN_STACK_BASE_TAG]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP3]], 56
+; CHECK-NEXT:    [[TMP6:%.*]] = or i64 [[TMP5]], 72057594037927935
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP3]] to i8
+; CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP10:%.*]] = lshr i64 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP11]], i8 [[TMP8]], i64 1, i1 false)
+; CHECK-NEXT:    call void @use32(ptr nonnull [[X_HWASAN]])
+; CHECK-NEXT:    [[TMP12:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = lshr i64 [[TMP12]], 4
+; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP14]], i8 0, i64 1, i1 false)
+; CHECK-NEXT:    ret void
+;
 
-; CHECK: %[[X:[^ ]*]] = alloca { i32, [12 x i8] }, align 16
-; CHECK: %[[X_TAG:[^ ]*]] = xor i64 %[[BASE_TAG]], 0
-; CHECK: %[[X1:[^ ]*]] = ptrtoint ptr %[[X]] to i64
-; CHECK: %[[C:[^ ]*]] = shl i64 %[[X_TAG]], 56
-; CHECK: %[[D:[^ ]*]] = or i64 %[[C]], 72057594037927935
-; CHECK: %[[E:[^ ]*]] = and i64 %[[X1]], %[[D]]
-; CHECK: %[[X_HWASAN:[^ ]*]] = inttoptr i64 %[[E]] to ptr
 
 entry:
   %x = alloca i32, align 4

diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll b/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll
index d0f152d..45a8132 100644 (file)
--- a/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; RUN: opt -passes=hwasan -hwasan-use-stack-safety=0 -hwasan-use-after-scope -S < %s | FileCheck %s
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64-unknown-linux-android29"
@@ -8,12 +9,66 @@ target triple = "aarch64-unknown-linux-android29"
 declare void @may_jump()
 
 define dso_local noundef i1 @_Z6targetv() sanitize_hwaddress {
+; CHECK-LABEL: define dso_local noundef i1 @_Z6targetv
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call ptr @llvm.thread.pointer()
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 48
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = ashr i64 [[TMP2]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.read_register.i64(metadata [[META1:![0-9]+]])
+; CHECK-NEXT:    [[TMP5:%.*]] = call ptr @llvm.frameaddress.p0(i32 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 44
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP4]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    store i64 [[TMP8]], ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = ashr i64 [[TMP2]], 56
+; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 12
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP11]], -1
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP14:%.*]] = and i64 [[TMP13]], [[TMP12]]
+; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP15:%.*]] = or i64 [[TMP2]], 4294967295
+; CHECK-NEXT:    [[HWASAN_SHADOW:%.*]] = add i64 [[TMP15]], 1
+; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[HWASAN_SHADOW]] to ptr
+; CHECK-NEXT:    [[BUF:%.*]] = alloca [4096 x i8], align 16
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i64 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP18:%.*]] = ptrtoint ptr [[BUF]] to i64
+; CHECK-NEXT:    [[TMP19:%.*]] = shl i64 [[TMP17]], 56
+; CHECK-NEXT:    [[TMP20:%.*]] = or i64 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[BUF_HWASAN:%.*]] = inttoptr i64 [[TMP20]] to ptr
+; CHECK-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP17]] to i8
+; CHECK-NEXT:    [[TMP22:%.*]] = ptrtoint ptr [[BUF]] to i64
+; CHECK-NEXT:    [[TMP23:%.*]] = lshr i64 [[TMP22]], 4
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP23]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP24]], i8 [[TMP21]], i64 256, i1 false)
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @setjmp(ptr noundef @jbuf)
+; CHECK-NEXT:    switch i32 [[CALL]], label [[WHILE_BODY:%.*]] [
+; CHECK-NEXT:    i32 1, label [[RETURN:%.*]]
+; CHECK-NEXT:    i32 2, label [[SW_BB1:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.bb1:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       while.body:
+; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess(ptr [[TMP16]], ptr @stackbuf, i32 19)
+; CHECK-NEXT:    store ptr [[BUF_HWASAN]], ptr @stackbuf, align 8
+; CHECK-NEXT:    call void @may_jump()
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i1 [ true, [[WHILE_BODY]] ], [ true, [[SW_BB1]] ], [ false, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP25:%.*]] = ptrtoint ptr [[BUF]] to i64
+; CHECK-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP25]], 4
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP26]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP27]], i8 0, i64 256, i1 false)
+; CHECK-NEXT:    ret i1 [[RETVAL_0]]
+;
 entry:
   %buf = alloca [4096 x i8], align 1
   %call = call i32 @setjmp(ptr noundef @jbuf)
   switch i32 %call, label %while.body [
-    i32 1, label %return
-    i32 2, label %sw.bb1
+  i32 1, label %return
+  i32 2, label %sw.bb1
   ]
 
 sw.bb1:                                           ; preds = %entry
@@ -29,8 +84,6 @@ while.body:                                       ; preds = %entry
   call void @llvm.lifetime.end.p0(i64 4096, ptr nonnull %buf) #10
   br label %return
 
-; CHECK-LABEL: return:
-; CHECK: void @llvm.memset.p0.i64({{.*}}, i8 0, i64 256, i1 false)
 return:                                           ; preds = %entry, %while.body, %sw.bb1
   %retval.0 = phi i1 [ true, %while.body ], [ true, %sw.bb1 ], [ false, %entry ]
   ret i1 %retval.0