Currently, hwasan-match-all-tag flag is supported in inline memory access instrumentation and outline memory access instrumentation, but not supported in callback memory access instrumentation.
- For inline memory access instrumentation: a hwasan-match-all-tag check is added following the tag-mismtach check, if tag from pointer is mismatched with tag from shadow memory and tag from pointer is not equal with hwasan-match-all-tag, then a tag-mismatch will be report.
- For outline memory acess instrumentation: MatchAllTag is encoded in AccessInfo, when emit HWASAN memaccess symbols, asm-printer emits assembly instructions to check if tag from pointer is equal with hwasan-match-all-tag.
- For callback memory access instrumentation: hwasan-match-all-tag check is not implemented in `__hwasan_load`/`__hwasan_store`.
This patch implements a set of callback functions: `__hwasan_[load|store][1|2|4|8|16|n]_match_all` and `__hwasan_load[load|store][1|2|4|8|16|n]_match_all_noabort`, making hwasan-match-all-tag flag working for callback memory access instrumentation.
Reviewed By: vitalybuka
Differential Revision: https://reviews.llvm.org/D149580
CheckAddress<ErrorAction::Recover, AccessType::Load, 4>(p);
}
+void __hwasan_loadN_match_all(uptr p, uptr sz, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddressSized<ErrorAction::Abort, AccessType::Load>(p, sz);
+}
+void __hwasan_load1_match_all(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Abort, AccessType::Load, 0>(p);
+}
+void __hwasan_load2_match_all(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Abort, AccessType::Load, 1>(p);
+}
+void __hwasan_load4_match_all(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Abort, AccessType::Load, 2>(p);
+}
+void __hwasan_load8_match_all(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Abort, AccessType::Load, 3>(p);
+}
+void __hwasan_load16_match_all(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Abort, AccessType::Load, 4>(p);
+}
+
+void __hwasan_loadN_match_all_noabort(uptr p, uptr sz, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddressSized<ErrorAction::Recover, AccessType::Load>(p, sz);
+}
+void __hwasan_load1_match_all_noabort(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Recover, AccessType::Load, 0>(p);
+}
+void __hwasan_load2_match_all_noabort(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Recover, AccessType::Load, 1>(p);
+}
+void __hwasan_load4_match_all_noabort(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Recover, AccessType::Load, 2>(p);
+}
+void __hwasan_load8_match_all_noabort(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Recover, AccessType::Load, 3>(p);
+}
+void __hwasan_load16_match_all_noabort(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Recover, AccessType::Load, 4>(p);
+}
+
void __hwasan_storeN(uptr p, uptr sz) {
CheckAddressSized<ErrorAction::Abort, AccessType::Store>(p, sz);
}
CheckAddress<ErrorAction::Recover, AccessType::Store, 4>(p);
}
+void __hwasan_storeN_match_all(uptr p, uptr sz, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddressSized<ErrorAction::Abort, AccessType::Store>(p, sz);
+}
+void __hwasan_store1_match_all(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Abort, AccessType::Store, 0>(p);
+}
+void __hwasan_store2_match_all(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Abort, AccessType::Store, 1>(p);
+}
+void __hwasan_store4_match_all(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Abort, AccessType::Store, 2>(p);
+}
+void __hwasan_store8_match_all(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Abort, AccessType::Store, 3>(p);
+}
+void __hwasan_store16_match_all(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Abort, AccessType::Store, 4>(p);
+}
+
+void __hwasan_storeN_match_all_noabort(uptr p, uptr sz, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddressSized<ErrorAction::Recover, AccessType::Store>(p, sz);
+}
+void __hwasan_store1_match_all_noabort(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Recover, AccessType::Store, 0>(p);
+}
+void __hwasan_store2_match_all_noabort(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Recover, AccessType::Store, 1>(p);
+}
+void __hwasan_store4_match_all_noabort(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Recover, AccessType::Store, 2>(p);
+}
+void __hwasan_store8_match_all_noabort(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Recover, AccessType::Store, 3>(p);
+}
+void __hwasan_store16_match_all_noabort(uptr p, u8 match_all_tag) {
+ if (GetTagFromPointer(p) != match_all_tag)
+ CheckAddress<ErrorAction::Recover, AccessType::Store, 4>(p);
+}
+
void __hwasan_tag_memory(uptr p, u8 tag, uptr sz) {
TagMemoryAligned(UntagAddr(p), sz, tag);
}
void __hwasan_load16_noabort(uptr);
SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_loadN_match_all(uptr, uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load1_match_all(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load2_match_all(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load4_match_all(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load8_match_all(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load16_match_all(uptr, u8);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_loadN_match_all_noabort(uptr, uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load1_match_all_noabort(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load2_match_all_noabort(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load4_match_all_noabort(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load8_match_all_noabort(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_load16_match_all_noabort(uptr, u8);
+
+SANITIZER_INTERFACE_ATTRIBUTE
void __hwasan_storeN(uptr, uptr);
SANITIZER_INTERFACE_ATTRIBUTE
void __hwasan_store1(uptr);
void __hwasan_store16_noabort(uptr);
SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_storeN_match_all(uptr, uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store1_match_all(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store2_match_all(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store4_match_all(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store8_match_all(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store16_match_all(uptr, u8);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_storeN_match_all_noabort(uptr, uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store1_match_all_noabort(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store2_match_all_noabort(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store4_match_all_noabort(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store8_match_all_noabort(uptr, u8);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_store16_match_all_noabort(uptr, u8);
+
+SANITIZER_INTERFACE_ATTRIBUTE
void __hwasan_tag_memory(uptr p, u8 tag, uptr sz);
SANITIZER_INTERFACE_ATTRIBUTE
bool InstrumentStack;
bool DetectUseAfterScope;
bool UsePageAliases;
+ bool UseMatchAllCallback;
std::optional<uint8_t> MatchAllTag;
} else if (CompileKernel) {
MatchAllTag = 0xFF;
}
+ UseMatchAllCallback = !CompileKernel && MatchAllTag.has_value();
// If we don't have personality function support, fall back to landing pads.
InstrumentLandingPads = ClInstrumentLandingPads.getNumOccurrences()
for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
const std::string TypeStr = AccessIsWrite ? "store" : "load";
const std::string EndingStr = Recover ? "_noabort" : "";
+ const std::string MatchAllStr = UseMatchAllCallback ? "_match_all" : "";
+
+ FunctionType *HwasanMemoryAccessCallbackSizedFnTy,
+ *HwasanMemoryAccessCallbackFnTy;
+ if (UseMatchAllCallback) {
+ HwasanMemoryAccessCallbackSizedFnTy =
+ FunctionType::get(VoidTy, {IntptrTy, IntptrTy, Int8Ty}, false);
+ HwasanMemoryAccessCallbackFnTy =
+ FunctionType::get(VoidTy, {IntptrTy, Int8Ty}, false);
+ } else {
+ HwasanMemoryAccessCallbackSizedFnTy =
+ FunctionType::get(VoidTy, {IntptrTy, IntptrTy}, false);
+ HwasanMemoryAccessCallbackFnTy =
+ FunctionType::get(VoidTy, {IntptrTy}, false);
+ }
HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction(
- ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr,
- FunctionType::get(VoidTy, {IntptrTy, IntptrTy}, false));
+ ClMemoryAccessCallbackPrefix + TypeStr + "N" + MatchAllStr + EndingStr,
+ HwasanMemoryAccessCallbackSizedFnTy);
for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
AccessSizeIndex++) {
HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr +
- itostr(1ULL << AccessSizeIndex) + EndingStr,
- FunctionType::get(VoidTy, {IntptrTy}, false));
+ itostr(1ULL << AccessSizeIndex) +
+ MatchAllStr + EndingStr,
+ HwasanMemoryAccessCallbackFnTy);
}
}
*O.Alignment >= O.TypeStoreSize / 8)) {
size_t AccessSizeIndex = TypeSizeToSizeIndex(O.TypeStoreSize);
if (InstrumentWithCalls) {
- IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
- IRB.CreatePointerCast(Addr, IntptrTy));
+ if (UseMatchAllCallback) {
+ IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
+ {IRB.CreatePointerCast(Addr, IntptrTy),
+ ConstantInt::get(Int8Ty, *MatchAllTag)});
+ } else {
+ IRB.CreateCall(HwasanMemoryAccessCallback[O.IsWrite][AccessSizeIndex],
+ IRB.CreatePointerCast(Addr, IntptrTy));
+ }
} else if (OutlinedChecks) {
instrumentMemAccessOutline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn());
} else {
instrumentMemAccessInline(Addr, O.IsWrite, AccessSizeIndex, O.getInsn());
}
} else {
- IRB.CreateCall(HwasanMemoryAccessCallbackSized[O.IsWrite],
- {IRB.CreatePointerCast(Addr, IntptrTy),
- IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy,
- O.TypeStoreSize),
- ConstantInt::get(IntptrTy, 8))});
+ if (UseMatchAllCallback) {
+ IRB.CreateCall(
+ HwasanMemoryAccessCallbackSized[O.IsWrite],
+ {IRB.CreatePointerCast(Addr, IntptrTy),
+ IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy, O.TypeStoreSize),
+ ConstantInt::get(IntptrTy, 8)),
+ ConstantInt::get(Int8Ty, *MatchAllTag)});
+ } else {
+ IRB.CreateCall(
+ HwasanMemoryAccessCallbackSized[O.IsWrite],
+ {IRB.CreatePointerCast(Addr, IntptrTy),
+ IRB.CreateUDiv(IRB.CreateTypeSize(IntptrTy, O.TypeStoreSize),
+ ConstantInt::get(IntptrTy, 8))});
+ }
}
untagPointerOperand(O.getInsn(), Addr);
;
; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -S | FileCheck %s --check-prefixes=CHECK,ABORT
; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-recover=1 -S | FileCheck %s --check-prefixes=CHECK,RECOVER
+; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-match-all-tag=0 -S | FileCheck %s --check-prefixes=CHECK,MATCH-ALL-TAG-ABORT
+; RUN: opt < %s -passes=hwasan -hwasan-instrument-with-calls -hwasan-recover=1 -hwasan-match-all-tag=0 -S | FileCheck %s --check-prefixes=CHECK,MATCH-ALL-TAG-RECOVER
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-android"
; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
; ABORT: call void @__hwasan_load1(i64 %[[A]])
; RECOVER: call void @__hwasan_load1_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_load1_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load1_match_all_noabort(i64 %[[A]], i8 0)
; CHECK: %[[B:[^ ]*]] = load i8, ptr %a
; CHECK: ret i8 %[[B]]
; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
; ABORT: call void @__hwasan_load2(i64 %[[A]])
; RECOVER: call void @__hwasan_load2_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_load2_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load2_match_all_noabort(i64 %[[A]], i8 0)
; CHECK: %[[B:[^ ]*]] = load i16, ptr %a
; CHECK: ret i16 %[[B]]
; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
; ABORT: call void @__hwasan_load4(i64 %[[A]])
; RECOVER: call void @__hwasan_load4_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_load4_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load4_match_all_noabort(i64 %[[A]], i8 0)
; CHECK: %[[B:[^ ]*]] = load i32, ptr %a
; CHECK: ret i32 %[[B]]
; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
; ABORT: call void @__hwasan_load8(i64 %[[A]])
; RECOVER: call void @__hwasan_load8_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_load8_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load8_match_all_noabort(i64 %[[A]], i8 0)
; CHECK: %[[B:[^ ]*]] = load i64, ptr %a
; CHECK: ret i64 %[[B]]
; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
; ABORT: call void @__hwasan_load16(i64 %[[A]])
; RECOVER: call void @__hwasan_load16_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_load16_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_load16_match_all_noabort(i64 %[[A]], i8 0)
; CHECK: %[[B:[^ ]*]] = load i128, ptr %a
; CHECK: ret i128 %[[B]]
; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
; ABORT: call void @__hwasan_loadN(i64 %[[A]], i64 5)
; RECOVER: call void @__hwasan_loadN_noabort(i64 %[[A]], i64 5)
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_loadN_match_all(i64 %[[A]], i64 5, i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_loadN_match_all_noabort(i64 %[[A]], i64 5, i8 0)
; CHECK: %[[B:[^ ]*]] = load i40, ptr %a
; CHECK: ret i40 %[[B]]
; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
; ABORT: call void @__hwasan_store1(i64 %[[A]])
; RECOVER: call void @__hwasan_store1_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_store1_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store1_match_all_noabort(i64 %[[A]], i8 0)
; CHECK: store i8 %b, ptr %a
; CHECK: ret void
; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
; ABORT: call void @__hwasan_store2(i64 %[[A]])
; RECOVER: call void @__hwasan_store2_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_store2_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store2_match_all_noabort(i64 %[[A]], i8 0)
; CHECK: store i16 %b, ptr %a
; CHECK: ret void
; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
; ABORT: call void @__hwasan_store4(i64 %[[A]])
; RECOVER: call void @__hwasan_store4_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_store4_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store4_match_all_noabort(i64 %[[A]], i8 0)
; CHECK: store i32 %b, ptr %a
; CHECK: ret void
; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
; ABORT: call void @__hwasan_store8(i64 %[[A]])
; RECOVER: call void @__hwasan_store8_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_store8_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store8_match_all_noabort(i64 %[[A]], i8 0)
; CHECK: store i64 %b, ptr %a
; CHECK: ret void
; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
; ABORT: call void @__hwasan_store16(i64 %[[A]])
; RECOVER: call void @__hwasan_store16_noabort(i64 %[[A]])
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_store16_match_all(i64 %[[A]], i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_store16_match_all_noabort(i64 %[[A]], i8 0)
; CHECK: store i128 %b, ptr %a
; CHECK: ret void
; CHECK: %[[A:[^ ]*]] = ptrtoint ptr %a to i64
; ABORT: call void @__hwasan_storeN(i64 %[[A]], i64 5)
; RECOVER: call void @__hwasan_storeN_noabort(i64 %[[A]], i64 5)
+; MATCH-ALL-TAG-ABORT: call void @__hwasan_storeN_match_all(i64 %[[A]], i64 5, i8 0)
+; MATCH-ALL-TAG-RECOVER: call void @__hwasan_storeN_match_all_noabort(i64 %[[A]], i64 5, i8 0)
; CHECK: store i40 %b, ptr %a
; CHECK: ret void