CodeGenOpt::Level OptLevel);
FunctionPass *createAArch64StorePairSuppressPass();
FunctionPass *createAArch64ExpandPseudoPass();
+FunctionPass *createAArch64SLSHardeningPass();
FunctionPass *createAArch64SpeculationHardeningPass();
FunctionPass *createAArch64LoadStoreOptimizationPass();
FunctionPass *createAArch64SIMDInstrOptPass();
void initializeAArch64ConditionOptimizerPass(PassRegistry&);
void initializeAArch64DeadRegisterDefinitionsPass(PassRegistry&);
void initializeAArch64ExpandPseudoPass(PassRegistry&);
+void initializeAArch64SLSHardeningPass(PassRegistry&);
void initializeAArch64SpeculationHardeningPass(PassRegistry&);
void initializeAArch64LoadStoreOptPass(PassRegistry&);
void initializeAArch64SIMDInstrOptPass(PassRegistry&);
"true", "Permit use of TPIDR_EL"#i#" for the TLS base">;
//===----------------------------------------------------------------------===//
+// Control codegen mitigation against Straight Line Speculation vulnerability.
+//===----------------------------------------------------------------------===//
+
+def FeatureHardenSlsRetBr : SubtargetFeature<"harden-sls-retbr",
+ "HardenSlsRetBr", "true",
+ "Harden against straight line speculation across RET and BR instructions">;
+
+//===----------------------------------------------------------------------===//
// AArch64 Processors supported.
//
EmitToStreamer(*OutStreamer, TmpInst);
return;
}
+ case AArch64::SpeculationBarrierISBDSBEndBB: {
+ // Print DSB SYS + ISB
+ MCInst TmpInstDSB;
+ TmpInstDSB.setOpcode(AArch64::DSB);
+ TmpInstDSB.addOperand(MCOperand::createImm(0xf));
+ EmitToStreamer(*OutStreamer, TmpInstDSB);
+ MCInst TmpInstISB;
+ TmpInstISB.setOpcode(AArch64::ISB);
+ TmpInstISB.addOperand(MCOperand::createImm(0xf));
+ EmitToStreamer(*OutStreamer, TmpInstISB);
+ return;
+ }
+ case AArch64::SpeculationBarrierSBEndBB: {
+ // Print SB
+ MCInst TmpInstSB;
+ TmpInstSB.setOpcode(AArch64::SB);
+ EmitToStreamer(*OutStreamer, TmpInstSB);
+ return;
+ }
case AArch64::TLSDESC_CALLSEQ: {
/// lower this to:
/// adrp x0, :tlsdesc:var
// This gets lowered to an instruction sequence which takes 16 bytes
NumBytes = 16;
break;
+ case AArch64::SpeculationBarrierISBDSBEndBB:
+ // This gets lowered to 2 4-byte instructions.
+ NumBytes = 8;
+ break;
+ case AArch64::SpeculationBarrierSBEndBB:
+ // This gets lowered to 1 4-byte instructions.
+ NumBytes = 4;
+ break;
case AArch64::JumpTableDest32:
case AArch64::JumpTableDest16:
case AArch64::JumpTableDest8:
if (I == MBB.end())
return false;
+ // Skip over SpeculationBarrierEndBB terminators
+ if (I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
+ I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
+ --I;
+ }
+
if (!isUnpredicatedTerminator(*I))
return false;
}
static inline bool isIndirectBranchOpcode(int Opc) {
- return Opc == AArch64::BR;
+ switch (Opc) {
+ case AArch64::BR:
+ case AArch64::BRAA:
+ case AArch64::BRAB:
+ case AArch64::BRAAZ:
+ case AArch64::BRABZ:
+ return true;
+ }
+ return false;
}
// struct TSFlags {
: Pseudo<(outs GPR32:$dst), (ins GPR32:$src), []>, Sched<[]>;
}
+// SpeculationBarrierEndBB must only be used after an unconditional control
+// flow, i.e. after a terminator for which isBarrier is True.
+let hasSideEffects = 1, isCodeGenOnly = 1, isTerminator = 1, isBarrier = 1 in {
+ def SpeculationBarrierISBDSBEndBB
+ : Pseudo<(outs), (ins), []>, Sched<[]>;
+ def SpeculationBarrierSBEndBB
+ : Pseudo<(outs), (ins), []>, Sched<[]>;
+}
//===----------------------------------------------------------------------===//
// System instructions.
--- /dev/null
+//===- AArch64SLSHardening.cpp - Harden Straight Line Missspeculation -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains a pass to insert code to mitigate against side channel
+// vulnerabilities that may happen under straight line miss-speculation.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AArch64InstrInfo.h"
+#include "AArch64Subtarget.h"
+#include "Utils/AArch64BaseInfo.h"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/RegisterScavenging.h"
+#include "llvm/IR/DebugLoc.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/CodeGen.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Target/TargetMachine.h"
+#include <cassert>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "aarch64-sls-hardening"
+
+#define AARCH64_SLS_HARDENING_NAME "AArch64 sls hardening pass"
+
+namespace {
+
+class AArch64SLSHardening : public MachineFunctionPass {
+public:
+ const TargetInstrInfo *TII;
+ const TargetRegisterInfo *TRI;
+ const AArch64Subtarget *ST;
+
+ static char ID;
+
+ AArch64SLSHardening() : MachineFunctionPass(ID) {
+ initializeAArch64SLSHardeningPass(*PassRegistry::getPassRegistry());
+ }
+
+ bool runOnMachineFunction(MachineFunction &Fn) override;
+
+ StringRef getPassName() const override { return AARCH64_SLS_HARDENING_NAME; }
+
+private:
+ bool hardenReturnsAndBRs(MachineBasicBlock &MBB) const;
+ void insertSpeculationBarrier(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MBBI,
+ DebugLoc DL) const;
+};
+
+} // end anonymous namespace
+
+char AArch64SLSHardening::ID = 0;
+
+INITIALIZE_PASS(AArch64SLSHardening, "aarch64-sls-hardening",
+ AARCH64_SLS_HARDENING_NAME, false, false)
+
+void AArch64SLSHardening::insertSpeculationBarrier(
+ MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+ DebugLoc DL) const {
+ assert(MBBI != MBB.begin() &&
+ "Must not insert SpeculationBarrierEndBB as only instruction in MBB.");
+ assert(std::prev(MBBI)->isBarrier() &&
+ "SpeculationBarrierEndBB must only follow unconditional control flow "
+ "instructions.");
+ assert(std::prev(MBBI)->isTerminator() &&
+ "SpeculatoinBarrierEndBB must only follow terminators.");
+ if (ST->hasSB())
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::SpeculationBarrierSBEndBB));
+ else
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::SpeculationBarrierISBDSBEndBB));
+}
+
+bool AArch64SLSHardening::runOnMachineFunction(MachineFunction &MF) {
+ ST = &MF.getSubtarget<AArch64Subtarget>();
+ TII = MF.getSubtarget().getInstrInfo();
+ TRI = MF.getSubtarget().getRegisterInfo();
+
+ bool Modified = false;
+ for (auto &MBB : MF)
+ Modified |= hardenReturnsAndBRs(MBB);
+
+ return Modified;
+}
+
+bool AArch64SLSHardening::hardenReturnsAndBRs(MachineBasicBlock &MBB) const {
+ if (!ST->hardenSlsRetBr())
+ return false;
+ bool Modified = false;
+ MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator(), E = MBB.end();
+ MachineBasicBlock::iterator NextMBBI;
+ for (; MBBI != E; MBBI = NextMBBI) {
+ MachineInstr &MI = *MBBI;
+ NextMBBI = std::next(MBBI);
+ if (MI.isReturn() || isIndirectBranchOpcode(MI.getOpcode())) {
+ assert(MI.isTerminator());
+ insertSpeculationBarrier(MBB, std::next(MBBI), MI.getDebugLoc());
+ Modified = true;
+ }
+ }
+ return Modified;
+}
+
+FunctionPass *llvm::createAArch64SLSHardeningPass() {
+ return new AArch64SLSHardening();
+}
bool UseEL2ForTP = false;
bool UseEL3ForTP = false;
bool AllowTaggedGlobals = false;
+ bool HardenSlsRetBr = false;
uint8_t MaxInterleaveFactor = 2;
uint8_t VectorInsertExtractBaseCost = 3;
uint16_t CacheLineSize = 0;
hasFuseCCSelect() || hasFuseLiterals();
}
+ bool hardenSlsRetBr() const { return HardenSlsRetBr; }
+
bool useEL1ForTP() const { return UseEL1ForTP; }
bool useEL2ForTP() const { return UseEL2ForTP; }
bool useEL3ForTP() const { return UseEL3ForTP; }
initializeLDTLSCleanupPass(*PR);
initializeSVEIntrinsicOptsPass(*PR);
initializeAArch64SpeculationHardeningPass(*PR);
+ initializeAArch64SLSHardeningPass(*PR);
initializeAArch64StackTaggingPass(*PR);
initializeAArch64StackTaggingPreRAPass(*PR);
}
// info.
addPass(createAArch64SpeculationHardeningPass());
+ addPass(createAArch64SLSHardeningPass());
+
if (TM->getOptLevel() != CodeGenOpt::None) {
if (EnableFalkorHWPFFix)
addPass(createFalkorHWPFFixPass());
AArch64PromoteConstant.cpp
AArch64PBQPRegAlloc.cpp
AArch64RegisterInfo.cpp
+ AArch64SLSHardening.cpp
AArch64SelectionDAGInfo.cpp
AArch64SpeculationHardening.cpp
AArch64StackTagging.cpp
; CHECK-NEXT: Post-RA pseudo instruction expansion pass
; CHECK-NEXT: AArch64 pseudo instruction expansion pass
; CHECK-NEXT: AArch64 speculation hardening pass
+; CHECK-NEXT: AArch64 sls hardening pass
; CHECK-NEXT: Analyze Machine Code For Garbage Collection
; CHECK-NEXT: Insert fentry calls
; CHECK-NEXT: Insert XRay ops
; CHECK-NEXT: AArch64 pseudo instruction expansion pass
; CHECK-NEXT: AArch64 load / store optimization pass
; CHECK-NEXT: AArch64 speculation hardening pass
+; CHECK-NEXT: AArch64 sls hardening pass
; CHECK-NEXT: MachineDominator Tree Construction
; CHECK-NEXT: Machine Natural Loop Construction
; CHECK-NEXT: Falkor HW Prefetch Fix Late Phase
--- /dev/null
+; RUN: llc -mattr=harden-sls-retbr -verify-machineinstrs -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,ISBDSB
+; RUN: llc -mattr=harden-sls-retbr -mattr=+sb -verify-machineinstrs -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,SB
+
+
+; Function Attrs: norecurse nounwind readnone
+define dso_local i32 @double_return(i32 %a, i32 %b) local_unnamed_addr {
+entry:
+ %cmp = icmp sgt i32 %a, 0
+ br i1 %cmp, label %if.then, label %if.else
+
+if.then: ; preds = %entry
+ %div = sdiv i32 %a, %b
+ ret i32 %div
+
+if.else: ; preds = %entry
+ %div1 = sdiv i32 %b, %a
+ ret i32 %div1
+; CHECK-LABEL: double_return:
+; CHECK: {{ret$}}
+; ISBDSB-NEXT: dsb sy
+; ISBDSB-NEXT: isb
+; SB-NEXT: {{ sb$}}
+; CHECK: {{ret$}}
+; ISBDSB-NEXT: dsb sy
+; ISBDSB-NEXT: isb
+; SB-NEXT: {{ sb$}}
+}
+
+@__const.indirect_branch.ptr = private unnamed_addr constant [2 x i8*] [i8* blockaddress(@indirect_branch, %return), i8* blockaddress(@indirect_branch, %l2)], align 8
+
+; Function Attrs: norecurse nounwind readnone
+define dso_local i32 @indirect_branch(i32 %a, i32 %b, i32 %i) {
+entry:
+ %idxprom = sext i32 %i to i64
+ %arrayidx = getelementptr inbounds [2 x i8*], [2 x i8*]* @__const.indirect_branch.ptr, i64 0, i64 %idxprom
+ %0 = load i8*, i8** %arrayidx, align 8
+ indirectbr i8* %0, [label %return, label %l2]
+
+l2: ; preds = %entry
+ br label %return
+
+return: ; preds = %entry, %l2
+ %retval.0 = phi i32 [ 1, %l2 ], [ 0, %entry ]
+ ret i32 %retval.0
+; CHECK-LABEL: indirect_branch:
+; CHECK: br x
+; ISBDSB-NEXT: dsb sy
+; ISBDSB-NEXT: isb
+; SB-NEXT: {{ sb$}}
+; CHECK: {{ret$}}
+; ISBDSB-NEXT: dsb sy
+; ISBDSB-NEXT: isb
+; SB-NEXT: {{ sb$}}
+}
+
+; Check that RETAA and RETAB instructions are also protected as expected.
+define dso_local i32 @ret_aa(i32 returned %a) local_unnamed_addr "target-features"="+neon,+v8.3a" "sign-return-address"="all" "sign-return-address-key"="a_key" {
+entry:
+; CHECK-LABEL: ret_aa:
+; CHECK: {{ retaa$}}
+; ISBDSB-NEXT: dsb sy
+; ISBDSB-NEXT: isb
+; SB-NEXT: {{ sb$}}
+ ret i32 %a
+}
+
+define dso_local i32 @ret_ab(i32 returned %a) local_unnamed_addr "target-features"="+neon,+v8.3a" "sign-return-address"="all" "sign-return-address-key"="b_key" {
+entry:
+; CHECK-LABEL: ret_ab:
+; CHECK: {{ retab$}}
+; ISBDSB-NEXT: dsb sy
+; ISBDSB-NEXT: isb
+; SB-NEXT: {{ sb$}}
+ ret i32 %a
+}
+
+define i32 @asmgoto() {
+entry:
+; CHECK-LABEL: asmgoto:
+ callbr void asm sideeffect "B $0", "X"(i8* blockaddress(@asmgoto, %d))
+ to label %asm.fallthrough [label %d]
+ ; The asm goto above produces a direct branch:
+; CHECK: //APP
+; CHECK-NEXT: {{^[ \t]+b }}
+; CHECK-NEXT: //NO_APP
+ ; For direct branches, no mitigation is needed.
+; ISDDSB-NOT: dsb sy
+; SB-NOT: {{ sb$}}
+
+asm.fallthrough: ; preds = %entry
+ ret i32 0
+; CHECK: {{ret$}}
+; ISBDSB-NEXT: dsb sy
+; ISBDSB-NEXT: isb
+; SB-NEXT: {{ sb$}}
+
+d: ; preds = %asm.fallthrough, %entry
+ ret i32 1
+; CHECK: {{ret$}}
+; ISBDSB-NEXT: dsb sy
+; ISBDSB-NEXT: isb
+; SB-NEXT: {{ sb$}}
+; CHECK-NEXT: .Lfunc_end
+}
--- /dev/null
+# RUN: llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu \
+# RUN: -start-before aarch64-sls-hardening -o - %s \
+# RUN: -mattr=harden-sls-retbr \
+# RUN: | FileCheck %s --check-prefixes=CHECK,ISBDSB
+# RUN: llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu \
+# RUN: -start-before aarch64-sls-hardening -o - %s \
+# RUN: -mattr=harden-sls-retbr -mattr=+sb \
+# RUN: | FileCheck %s --check-prefixes=CHECK,SB
+
+# Check that the SLS hardening pass also protects BRA* indirect branches that
+# llvm currently does not generate.
+--- |
+ @ptr_aa = private unnamed_addr constant [2 x i8*] [i8* blockaddress(@br_aa, %return), i8* blockaddress(@br_aa, %l2)], align 8
+ @ptr_aaz = private unnamed_addr constant [2 x i8*] [i8* blockaddress(@br_aaz, %return), i8* blockaddress(@br_aaz, %l2)], align 8
+ @ptr_ab = private unnamed_addr constant [2 x i8*] [i8* blockaddress(@br_ab, %return), i8* blockaddress(@br_ab, %l2)], align 8
+ @ptr_abz = private unnamed_addr constant [2 x i8*] [i8* blockaddress(@br_abz, %return), i8* blockaddress(@br_abz, %l2)], align 8
+
+ define dso_local i32 @br_aa(i32 %a, i32 %b, i32 %i) {
+ entry:
+ br label %l2
+ l2:
+ br label %return
+ return:
+ ret i32 undef
+ }
+ define dso_local i32 @br_aaz(i32 %a, i32 %b, i32 %i) {
+ entry:
+ br label %l2
+ l2:
+ br label %return
+ return:
+ ret i32 undef
+ }
+ define dso_local i32 @br_ab(i32 %a, i32 %b, i32 %i) {
+ entry:
+ br label %l2
+ l2:
+ br label %return
+ return:
+ ret i32 undef
+ }
+ define dso_local i32 @br_abz(i32 %a, i32 %b, i32 %i) {
+ entry:
+ br label %l2
+ l2:
+ br label %return
+ return:
+ ret i32 undef
+ }
+...
+---
+name: br_aa
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: br_aa:
+ bb.0.entry:
+ successors: %bb.2, %bb.1
+ liveins: $w2
+ $x8 = ADRP target-flags(aarch64-page) @ptr_aa
+ renamable $x8 = ADDXri $x8, target-flags(aarch64-pageoff, aarch64-nc) @ptr_aa, 0
+ renamable $x8 = LDRXroW killed renamable $x8, killed renamable $w2, 1, 1
+ BRAA killed renamable $x8, $sp
+ ; CHECK: braa x8, sp
+ ; ISBDSB-NEXT: dsb sy
+ ; ISBDSB-NEXT: isb
+ ; SB-NEXT: {{ sb$}}
+
+ bb.1.l2 (address-taken):
+ renamable $w0 = MOVZWi 1, 0
+ RET undef $lr, implicit $w0
+
+ bb.2.return (address-taken):
+ $w0 = ORRWrs $wzr, $wzr, 0
+ RET undef $lr, implicit $w0
+...
+---
+name: br_aaz
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: br_aaz:
+ bb.0.entry:
+ successors: %bb.2, %bb.1
+ liveins: $w2
+ $x8 = ADRP target-flags(aarch64-page) @ptr_aaz
+ renamable $x8 = ADDXri $x8, target-flags(aarch64-pageoff, aarch64-nc) @ptr_aaz, 0
+ renamable $x8 = LDRXroW killed renamable $x8, killed renamable $w2, 1, 1
+ BRAAZ killed renamable $x8
+ ; CHECK: braaz x8
+ ; ISBDSB-NEXT: dsb sy
+ ; ISBDSB-NEXT: isb
+ ; SB-NEXT: {{ sb$}}
+
+ bb.1.l2 (address-taken):
+ renamable $w0 = MOVZWi 1, 0
+ RET undef $lr, implicit $w0
+
+ bb.2.return (address-taken):
+ $w0 = ORRWrs $wzr, $wzr, 0
+ RET undef $lr, implicit $w0
+...
+---
+name: br_ab
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: br_ab:
+ bb.0.entry:
+ successors: %bb.2, %bb.1
+ liveins: $w2
+ $x8 = ADRP target-flags(aarch64-page) @ptr_ab
+ renamable $x8 = ADDXri $x8, target-flags(aarch64-pageoff, aarch64-nc) @ptr_ab, 0
+ renamable $x8 = LDRXroW killed renamable $x8, killed renamable $w2, 1, 1
+ BRAA killed renamable $x8, $sp
+ ; CHECK: braa x8, sp
+ ; ISBDSB-NEXT: dsb sy
+ ; ISBDSB-NEXT: isb
+ ; SB-NEXT: {{ sb$}}
+
+ bb.1.l2 (address-taken):
+ renamable $w0 = MOVZWi 1, 0
+ RET undef $lr, implicit $w0
+
+ bb.2.return (address-taken):
+ $w0 = ORRWrs $wzr, $wzr, 0
+ RET undef $lr, implicit $w0
+...
+---
+name: br_abz
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: br_abz:
+ bb.0.entry:
+ successors: %bb.2, %bb.1
+ liveins: $w2
+ $x8 = ADRP target-flags(aarch64-page) @ptr_abz
+ renamable $x8 = ADDXri $x8, target-flags(aarch64-pageoff, aarch64-nc) @ptr_abz, 0
+ renamable $x8 = LDRXroW killed renamable $x8, killed renamable $w2, 1, 1
+ BRAAZ killed renamable $x8
+ ; CHECK: braaz x8
+ ; ISBDSB-NEXT: dsb sy
+ ; ISBDSB-NEXT: isb
+ ; SB-NEXT: {{ sb$}}
+
+ bb.1.l2 (address-taken):
+ renamable $w0 = MOVZWi 1, 0
+ RET undef $lr, implicit $w0
+
+ bb.2.return (address-taken):
+ $w0 = ORRWrs $wzr, $wzr, 0
+ RET undef $lr, implicit $w0
+...