// code section (see IHI0056B). $d identifies a symbol starting data contents.
if (BC.isAArch64() && Symbol.getType() &&
cantFail(Symbol.getType()) == SymbolRef::ST_Unknown && SymbolSize == 0 &&
- Symbol.getName() && cantFail(Symbol.getName()) == "$d")
+ Symbol.getName() &&
+ (cantFail(Symbol.getName()) == "$d" ||
+ cantFail(Symbol.getName()).startswith("$d.")))
return true;
return false;
}
// end of a data chunk inside code.
if (BC.isAArch64() && Symbol.getType() &&
cantFail(Symbol.getType()) == SymbolRef::ST_Unknown && SymbolSize == 0 &&
- Symbol.getName() && cantFail(Symbol.getName()) == "$x")
+ Symbol.getName() &&
+ (cantFail(Symbol.getName()) == "$x" ||
+ cantFail(Symbol.getName()).startswith("$x.")))
return true;
return false;
}
case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
case ELF::R_AARCH64_LDST128_ABS_LO12_NC:
case ELF::R_AARCH64_ADR_GOT_PAGE:
+ case ELF::R_AARCH64_TLSDESC_ADR_PREL21:
case ELF::R_AARCH64_TLSDESC_ADR_PAGE21:
+ case ELF::R_AARCH64_ADR_PREL_LO21:
case ELF::R_AARCH64_ADR_PREL_PG_HI21:
+ case ELF::R_AARCH64_ADR_PREL_PG_HI21_NC:
Relocations[Offset] = Relocation{Offset, Symbol, RelType, Addend, Value};
break;
case ELF::R_X86_64_PC32:
//===----------------------------------------------------------------------===//
#include "BinaryPassManager.h"
+#include "Passes/ADRRelaxationPass.h"
#include "Passes/Aligner.h"
#include "Passes/AllocCombiner.h"
#include "Passes/FrameOptimizer.h"
opts::ICF);
if (BC.isAArch64())
- Manager.registerPass(
- std::make_unique<VeneerElimination>(PrintVeneerElimination));
+ Manager.registerPass(
+ std::make_unique<VeneerElimination>(PrintVeneerElimination));
Manager.registerPass(
std::make_unique<SpecializeMemcpy1>(NeverPrint, opts::SpecializeMemcpy1),
if (BC.HasRelocations)
Manager.registerPass(std::make_unique<PatchEntries>());
- // Tighten branches according to offset differences between branch and
- // targets. No extra instructions after this pass, otherwise we may have
- // relocations out of range and crash during linking.
- if (BC.isAArch64())
+ if (BC.isAArch64()) {
+ Manager.registerPass(std::make_unique<ADRRelaxationPass>());
+
+ // Tighten branches according to offset differences between branch and
+ // targets. No extra instructions after this pass, otherwise we may have
+ // relocations out of range and crash during linking.
Manager.registerPass(std::make_unique<LongJmpPass>(PrintLongJmp));
+ }
// This pass turns tail calls into jumps which makes them invisible to
// function reordering. It's unsafe to use any CFG or instruction analysis
return false;
}
+ virtual void getADRReg(const MCInst &Inst, MCPhysReg &RegName) const {
+ llvm_unreachable("not implemented");
+ }
+
virtual bool isMoveMem2Reg(const MCInst &Inst) const {
llvm_unreachable("not implemented");
return false;
return &cast<const MCSymbolRefExpr>(Expr)->getSymbol();
}
+ /// Return addend that represents an offset from MCSymbol target
+ /// of this instruction at a given operand number \p OpNum.
+ /// If there's no symbol associated with the operand - return 0
+ virtual int64_t getTargetAddend(const MCInst &Inst,
+ unsigned OpNum = 0) const {
+ llvm_unreachable("not implemented");
+ return 0;
+ }
+
+ /// Return MCSymbol addend extracted from a target expression
+ virtual int64_t getTargetAddend(const MCExpr *Expr) const {
+ llvm_unreachable("not implemented");
+ return 0;
+ }
+
/// Return MCSymbol/offset extracted from a target expression
virtual std::pair<const MCSymbol *, uint64_t>
getTargetSymbolInfo(const MCExpr *Expr) const {
return false;
}
+ /// Store \p Target absolute adddress to \p RegName
+ virtual std::vector<MCInst> materializeAddress(const MCSymbol *Target,
+ MCContext *Ctx,
+ MCPhysReg RegName,
+ int64_t Addend = 0) const {
+ llvm_unreachable("not implemented");
+ return {};
+ }
+
/// Creates a new unconditional branch instruction in Inst and set its operand
/// to TBB.
///
--- /dev/null
+//===--- ADRRelaxationPass.cpp --------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "ADRRelaxationPass.h"
+#include "ParallelUtilities.h"
+
+using namespace llvm;
+
+namespace opts {
+extern cl::OptionCategory BoltCategory;
+
+static cl::opt<bool>
+ AdrPassOpt("adr-relaxation",
+ cl::desc("Replace ARM non-local ADR instructions with ADRP"),
+ cl::init(true), cl::cat(BoltCategory), cl::ReallyHidden);
+} // namespace opts
+
+namespace llvm {
+namespace bolt {
+
+void ADRRelaxationPass::runOnFunction(BinaryContext &BC, BinaryFunction &BF) {
+ for (BinaryBasicBlock *BB : BF.layout()) {
+ for (auto It = BB->begin(); It != BB->end(); ++It) {
+ MCInst &Inst = *It;
+ if (!BC.MIB->isADR(Inst))
+ continue;
+
+ const MCSymbol *Symbol = BC.MIB->getTargetSymbol(Inst);
+ if (!Symbol)
+ continue;
+
+ BinaryFunction::IslandInfo &Islands = BF.getIslandInfo();
+ if (Islands.Symbols.count(Symbol) || Islands.ProxySymbols.count(Symbol))
+ continue;
+
+ BinaryFunction *TargetBF = BC.getFunctionForSymbol(Symbol);
+ if (TargetBF && TargetBF == &BF)
+ continue;
+
+ MCPhysReg Reg;
+ BC.MIB->getADRReg(Inst, Reg);
+ int64_t Addend = BC.MIB->getTargetAddend(Inst);
+ std::vector<MCInst> Addr =
+ BC.MIB->materializeAddress(Symbol, BC.Ctx.get(), Reg, Addend);
+ It = BB->replaceInstruction(It, Addr);
+ }
+ }
+}
+
+void ADRRelaxationPass::runOnFunctions(BinaryContext &BC) {
+ if (!opts::AdrPassOpt || !BC.HasRelocations)
+ return;
+
+ ParallelUtilities::WorkFuncTy WorkFun = [&](BinaryFunction &BF) {
+ runOnFunction(BC, BF);
+ };
+
+ ParallelUtilities::runOnEachFunction(
+ BC, ParallelUtilities::SchedulingPolicy::SP_TRIVIAL, WorkFun, nullptr,
+ "ADRRelaxationPass", /* ForceSequential */ true);
+}
+
+} // end namespace bolt
+} // end namespace llvm
--- /dev/null
+//===--------- Passes/ADRRelaxationPass.h ---------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVM_BOLT_PASSES_ADRRELAXATIONPASS_H
+#define LLVM_TOOLS_LLVM_BOLT_PASSES_ADRRELAXATIONPASS_H
+
+#include "BinaryPasses.h"
+
+// This pass replaces AArch64 non-local ADR instructions
+// with ADRP + ADD due to small offset range of ADR instruction
+// (+- 1MB) which could be easely overflowed after BOLT optimizations
+// Such problems are usually connected with errata 843419
+// https://developer.arm.com/documentation/epm048406/2100/
+// The linker could replace ADRP instruction with ADR in some cases.
+
+namespace llvm {
+namespace bolt {
+
+class ADRRelaxationPass : public BinaryFunctionPass {
+public:
+ explicit ADRRelaxationPass() : BinaryFunctionPass(false) {}
+
+ const char *getName() const override { return "adr-relaxation"; }
+
+ /// Pass entry point
+ void runOnFunctions(BinaryContext &BC) override;
+ void runOnFunction(BinaryContext &BC, BinaryFunction &BF);
+};
+
+} // namespace bolt
+} // namespace llvm
+
+#endif
add_llvm_library(LLVMBOLTPasses
+ ADRRelaxationPass.cpp
Aligner.cpp
AllocCombiner.cpp
BinaryPasses.cpp
default:
return false;
case ELF::R_AARCH64_CALL26:
+ case ELF::R_AARCH64_ADR_PREL_LO21:
case ELF::R_AARCH64_ADR_PREL_PG_HI21:
+ case ELF::R_AARCH64_ADR_PREL_PG_HI21_NC:
case ELF::R_AARCH64_LDST64_ABS_LO12_NC:
case ELF::R_AARCH64_ADD_ABS_LO12_NC:
case ELF::R_AARCH64_LDST128_ABS_LO12_NC:
case ELF::R_AARCH64_LDST16_ABS_LO12_NC:
case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
case ELF::R_AARCH64_ADR_GOT_PAGE:
+ case ELF::R_AARCH64_TLSDESC_ADR_PREL21:
case ELF::R_AARCH64_TLSDESC_ADR_PAGE21:
case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
case ELF::R_AARCH64_TLSLE_ADD_TPREL_HI12:
dbgs() << "Reloc num: " << Type << "\n";
llvm_unreachable("unsupported relocation type");
case ELF::R_AARCH64_CALL26:
+ case ELF::R_AARCH64_ADR_PREL_LO21:
case ELF::R_AARCH64_ADR_PREL_PG_HI21:
+ case ELF::R_AARCH64_ADR_PREL_PG_HI21_NC:
case ELF::R_AARCH64_LDST64_ABS_LO12_NC:
case ELF::R_AARCH64_ADD_ABS_LO12_NC:
case ELF::R_AARCH64_LDST128_ABS_LO12_NC:
case ELF::R_AARCH64_LDST16_ABS_LO12_NC:
case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
case ELF::R_AARCH64_ADR_GOT_PAGE:
+ case ELF::R_AARCH64_TLSDESC_ADR_PREL21:
case ELF::R_AARCH64_TLSDESC_ADR_PAGE21:
case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
case ELF::R_AARCH64_TLSLE_ADD_TPREL_HI12:
Contents &= ~0xfffffffffc000000ULL;
return static_cast<int64_t>(PC) + SignExtend64<28>(Contents << 2);
case ELF::R_AARCH64_ADR_GOT_PAGE:
+ case ELF::R_AARCH64_TLSDESC_ADR_PREL21:
case ELF::R_AARCH64_TLSDESC_ADR_PAGE21:
case ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
- case ELF::R_AARCH64_ADR_PREL_PG_HI21: {
+ case ELF::R_AARCH64_ADR_PREL_LO21:
+ case ELF::R_AARCH64_ADR_PREL_PG_HI21:
+ case ELF::R_AARCH64_ADR_PREL_PG_HI21_NC: {
// Bits 32:12 of Symbol address goes in bits 30:29 + 23:5 of ADRP
- // instruction
+ // and ADR instructions
+ bool IsAdr = !!(((Contents >> 31) & 0x1) == 0);
Contents &= ~0xffffffff9f00001fUll;
uint64_t LowBits = (Contents >> 29) & 0x3;
uint64_t HighBits = (Contents >> 5) & 0x7ffff;
Contents = LowBits | (HighBits << 2);
+ if (IsAdr)
+ return static_cast<int64_t>(PC) + SignExtend64<21>(Contents);
+
+ // ADRP instruction
Contents = static_cast<int64_t>(PC) + SignExtend64<33>(Contents << 12);
Contents &= ~0xfffUll;
return Contents;
case ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
case ELF::R_AARCH64_TLSLE_ADD_TPREL_HI12:
case ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
+ case ELF::R_AARCH64_TLSDESC_ADR_PREL21:
case ELF::R_AARCH64_TLSDESC_ADR_PAGE21:
case ELF::R_AARCH64_TLSDESC_LD64_LO12:
case ELF::R_AARCH64_TLSDESC_ADD_LO12:
switch (Type) {
default:
return false;
+ case ELF::R_AARCH64_TLSDESC_ADR_PREL21:
case ELF::R_AARCH64_TLSDESC_ADR_PAGE21:
case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
case ELF::R_AARCH64_TLSLE_ADD_TPREL_HI12:
return false;
case ELF::R_AARCH64_TLSDESC_CALL:
case ELF::R_AARCH64_CALL26:
+ case ELF::R_AARCH64_ADR_PREL_LO21:
case ELF::R_AARCH64_ADR_PREL_PG_HI21:
+ case ELF::R_AARCH64_ADR_PREL_PG_HI21_NC:
case ELF::R_AARCH64_ADR_GOT_PAGE:
case ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
+ case ELF::R_AARCH64_TLSDESC_ADR_PREL21:
case ELF::R_AARCH64_TLSDESC_ADR_PAGE21:
case ELF::R_AARCH64_JUMP26:
case ELF::R_AARCH64_PREL32:
[](const SymbolRef &Symbol) {
StringRef Name = cantFail(Symbol.getName());
return !(cantFail(Symbol.getType()) == SymbolRef::ST_Unknown &&
- (Name == "$d" || Name == "$x"));
+ (Name == "$d" || Name.startswith("$d.") || Name == "$x" ||
+ Name.startswith("$x.")));
});
--LastSymbol;
}
return Inst.getOpcode() == AArch64::ADR;
}
+ void getADRReg(const MCInst &Inst, MCPhysReg &RegName) const override {
+ assert((isADR(Inst) || isADRP(Inst)) && "Not an ADR instruction");
+ assert(MCPlus::getNumPrimeOperands(Inst) != 0 &&
+ "No operands for ADR instruction");
+ assert(Inst.getOperand(0).isReg() &&
+ "Unexpected operand in ADR instruction");
+ RegName = Inst.getOperand(0).getReg();
+ }
+
bool isTB(const MCInst &Inst) const {
return (Inst.getOpcode() == AArch64::TBNZW ||
Inst.getOpcode() == AArch64::TBNZX ||
const MCExpr *getTargetExprFor(MCInst &Inst, const MCExpr *Expr,
MCContext &Ctx,
uint64_t RelType) const override {
- if (RelType == ELF::R_AARCH64_ADR_GOT_PAGE ||
- RelType == ELF::R_AARCH64_TLSDESC_ADR_PAGE21) {
+
+ if (isADR(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_LO21 ||
+ RelType == ELF::R_AARCH64_TLSDESC_ADR_PREL21) {
+ return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS, Ctx);
+ } else if (isADRP(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21 ||
+ RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21_NC ||
+ RelType == ELF::R_AARCH64_TLSDESC_ADR_PAGE21 ||
+ RelType == ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 ||
+ RelType == ELF::R_AARCH64_ADR_GOT_PAGE) {
// Never emit a GOT reloc, we handled this in
// RewriteInstance::readRelocations().
return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE, Ctx);
- } else if (Inst.getOpcode() == AArch64::ADRP) {
- return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE, Ctx);
} else {
switch(RelType) {
case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
return Expr;
}
+ bool getSymbolRefOperandNum(const MCInst &Inst, unsigned &OpNum) const {
+ if (OpNum >= MCPlus::getNumPrimeOperands(Inst))
+ return false;
+
+ // Auto-select correct operand number
+ if (OpNum == 0) {
+ if (isConditionalBranch(Inst) || isADR(Inst) || isADRP(Inst))
+ OpNum = 1;
+ if (isTB(Inst))
+ OpNum = 2;
+ if (isMOVW(Inst))
+ OpNum = 1;
+ }
+
+ return true;
+ }
+
const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override {
auto *AArchExpr = dyn_cast<AArch64MCExpr>(Expr);
- if (AArchExpr && AArchExpr->getSubExpr()) {
+ if (AArchExpr && AArchExpr->getSubExpr())
return getTargetSymbol(AArchExpr->getSubExpr());
- }
+
+ auto *BinExpr = dyn_cast<MCBinaryExpr>(Expr);
+ if (BinExpr)
+ return getTargetSymbol(BinExpr->getLHS());
auto *SymExpr = dyn_cast<MCSymbolRefExpr>(Expr);
- if (!SymExpr || SymExpr->getKind() != MCSymbolRefExpr::VK_None)
- return nullptr;
+ if (SymExpr && SymExpr->getKind() == MCSymbolRefExpr::VK_None)
+ return &SymExpr->getSymbol();
- return &SymExpr->getSymbol();
+ return nullptr;
}
const MCSymbol *getTargetSymbol(const MCInst &Inst,
unsigned OpNum = 0) const override {
- if (OpNum >= MCPlus::getNumPrimeOperands(Inst))
+ if (!getSymbolRefOperandNum(Inst, OpNum))
return nullptr;
- // Auto-select correct operand number
- if (OpNum == 0) {
- if (isConditionalBranch(Inst))
- OpNum = 1;
- if (isTB(Inst))
- OpNum = 2;
- if (isMOVW(Inst))
- OpNum = 1;
- }
-
const MCOperand &Op = Inst.getOperand(OpNum);
if (!Op.isExpr())
return nullptr;
return getTargetSymbol(Op.getExpr());
}
+ int64_t getTargetAddend(const MCExpr *Expr) const override {
+ auto *AArchExpr = dyn_cast<AArch64MCExpr>(Expr);
+ if (AArchExpr && AArchExpr->getSubExpr())
+ return getTargetAddend(AArchExpr->getSubExpr());
+
+ auto *BinExpr = dyn_cast<MCBinaryExpr>(Expr);
+ if (BinExpr && BinExpr->getOpcode() == MCBinaryExpr::Add)
+ return getTargetAddend(BinExpr->getRHS());
+
+ auto *ConstExpr = dyn_cast<MCConstantExpr>(Expr);
+ if (ConstExpr)
+ return ConstExpr->getValue();
+
+ return 0;
+ }
+
+ int64_t getTargetAddend(const MCInst &Inst,
+ unsigned OpNum = 0) const override {
+ if (!getSymbolRefOperandNum(Inst, OpNum))
+ return 0;
+
+ const MCOperand &Op = Inst.getOperand(OpNum);
+ if (!Op.isExpr())
+ return 0;
+
+ return getTargetAddend(Op.getExpr());
+ }
+
bool evaluateBranch(const MCInst &Inst, uint64_t Addr, uint64_t Size,
uint64_t &Target) const override {
size_t OpNum = 0;
Inst.addOperand(MCOperand::createReg(AArch64::LR));
return true;
}
+
+ std::vector<MCInst> materializeAddress(const MCSymbol *Target, MCContext *Ctx,
+ MCPhysReg RegName,
+ int64_t Addend) const override {
+ // Get page-aligned address and add page offset
+ std::vector<MCInst> Insts(2);
+ Insts[0].setOpcode(AArch64::ADRP);
+ Insts[0].clear();
+ Insts[0].addOperand(MCOperand::createReg(RegName));
+ Insts[0].addOperand(MCOperand::createImm(0));
+ setOperandToSymbolRef(Insts[0], /* OpNum */ 1, Target, Addend, Ctx,
+ ELF::R_AARCH64_NONE);
+ Insts[1].setOpcode(AArch64::ADDXri);
+ Insts[1].clear();
+ Insts[1].addOperand(MCOperand::createReg(RegName));
+ Insts[1].addOperand(MCOperand::createReg(RegName));
+ Insts[1].addOperand(MCOperand::createImm(0));
+ Insts[1].addOperand(MCOperand::createImm(0));
+ setOperandToSymbolRef(Insts[1], /* OpNum */ 2, Target, Addend, Ctx,
+ ELF::R_AARCH64_ADD_ABS_LO12_NC);
+ return Insts;
+ }
};
} // end anonymous namespace
--- /dev/null
+# The second and third ADR instructions are non-local to functions
+# and must be replaced with ADRP + ADD by BOLT
+
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown \
+# RUN: %s -o %t.o
+# RUN: %host_cc %cflags %t.o -o %t.exe -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.bolt -adr-relaxation=true
+# RUN: llvm-objdump -d --disassemble-symbols=main %t.bolt | FileCheck %s
+# RUN: %t.bolt
+
+ .data
+ .align 8
+ .global Gvar
+Gvar: .dword 0x0
+ .global Gvar2
+Gvar2: .dword 0x42
+
+ .text
+ .align 4
+ .global test
+ .type test, %function
+test:
+ mov x0, xzr
+ ret
+ .size test, .-test
+
+ .align 4
+ .global main
+ .type main, %function
+main:
+ adr x0, .CI
+ adr x1, test
+ adr x2, Gvar2
+ adr x3, br
+br:
+ br x1
+ .size main, .-main
+.CI:
+ .word 0xff
+
+# CHECK: <main>:
+# CHECK-NEXT: adr x0, #28
+# CHECK-NEXT: adrp x1, 0x{{[1-8a-f][0-9a-f]*}}
+# CHECK-NEXT: add x1, x1, #{{[1-8a-f][0-9a-f]*}}
+# CHECK-NEXT: adrp x2, 0x{{[1-8a-f][0-9a-f]*}}
+# CHECK-NEXT: add x2, x2, #{{[1-8a-f][0-9a-f]*}}
+# CHECK-NEXT: adr x3, #4
--- /dev/null
+if config.host_arch not in ['AArch64']:
+ config.unsupported = True
--- /dev/null
+if config.host_arch not in ['x86', 'X86', 'x86_64']:
+ config.unsupported = True
projects / platform / upstream / llvm.git / commitdiff