uint32_t DuplicatedJumpTables{0x10000000};
/// Function fragments to skip.
- std::vector<BinaryFunction *> FragmentsToSkip;
+ std::unordered_set<BinaryFunction *> FragmentsToSkip;
/// The runtime library.
std::unique_ptr<RuntimeLibrary> RtLibrary;
MIB = std::move(TargetBuilder);
}
+ /// Return function fragments to skip.
+ const std::unordered_set<BinaryFunction *> &getFragmentsToSkip() {
+ return FragmentsToSkip;
+ }
+
+ /// Add function fragment to skip
+ void addFragmentsToSkip(BinaryFunction *Function) {
+ FragmentsToSkip.insert(Function);
+ }
+
+ void clearFragmentsToSkip() { FragmentsToSkip.clear(); }
+
/// Given DWOId returns CU if it exists in DWOCUs.
Optional<DWARFUnit *> getDWOCU(uint64_t DWOId);
/// If \p NextJTAddress is different from zero, it is used as an upper
/// bound for jump table memory layout.
///
- /// Optionally, populate \p Offsets with jump table entries. The entries
+ /// Optionally, populate \p Address from jump table entries. The entries
/// could be partially populated if the jump table detection fails.
bool analyzeJumpTable(const uint64_t Address,
const JumpTable::JumpTableType Type, BinaryFunction &BF,
const uint64_t NextJTAddress = 0,
- JumpTable::OffsetsType *Offsets = nullptr);
+ JumpTable::AddressesType *EntriesAsAddress = nullptr);
/// After jump table locations are established, this function will populate
- /// their OffsetEntries based on memory contents.
+ /// their EntriesAsAddress based on memory contents.
void populateJumpTables();
/// Returns a jump table ID and label pointing to the duplicated jump table.
/// to function \p BF.
std::string generateJumpTableName(const BinaryFunction &BF, uint64_t Address);
- /// Free memory used by jump table offsets
- void clearJumpTableOffsets() {
+ /// Free memory used by JumpTable's EntriesAsAddress
+ void clearJumpTableTempData() {
for (auto &JTI : JumpTables) {
JumpTable &JT = *JTI.second;
- JumpTable::OffsetsType Temp;
- Temp.swap(JT.OffsetEntries);
+ JumpTable::AddressesType Temp;
+ Temp.swap(JT.EntriesAsAddress);
}
}
/// Return true if the array of bytes represents a valid code padding.
/// True if the original entry point was patched.
bool IsPatched{false};
- /// True if the function contains jump table with entries pointing to
- /// locations in fragments.
- bool HasSplitJumpTable{false};
+ /// True if the function contains explicit or implicit indirect branch to its
+ /// split fragments, e.g., split jump table, landing pad in split fragment
+ bool HasIndirectTargetToSplitFragment{false};
/// True if there are no control-flow edges with successors in other functions
/// (i.e. if tail calls have edges to function-local basic blocks).
/// otherwise processed.
bool isPseudo() const { return IsPseudo; }
- /// Return true if the function contains a jump table with entries pointing
- /// to split fragments.
- bool hasSplitJumpTable() const { return HasSplitJumpTable; }
+ /// Return true if the function contains explicit or implicit indirect branch
+ /// to its split fragments, e.g., split jump table, landing pad in split
+ /// fragment.
+ bool hasIndirectTargetToSplitFragment() const {
+ return HasIndirectTargetToSplitFragment;
+ }
/// Return true if all CFG edges have local successors.
bool hasCanonicalCFG() const { return HasCanonicalCFG; }
void setIsPatched(bool V) { IsPatched = V; }
- void setHasSplitJumpTable(bool V) { HasSplitJumpTable = V; }
+ void setHasIndirectTargetToSplitFragment(bool V) {
+ HasIndirectTargetToSplitFragment = V;
+ }
void setHasCanonicalCFG(bool V) { HasCanonicalCFG = V; }
/// All the entries as labels.
std::vector<MCSymbol *> Entries;
- /// All the entries as offsets into a function. Invalid after CFG is built.
- using OffsetsType = std::vector<uint64_t>;
- OffsetsType OffsetEntries;
+ /// All the entries as absolute addresses. Invalid after disassembly is done.
+ using AddressesType = std::vector<uint64_t>;
+ AddressesType EntriesAsAddress;
/// Map <Offset> -> <Label> used for embedded jump tables. Label at 0 offset
/// is the main label for the jump table.
uint64_t Count{0};
/// BinaryFunction this jump tables belongs to.
- BinaryFunction *Parent{nullptr};
+ SmallVector<BinaryFunction *, 1> Parents;
private:
/// Constructor should only be called by a BinaryContext.
JumpTable(MCSymbol &Symbol, uint64_t Address, size_t EntrySize,
- JumpTableType Type, LabelMapType &&Labels, BinaryFunction &BF,
- BinarySection &Section);
+ JumpTableType Type, LabelMapType &&Labels, BinarySection &Section);
public:
/// Return the size of the jump table.
uint64_t getSize() const {
- return std::max(OffsetEntries.size(), Entries.size()) * EntrySize;
+ return std::max(EntriesAsAddress.size(), Entries.size()) * EntrySize;
}
const MCSymbol *getFirstLabel() const {
return false;
}
-bool BinaryContext::analyzeJumpTable(const uint64_t Address,
- const JumpTable::JumpTableType Type,
- BinaryFunction &BF,
- const uint64_t NextJTAddress,
- JumpTable::OffsetsType *Offsets) {
+bool BinaryContext::analyzeJumpTable(
+ const uint64_t Address, const JumpTable::JumpTableType Type,
+ BinaryFunction &BF, const uint64_t NextJTAddress,
+ JumpTable::AddressesType *EntriesAsAddress) {
// Is one of the targets __builtin_unreachable?
bool HasUnreachable = false;
// Number of targets other than __builtin_unreachable.
uint64_t NumRealEntries = 0;
- constexpr uint64_t INVALID_OFFSET = std::numeric_limits<uint64_t>::max();
- auto addOffset = [&](uint64_t Offset) {
- if (Offsets)
- Offsets->emplace_back(Offset);
+ auto addEntryAddress = [&](uint64_t EntryAddress) {
+ if (EntriesAsAddress)
+ EntriesAsAddress->emplace_back(EntryAddress);
};
auto doesBelongToFunction = [&](const uint64_t Addr,
// __builtin_unreachable() case.
if (Value == BF.getAddress() + BF.getSize()) {
- addOffset(Value - BF.getAddress());
+ addEntryAddress(Value);
HasUnreachable = true;
LLVM_DEBUG(dbgs() << "OK: __builtin_unreachable\n");
continue;
++NumRealEntries;
- if (TargetBF == &BF) {
- // Address inside the function.
- addOffset(Value - TargetBF->getAddress());
- LLVM_DEBUG(dbgs() << "OK: real entry\n");
- } else {
- // Address in split fragment.
- BF.setHasSplitJumpTable(true);
- // Add invalid offset for proper identification of jump table size.
- addOffset(INVALID_OFFSET);
- LLVM_DEBUG(dbgs() << "OK: address in split fragment "
- << TargetBF->getPrintName() << '\n');
- }
+ if (TargetBF != &BF)
+ BF.setHasIndirectTargetToSplitFragment(true);
+ addEntryAddress(Value);
}
// It's a jump table if the number of real entries is more than 1, or there's
for (auto JTI = JumpTables.begin(), JTE = JumpTables.end(); JTI != JTE;
++JTI) {
JumpTable *JT = JTI->second;
- BinaryFunction &BF = *JT->Parent;
- if (!BF.isSimple())
+ bool NonSimpleParent = false;
+ for (BinaryFunction *BF : JT->Parents)
+ NonSimpleParent |= !BF->isSimple();
+ if (NonSimpleParent)
continue;
uint64_t NextJTAddress = 0;
if (NextJTI != JTE)
NextJTAddress = NextJTI->second->getAddress();
- const bool Success = analyzeJumpTable(JT->getAddress(), JT->Type, BF,
- NextJTAddress, &JT->OffsetEntries);
+ const bool Success =
+ analyzeJumpTable(JT->getAddress(), JT->Type, *(JT->Parents[0]),
+ NextJTAddress, &JT->EntriesAsAddress);
if (!Success) {
- dbgs() << "failed to analyze jump table in function " << BF << '\n';
+ LLVM_DEBUG(ListSeparator LS;
+ dbgs() << "failed to analyze jump table in function ";
+ for (BinaryFunction *Frag
+ : JT->Parents) dbgs()
+ << LS << *Frag;
+ dbgs() << '\n';);
JT->print(dbgs());
if (NextJTI != JTE) {
- dbgs() << "next jump table at 0x"
- << Twine::utohexstr(NextJTI->second->getAddress())
- << " belongs to function " << *NextJTI->second->Parent << '\n';
+ LLVM_DEBUG(ListSeparator LS;
+ dbgs() << "next jump table at 0x"
+ << Twine::utohexstr(NextJTI->second->getAddress())
+ << " belongs to function ";
+ for (BinaryFunction *Frag
+ : NextJTI->second->Parents) dbgs()
+ << LS << *Frag;
+ dbgs() << "\n";);
NextJTI->second->print(dbgs());
}
llvm_unreachable("jump table heuristic failure");
}
-
- for (uint64_t EntryOffset : JT->OffsetEntries) {
- if (EntryOffset == BF.getSize())
- BF.IgnoredBranches.emplace_back(EntryOffset, BF.getSize());
- else
- BF.registerReferencedOffset(EntryOffset);
+ for (BinaryFunction *Frag : JT->Parents) {
+ for (uint64_t EntryAddress : JT->EntriesAsAddress)
+ // if target is builtin_unreachable
+ if (EntryAddress == Frag->getAddress() + Frag->getSize()) {
+ Frag->IgnoredBranches.emplace_back(EntryAddress - Frag->getAddress(),
+ Frag->getSize());
+ } else if (EntryAddress >= Frag->getAddress() &&
+ EntryAddress < Frag->getAddress() + Frag->getSize()) {
+ Frag->registerReferencedOffset(EntryAddress - Frag->getAddress());
+ }
}
// In strict mode, erase PC-relative relocation record. Later we check that
}
// Mark to skip the function and all its fragments.
- if (BF.hasSplitJumpTable())
- FragmentsToSkip.push_back(&BF);
+ for (BinaryFunction *Frag : JT->Parents)
+ if (Frag->hasIndirectTargetToSplitFragment())
+ addFragmentsToSkip(Frag);
}
if (opts::StrictMode && DataPCRelocations.size()) {
}
void BinaryContext::skipMarkedFragments() {
- // Unique functions in the vector.
- std::unordered_set<BinaryFunction *> UniqueFunctions(FragmentsToSkip.begin(),
- FragmentsToSkip.end());
- // Copy the functions back to FragmentsToSkip.
- FragmentsToSkip.assign(UniqueFunctions.begin(), UniqueFunctions.end());
+ std::vector<BinaryFunction *> FragmentQueue;
+ // Copy the functions to FragmentQueue.
+ FragmentQueue.assign(FragmentsToSkip.begin(), FragmentsToSkip.end());
auto addToWorklist = [&](BinaryFunction *Function) -> void {
- if (UniqueFunctions.count(Function))
+ if (FragmentsToSkip.count(Function))
return;
- FragmentsToSkip.push_back(Function);
- UniqueFunctions.insert(Function);
+ FragmentQueue.push_back(Function);
+ addFragmentsToSkip(Function);
};
// Functions containing split jump tables need to be skipped with all
// fragments (transitively).
- for (size_t I = 0; I != FragmentsToSkip.size(); I++) {
- BinaryFunction *BF = FragmentsToSkip[I];
- assert(UniqueFunctions.count(BF) &&
+ for (size_t I = 0; I != FragmentQueue.size(); I++) {
+ BinaryFunction *BF = FragmentQueue[I];
+ assert(FragmentsToSkip.count(BF) &&
"internal error in traversing function fragments");
if (opts::Verbosity >= 1)
errs() << "BOLT-WARNING: Ignoring " << BF->getPrintName() << '\n';
BF->setSimple(false);
- BF->setHasSplitJumpTable(true);
+ BF->setHasIndirectTargetToSplitFragment(true);
llvm::for_each(BF->Fragments, addToWorklist);
llvm::for_each(BF->ParentFragments, addToWorklist);
errs() << "BOLT-WARNING: skipped " << FragmentsToSkip.size() << " function"
<< (FragmentsToSkip.size() == 1 ? "" : "s")
<< " due to cold fragments\n";
- FragmentsToSkip.clear();
}
MCSymbol *BinaryContext::getOrCreateGlobalSymbol(uint64_t Address, Twine Prefix,
return (Fragment->isFragment() && Fragment->isParentFragment(Parent));
};
+ // Two fragments of same function access same jump table
if (JumpTable *JT = getJumpTableContainingAddress(Address)) {
assert(JT->Type == Type && "jump table types have to match");
- bool HasMultipleParents = isFragmentOf(JT->Parent, &Function) ||
- isFragmentOf(&Function, JT->Parent);
- assert((JT->Parent == &Function || HasMultipleParents) &&
- "cannot re-use jump table of a different function");
assert(Address == JT->getAddress() && "unexpected non-empty jump table");
- // Flush OffsetEntries with INVALID_OFFSET if multiple parents
- // Duplicate the entry for the parent function for easy access
- if (HasMultipleParents) {
+ // Prevent associating a jump table to a specific fragment twice.
+ // This simple check arises from the assumption: no more than 2 fragments.
+ if (JT->Parents.size() == 1 && JT->Parents[0] != &Function) {
+ bool SameFunction = isFragmentOf(JT->Parents[0], &Function) ||
+ isFragmentOf(&Function, JT->Parents[0]);
+ assert(SameFunction &&
+ "cannot re-use jump table of a different function");
+ // Duplicate the entry for the parent function for easy access
+ JT->Parents.push_back(&Function);
if (opts::Verbosity > 2) {
- outs() << "BOLT-WARNING: Multiple fragments access same jump table: "
- << JT->Parent->getPrintName() << "; " << Function.getPrintName()
- << "\n";
+ outs() << "BOLT-INFO: Multiple fragments access same jump table: "
+ << JT->Parents[0]->getPrintName() << "; "
+ << Function.getPrintName() << "\n";
+ JT->print(outs());
}
- constexpr uint64_t INVALID_OFFSET = std::numeric_limits<uint64_t>::max();
- for (unsigned I = 0; I < JT->OffsetEntries.size(); ++I)
- JT->OffsetEntries[I] = INVALID_OFFSET;
Function.JumpTables.emplace(Address, JT);
+ JT->Parents[0]->setHasIndirectTargetToSplitFragment(true);
+ JT->Parents[1]->setHasIndirectTargetToSplitFragment(true);
}
+
+ bool IsJumpTableParent = false;
+ for (BinaryFunction *Frag : JT->Parents)
+ if (Frag == &Function)
+ IsJumpTableParent = true;
+ assert(IsJumpTableParent &&
+ "cannot re-use jump table of a different function");
return JT->getFirstLabel();
}
<< " in function " << Function << '\n');
JumpTable *JT = new JumpTable(*JTLabel, Address, EntrySize, Type,
- JumpTable::LabelMapType{{0, JTLabel}}, Function,
+ JumpTable::LabelMapType{{0, JTLabel}},
*getSectionForAddress(Address));
+ JT->Parents.push_back(&Function);
+ if (opts::Verbosity > 2)
+ JT->print(outs());
JumpTables.emplace(Address, JT);
// Duplicate the entry for the parent function for easy access.
Function.JumpTables.emplace(Address, JT);
-
return JTLabel;
}
MCSymbol *NewLabel = Ctx->createNamedTempSymbol("duplicatedJT");
JumpTable *NewJT =
new JumpTable(*NewLabel, JT->getAddress(), JT->EntrySize, JT->Type,
- JumpTable::LabelMapType{{Offset, NewLabel}}, Function,
+ JumpTable::LabelMapType{{Offset, NewLabel}},
*getSectionForAddress(JT->getAddress()));
+ NewJT->Parents = JT->Parents;
NewJT->Entries = JT->Entries;
NewJT->Counts = JT->Counts;
uint64_t JumpTableID = ++DuplicatedJumpTables;
<< *this << '\n';
}
if (JT.Entries.empty()) {
- for (unsigned I = 0; I < JT.OffsetEntries.size(); ++I) {
- MCSymbol *Label =
- getOrCreateLocalLabel(getAddress() + JT.OffsetEntries[I],
- /*CreatePastEnd*/ true);
- JT.Entries.push_back(Label);
+ bool HasOneParent = (JT.Parents.size() == 1);
+ for (unsigned I = 0; I < JT.EntriesAsAddress.size(); ++I) {
+ uint64_t EntryAddress = JT.EntriesAsAddress[I];
+ // builtin_unreachable does not belong to any function
+ // Need to handle separately
+ bool IsBuiltIn = false;
+ for (BinaryFunction *Parent : JT.Parents) {
+ if (EntryAddress == Parent->getAddress() + Parent->getSize()) {
+ IsBuiltIn = true;
+ // Specify second parameter as true to accept builtin_unreachable
+ MCSymbol *Label = getOrCreateLocalLabel(EntryAddress, true);
+ JT.Entries.push_back(Label);
+ break;
+ }
+ }
+ if (IsBuiltIn)
+ continue;
+ // Create local label for targets cannot be reached by other fragments
+ // Otherwise, secondary entry point to target function
+ BinaryFunction *TargetBF =
+ BC.getBinaryFunctionContainingAddress(EntryAddress);
+ if (TargetBF->getAddress() != EntryAddress) {
+ MCSymbol *Label =
+ (HasOneParent && TargetBF == this)
+ ? getOrCreateLocalLabel(JT.EntriesAsAddress[I], true)
+ : TargetBF->addEntryPointAtOffset(EntryAddress -
+ TargetBF->getAddress());
+ JT.Entries.push_back(Label);
+ }
}
}
uint64_t EntryOffset = JTAddress - JT->getAddress();
while (EntryOffset < JT->getSize()) {
- uint64_t TargetOffset = JT->OffsetEntries[EntryOffset / JT->EntrySize];
+ uint64_t EntryAddress = JT->EntriesAsAddress[EntryOffset / JT->EntrySize];
+ uint64_t TargetOffset = EntryAddress - getAddress();
if (TargetOffset < getSize()) {
TakenBranches.emplace_back(JTSiteOffset, TargetOffset);
bolt::JumpTable::JumpTable(MCSymbol &Symbol, uint64_t Address, size_t EntrySize,
JumpTableType Type, LabelMapType &&Labels,
- BinaryFunction &BF, BinarySection &Section)
+ BinarySection &Section)
: BinaryData(Symbol, Address, 0, EntrySize, Section), EntrySize(EntrySize),
- OutputEntrySize(EntrySize), Type(Type), Labels(Labels), Parent(&BF) {}
+ OutputEntrySize(EntrySize), Type(Type), Labels(Labels) {}
std::pair<size_t, size_t>
bolt::JumpTable::getEntriesForAddress(const uint64_t Addr) const {
uint64_t Offset = 0;
if (Type == JTT_PIC)
OS << "PIC ";
- OS << "Jump table " << getName() << " for function " << *Parent << " at 0x"
- << Twine::utohexstr(getAddress()) << " with a total count of " << Count
- << ":\n";
- for (const uint64_t EntryOffset : OffsetEntries)
- OS << " 0x" << Twine::utohexstr(EntryOffset) << '\n';
+ ListSeparator LS;
+
+ OS << "Jump table " << getName() << " for function ";
+ for (BinaryFunction *Frag : Parents)
+ OS << LS << *Frag;
+ OS << " at 0x" << Twine::utohexstr(getAddress()) << " with a total count of "
+ << Count << ":\n";
+ for (const uint64_t EntryAddress : EntriesAsAddress)
+ OS << " absolute offset: 0x" << Twine::utohexstr(EntryAddress) << '\n';
for (const MCSymbol *Entry : Entries) {
auto LI = Labels.find(Offset);
if (Offset && LI != Labels.end()) {
const MCInst &Instr, const std::string &BranchLabel) {
StringRef FunctionName = BF.getOneName();
const JumpTable *JT = BF.getJumpTable(Instr);
- for (const uint64_t EntryOffset : JT->OffsetEntries) {
- auto LI = JT->Labels.find(EntryOffset);
- StringRef TargetName = LI->second->getName();
- const uint64_t Mispreds = JT->Counts[EntryOffset].Mispreds;
- const uint64_t Count = JT->Counts[EntryOffset].Count;
+ for (uint32_t i = 0; i < JT->Entries.size(); ++i) {
+ StringRef TargetName = JT->Entries[i]->getName();
+ const uint64_t Mispreds = JT->Counts[i].Mispreds;
+ const uint64_t Count = JT->Counts[i].Count;
OS << "# FDATA: 1 " << FunctionName << " #" << BranchLabel << "# "
<< "1 " << FunctionName << " #" << TargetName << "# " << Mispreds << " "
<< Count << '\n';
}
BC->processInterproceduralReferences();
- BC->clearJumpTableOffsets();
BC->populateJumpTables();
- BC->skipMarkedFragments();
for (auto &BFI : BC->getBinaryFunctions()) {
BinaryFunction &Function = BFI.second;
Function.postProcessJumpTables();
}
+ BC->clearJumpTableTempData();
BC->adjustCodePadding();
for (auto &BFI : BC->getBinaryFunctions()) {
if (!Function.isSimple()) {
assert((!BC->HasRelocations || Function.getSize() == 0 ||
- Function.hasSplitJumpTable()) &&
+ Function.hasIndirectTargetToSplitFragment()) &&
"unexpected non-simple function in relocation mode");
continue;
}
}
void RewriteInstance::postProcessFunctions() {
+ // We mark fragments as non-simple here, not during disassembly,
+ // So we can build their CFGs.
+ BC->skipMarkedFragments();
+ BC->clearFragmentsToSkip();
+
BC->TotalScore = 0;
BC->SumExecutionCount = 0;
for (auto &BFI : BC->getBinaryFunctions()) {
--- /dev/null
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o
+# RUN: %clang %cflags -no-pie %t.o -o %t.exe -Wl,-q
+# RUN: llvm-bolt %t.exe -o %t.out --lite=0 -v=1 --jump-tables=move 2>&1 | FileCheck %s
+
+# CHECK-NOT: unclaimed PC-relative relocations left in data
+# CHECK: BOLT-INFO: marking main.cold.1 as a fragment of main
+# CHECK: BOLT-WARNING: Ignoring main.cold.1
+# CHECK: BOLT-WARNING: Ignoring main
+
+ .text
+ .globl main
+ .type main, %function
+ .p2align 2
+main:
+ cmpl $0x67, %edi
+ jne main.cold.1
+LBB0:
+ retq
+.size main, .-main
+
+ .globl main.cold.1
+ .type main.cold.1, %function
+ .p2align 2
+main.cold.1:
+ jmpq *JUMP_TABLE(,%rcx,8)
+.size main.cold.1, .-main.cold.1
+
+ .rodata
+ .globl JUMP_TABLE
+JUMP_TABLE:
+ .quad LBB0
+ .quad LBB0
# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o
# RUN: llvm-strip --strip-unneeded %t.o
# RUN: %clang %cflags %t.o -o %t.exe -Wl,-q
-# RUN: llvm-bolt -v=3 %t.exe -o %t.out 2>&1 | FileCheck %s
+# RUN: llvm-bolt -print-cfg -v=3 %t.exe -o %t.out 2>&1 | FileCheck %s
-# CHECK: BOLT-WARNING: Multiple fragments access same jump table: main; main.cold.1
+# CHECK: BOLT-INFO: Multiple fragments access same jump table: main; main.cold.1
+# CHECK: PIC Jump table JUMP_TABLE1 for function main, main.cold.1 at {{.*}} with a total count of 0:
.text
.globl main
projects / platform / upstream / llvm.git / commitdiff