// defined in the same section and hence cannot place the landing pad into a
// cold fragment when the corresponding call site is in the hot fragment.
// Because of this issue and the previously described issue of possible
- // zero-offset landing pad we disable splitting of exception-handling
- // code for shared objects.
+ // zero-offset landing pad we have to place landing pads in the same section
+ // as the corresponding invokes for shared objects.
std::function<void(const MCSymbol *)> emitLandingPad;
if (BC.HasFixedLoadAddress) {
Streamer.emitIntValue(dwarf::DW_EH_PE_udata4, 1); // LPStart format
Streamer.emitSymbolValue(LPSymbol, 4);
};
} else {
- assert(!EmitColdPart &&
- "cannot have exceptions in cold fragment for shared object");
Streamer.emitIntValue(dwarf::DW_EH_PE_omit, 1); // LPStart format
emitLandingPad = [&](const MCSymbol *LPSymbol) {
if (!LPSymbol)
bool AllCold = true;
for (BinaryBasicBlock *BB : BF.layout()) {
- uint64_t ExecCount = BB->getExecutionCount();
+ const uint64_t ExecCount = BB->getExecutionCount();
if (ExecCount == BinaryBasicBlock::COUNT_NO_PROFILE)
return;
if (ExecCount != 0)
<< " pre-split is <0x"
<< Twine::utohexstr(OriginalHotSize) << ", 0x"
<< Twine::utohexstr(ColdSize) << ">\n");
- }
-
- if (opts::SplitFunctions == SplitFunctions::ST_LARGE && !BC.HasRelocations) {
- // Split only if the function wouldn't fit.
- if (OriginalHotSize <= BF.getMaxSize())
- return;
+ if (opts::SplitFunctions == SplitFunctions::ST_LARGE &&
+ !BC.HasRelocations) {
+ // Split only if the function wouldn't fit.
+ if (OriginalHotSize <= BF.getMaxSize())
+ return;
+ }
}
// Never outline the first basic block.
BB->setCanOutline(false);
continue;
}
+
if (BF.hasEHRanges() && !opts::SplitEH) {
- // We cannot move landing pads (or rather entry points for landing
- // pads).
+ // We cannot move landing pads (or rather entry points for landing pads).
if (BB->isLandingPad()) {
BB->setCanOutline(false);
continue;
// that the block never throws, it is safe to move the block to
// decrease the size of the function.
for (MCInst &Instr : *BB) {
- if (BF.getBinaryContext().MIB->isInvoke(Instr)) {
+ if (BC.MIB->isInvoke(Instr)) {
BB->setCanOutline(false);
break;
}
BB->setIsCold(true);
}
+ // For shared objects, place invoke instructions and corresponding landing
+ // pads in the same fragment. To reduce hot code size, create trampoline
+ // landing pads that will redirect the execution to the real LP.
+ if (!BC.HasFixedLoadAddress && BF.hasEHRanges() && BF.isSplit())
+ createEHTrampolines(BF);
+
// Check the new size to see if it's worth splitting the function.
if (BC.isX86() && BF.isSplit()) {
std::tie(HotSize, ColdSize) = BC.calculateEmittedSize(BF);
}
}
+void SplitFunctions::createEHTrampolines(BinaryFunction &BF) const {
+ const auto &MIB = BF.getBinaryContext().MIB;
+
+ // Map real landing pads to the corresponding trampolines.
+ std::unordered_map<const MCSymbol *, const MCSymbol *> LPTrampolines;
+
+ // Iterate over the copy of basic blocks since we are adding new blocks to the
+ // function which will invalidate its iterators.
+ std::vector<BinaryBasicBlock *> Blocks(BF.pbegin(), BF.pend());
+ for (BinaryBasicBlock *BB : Blocks) {
+ for (MCInst &Instr : *BB) {
+ const Optional<MCPlus::MCLandingPad> EHInfo = MIB->getEHInfo(Instr);
+ if (!EHInfo || !EHInfo->first)
+ continue;
+
+ const MCSymbol *LPLabel = EHInfo->first;
+ BinaryBasicBlock *LPBlock = BF.getBasicBlockForLabel(LPLabel);
+ if (BB->isCold() == LPBlock->isCold())
+ continue;
+
+ const MCSymbol *TrampolineLabel = nullptr;
+ auto Iter = LPTrampolines.find(LPLabel);
+ if (Iter != LPTrampolines.end()) {
+ TrampolineLabel = Iter->second;
+ } else {
+ // Create a trampoline basic block in the same fragment as the thrower.
+ // Note: there's no need to insert the jump instruction, it will be
+ // added by fixBranches().
+ BinaryBasicBlock *TrampolineBB = BF.addBasicBlock();
+ TrampolineBB->setIsCold(BB->isCold());
+ TrampolineBB->setExecutionCount(LPBlock->getExecutionCount());
+ TrampolineBB->addSuccessor(LPBlock, TrampolineBB->getExecutionCount());
+ TrampolineBB->setCFIState(LPBlock->getCFIState());
+ TrampolineLabel = TrampolineBB->getLabel();
+ LPTrampolines.emplace(std::make_pair(LPLabel, TrampolineLabel));
+ }
+
+ // Substitute the landing pad with the trampoline.
+ MIB->updateEHInfo(Instr,
+ MCPlus::MCLandingPad(TrampolineLabel, EHInfo->second));
+ }
+ }
+
+ if (LPTrampolines.empty())
+ return;
+
+ // All trampoline blocks were added to the end of the function. Place them at
+ // the end of corresponding fragments.
+ std::stable_sort(BF.layout_begin(), BF.layout_end(),
+ [&](BinaryBasicBlock *A, BinaryBasicBlock *B) {
+ return A->isCold() < B->isCold();
+ });
+
+ // Conservatively introduce branch instructions.
+ BF.fixBranches();
+
+ // Update exception-handling CFG for the function.
+ BF.recomputeLandingPads();
+}
+
} // namespace bolt
} // namespace llvm
--- /dev/null
+## Check that BOLT successfully splits C++ exception-handling code for
+## PIEs or shared objects.
+
+REQUIRES: system-linux
+
+RUN: %clangxx %cxxflags -pie -fPIC %p/Inputs/exceptions_split.cpp -Wl,-q -o %t
+RUN: llvm-bolt %t -o %t.instr --instrument --instrumentation-file=%t.fdata
+
+## Record profile with invocation that does not throw exceptions.
+RUN: %t.instr
+
+RUN: llvm-bolt %t -o %t.bolt --data %t.fdata --reorder-blocks=ext-tsp \
+RUN: --split-functions=1 --split-eh --print-after-lowering \
+RUN: --print-only=main 2>&1 | FileCheck %s
+
+## All calls to printf() should be from exception handling code that was
+## recorded as cold during the profile collection run. Check that the calls
+## are placed after the split point.
+CHECK-NOT: callq printf
+CHECK: HOT-COLD SPLIT POINT
+CHECK: callq printf
+
+## Verify the output still executes correctly when the exception path is being
+## taken.
+RUN: %t.bolt arg1 arg2 arg3 2>&1 | FileCheck --check-prefix=CHECK-BOLTED %s
+
+CHECK-BOLTED: catch 2
+CHECK-BOLTED-NEXT: catch 1
+
projects / platform / upstream / llvm.git / commitdiff