#define LLVM_ANALYSIS_INLINECOST_H
#include "llvm/Analysis/CallGraphSCCPass.h"
+#include "llvm/Analysis/AssumptionCache.h"
#include <cassert>
#include <climits>
///
/// Also note that calling this function *dynamically* computes the cost of
/// inlining the callsite. It is an expensive, heavyweight call.
-InlineCost getInlineCost(CallSite CS, int DefaultThreshold,
- TargetTransformInfo &CalleeTTI,
- AssumptionCacheTracker *ACT, ProfileSummaryInfo *PSI);
+InlineCost
+getInlineCost(CallSite CS, int DefaultThreshold, TargetTransformInfo &CalleeTTI,
+ std::function<AssumptionCache &(Function &)> &GetAssumptionCache,
+ ProfileSummaryInfo *PSI);
/// \brief Get an InlineCost with the callee explicitly specified.
/// This allows you to calculate the cost of inlining a function via a
/// pointer. This behaves exactly as the version with no explicit callee
/// parameter in all other respects.
//
-InlineCost getInlineCost(CallSite CS, Function *Callee, int DefaultThreshold,
- TargetTransformInfo &CalleeTTI,
- AssumptionCacheTracker *ACT, ProfileSummaryInfo *PSI);
+InlineCost
+getInlineCost(CallSite CS, Function *Callee, int DefaultThreshold,
+ TargetTransformInfo &CalleeTTI,
+ std::function<AssumptionCache &(Function &)> &GetAssumptionCache,
+ ProfileSummaryInfo *PSI);
int computeThresholdFromOptLevels(unsigned OptLevel, unsigned SizeOptLevel);
#define LLVM_TRANSFORMS_IPO_INLINERPASS_H
#include "llvm/Analysis/CallGraphSCCPass.h"
+#include "llvm/Analysis/InlineCost.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
namespace llvm {
class AssumptionCacheTracker;
// InsertLifetime - Insert @llvm.lifetime intrinsics.
bool InsertLifetime;
- /// shouldInline - Return true if the inliner should attempt to
- /// inline at the given CallSite.
- bool shouldInline(CallSite CS);
- /// Return true if inlining of CS can block the caller from being
- /// inlined which is proved to be more beneficial. \p IC is the
- /// estimated inline cost associated with callsite \p CS.
- /// \p TotalAltCost will be set to the estimated cost of inlining the caller
- /// if \p CS is suppressed for inlining.
- bool shouldBeDeferred(Function *Caller, CallSite CS, InlineCost IC,
- int &TotalAltCost);
-
protected:
AssumptionCacheTracker *ACT;
ProfileSummaryInfo *PSI;
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/IR/ValueMap.h"
#include "llvm/Transforms/Utils/ValueMapper.h"
class InlineFunctionInfo {
public:
explicit InlineFunctionInfo(CallGraph *cg = nullptr,
- AssumptionCacheTracker *ACT = nullptr)
- : CG(cg), ACT(ACT) {}
+ std::function<AssumptionCache &(Function &)>
+ *GetAssumptionCache = nullptr)
+ : CG(cg), GetAssumptionCache(GetAssumptionCache) {}
/// CG - If non-null, InlineFunction will update the callgraph to reflect the
/// changes it makes.
CallGraph *CG;
- AssumptionCacheTracker *ACT;
+ std::function<AssumptionCache &(Function &)> *GetAssumptionCache;
/// StaticAllocas - InlineFunction fills this in with all static allocas that
/// get copied into the caller.
/// The TargetTransformInfo available for this compilation.
const TargetTransformInfo &TTI;
- /// The cache of @llvm.assume intrinsics.
- AssumptionCacheTracker *ACT;
+ /// Getter for the cache of @llvm.assume intrinsics.
+ std::function<AssumptionCache &(Function &)> &GetAssumptionCache;
/// Profile summary information.
ProfileSummaryInfo *PSI;
bool visitUnreachableInst(UnreachableInst &I);
public:
- CallAnalyzer(const TargetTransformInfo &TTI, AssumptionCacheTracker *ACT,
+ CallAnalyzer(const TargetTransformInfo &TTI,
+ std::function<AssumptionCache &(Function &)> &GetAssumptionCache,
ProfileSummaryInfo *PSI, Function &Callee, int Threshold,
CallSite CSArg)
- : TTI(TTI), ACT(ACT), PSI(PSI), F(Callee), CandidateCS(CSArg),
- Threshold(Threshold), Cost(0), IsCallerRecursive(false),
- IsRecursiveCall(false), ExposesReturnsTwice(false),
- HasDynamicAlloca(false), ContainsNoDuplicateCall(false),
- HasReturn(false), HasIndirectBr(false), HasFrameEscape(false),
- AllocatedSize(0), NumInstructions(0), NumVectorInstructions(0),
- FiftyPercentVectorBonus(0), TenPercentVectorBonus(0), VectorBonus(0),
- NumConstantArgs(0), NumConstantOffsetPtrArgs(0), NumAllocaArgs(0),
- NumConstantPtrCmps(0), NumConstantPtrDiffs(0),
- NumInstructionsSimplified(0), SROACostSavings(0),
- SROACostSavingsLost(0) {}
+ : TTI(TTI), GetAssumptionCache(GetAssumptionCache), PSI(PSI), F(Callee),
+ CandidateCS(CSArg), Threshold(Threshold), Cost(0),
+ IsCallerRecursive(false), IsRecursiveCall(false),
+ ExposesReturnsTwice(false), HasDynamicAlloca(false),
+ ContainsNoDuplicateCall(false), HasReturn(false), HasIndirectBr(false),
+ HasFrameEscape(false), AllocatedSize(0), NumInstructions(0),
+ NumVectorInstructions(0), FiftyPercentVectorBonus(0),
+ TenPercentVectorBonus(0), VectorBonus(0), NumConstantArgs(0),
+ NumConstantOffsetPtrArgs(0), NumAllocaArgs(0), NumConstantPtrCmps(0),
+ NumConstantPtrDiffs(0), NumInstructionsSimplified(0),
+ SROACostSavings(0), SROACostSavingsLost(0) {}
bool analyzeCall(CallSite CS);
// during devirtualization and so we want to give it a hefty bonus for
// inlining, but cap that bonus in the event that inlining wouldn't pan
// out. Pretend to inline the function, with a custom threshold.
- CallAnalyzer CA(TTI, ACT, PSI, *F, InlineConstants::IndirectCallThreshold,
- CS);
+ CallAnalyzer CA(TTI, GetAssumptionCache, PSI, *F,
+ InlineConstants::IndirectCallThreshold, CS);
if (CA.analyzeCall(CS)) {
// We were able to inline the indirect call! Subtract the cost from the
// threshold to get the bonus we want to apply, but don't go below zero.
// the ephemeral values multiple times (and they're completely determined by
// the callee, so this is purely duplicate work).
SmallPtrSet<const Value *, 32> EphValues;
- CodeMetrics::collectEphemeralValues(&F, &ACT->getAssumptionCache(F),
- EphValues);
+ CodeMetrics::collectEphemeralValues(&F, &GetAssumptionCache(F), EphValues);
// The worklist of live basic blocks in the callee *after* inlining. We avoid
// adding basic blocks of the callee which can be proven to be dead for this
AttributeFuncs::areInlineCompatible(*Caller, *Callee);
}
-InlineCost llvm::getInlineCost(CallSite CS, int DefaultThreshold,
- TargetTransformInfo &CalleeTTI,
- AssumptionCacheTracker *ACT,
- ProfileSummaryInfo *PSI) {
+InlineCost llvm::getInlineCost(
+ CallSite CS, int DefaultThreshold, TargetTransformInfo &CalleeTTI,
+ std::function<AssumptionCache &(Function &)> &GetAssumptionCache,
+ ProfileSummaryInfo *PSI) {
return getInlineCost(CS, CS.getCalledFunction(), DefaultThreshold, CalleeTTI,
- ACT, PSI);
+ GetAssumptionCache, PSI);
}
int llvm::computeThresholdFromOptLevels(unsigned OptLevel,
int llvm::getDefaultInlineThreshold() { return DefaultInlineThreshold; }
-InlineCost llvm::getInlineCost(CallSite CS, Function *Callee,
- int DefaultThreshold,
- TargetTransformInfo &CalleeTTI,
- AssumptionCacheTracker *ACT,
- ProfileSummaryInfo *PSI) {
+InlineCost llvm::getInlineCost(
+ CallSite CS, Function *Callee, int DefaultThreshold,
+ TargetTransformInfo &CalleeTTI,
+ std::function<AssumptionCache &(Function &)> &GetAssumptionCache,
+ ProfileSummaryInfo *PSI) {
// Cannot inline indirect calls.
if (!Callee)
DEBUG(llvm::dbgs() << " Analyzing call of " << Callee->getName()
<< "...\n");
- CallAnalyzer CA(CalleeTTI, ACT, PSI, *Callee, DefaultThreshold, CS);
+ CallAnalyzer CA(CalleeTTI, GetAssumptionCache, PSI, *Callee, DefaultThreshold, CS);
bool ShouldInline = CA.analyzeCall(CS);
DEBUG(CA.dump());
InlineCost getInlineCost(CallSite CS) override {
Function *Callee = CS.getCalledFunction();
TargetTransformInfo &TTI = TTIWP->getTTI(*Callee);
- return llvm::getInlineCost(CS, DefaultThreshold, TTI, ACT, PSI);
+ std::function<AssumptionCache &(Function &)> GetAssumptionCache = [&](
+ Function &F) -> AssumptionCache & {
+ return ACT->getAssumptionCache(F);
+ };
+ return llvm::getInlineCost(CS, DefaultThreshold, TTI, GetAssumptionCache,
+ PSI);
}
bool runOnSCC(CallGraphSCC &SCC) override;
/// available from other functions inlined into the caller. If we are able to
/// inline this call site we attempt to reuse already available allocas or add
/// any new allocas to the set if not possible.
-static bool InlineCallIfPossible(Pass &P, CallSite CS, InlineFunctionInfo &IFI,
- InlinedArrayAllocasTy &InlinedArrayAllocas,
- int InlineHistory, bool InsertLifetime) {
+static bool
+InlineCallIfPossible(CallSite CS, InlineFunctionInfo &IFI,
+ InlinedArrayAllocasTy &InlinedArrayAllocas,
+ int InlineHistory, bool InsertLifetime,
+ std::function<AAResults &(Function &)> &AARGetter) {
Function *Callee = CS.getCalledFunction();
Function *Caller = CS.getCaller();
- // We need to manually construct BasicAA directly in order to disable
- // its use of other function analyses.
- BasicAAResult BAR(createLegacyPMBasicAAResult(P, *Callee));
-
- // Construct our own AA results for this function. We do this manually to
- // work around the limitations of the legacy pass manager.
- AAResults AAR(createLegacyPMAAResults(P, *Callee, BAR));
+ AAResults &AAR = AARGetter(*Callee);
// Try to inline the function. Get the list of static allocas that were
// inlined.
emitOptimizationRemarkAnalysis(Ctx, DEBUG_TYPE, *Caller, DLoc, Msg);
}
-bool Inliner::shouldBeDeferred(Function *Caller, CallSite CS, InlineCost IC,
- int &TotalSecondaryCost) {
+/// Return true if inlining of CS can block the caller from being
+/// inlined which is proved to be more beneficial. \p IC is the
+/// estimated inline cost associated with callsite \p CS.
+/// \p TotalAltCost will be set to the estimated cost of inlining the caller
+/// if \p CS is suppressed for inlining.
+static bool
+shouldBeDeferred(Function *Caller, CallSite CS, InlineCost IC,
+ int &TotalSecondaryCost,
+ std::function<InlineCost(CallSite CS)> &GetInlineCost) {
// For now we only handle local or inline functions.
if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
continue;
}
- InlineCost IC2 = getInlineCost(CS2);
+ InlineCost IC2 = GetInlineCost(CS2);
++NumCallerCallersAnalyzed;
if (!IC2) {
callerWillBeRemoved = false;
}
/// Return true if the inliner should attempt to inline at the given CallSite.
-bool Inliner::shouldInline(CallSite CS) {
- InlineCost IC = getInlineCost(CS);
+static bool shouldInline(CallSite CS,
+ std::function<InlineCost(CallSite CS)> GetInlineCost) {
+ InlineCost IC = GetInlineCost(CS);
if (IC.isAlways()) {
DEBUG(dbgs() << " Inlining: cost=always"
}
int TotalSecondaryCost = 0;
- if (shouldBeDeferred(Caller, CS, IC, TotalSecondaryCost)) {
+ if (shouldBeDeferred(Caller, CS, IC, TotalSecondaryCost, GetInlineCost)) {
DEBUG(dbgs() << " NOT Inlining: " << *CS.getInstruction()
<< " Cost = " << IC.getCost()
<< ", outer Cost = " << TotalSecondaryCost << '\n');
bool Inliner::runOnSCC(CallGraphSCC &SCC) {
if (skipSCC(SCC))
return false;
+
return inlineCalls(SCC);
}
-bool Inliner::inlineCalls(CallGraphSCC &SCC) {
- CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
- ACT = &getAnalysis<AssumptionCacheTracker>();
- PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(CG.getModule());
- auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
-
+static bool
+inlineCallsImpl(CallGraphSCC &SCC, CallGraph &CG,
+ std::function<AssumptionCache &(Function &)> GetAssumptionCache,
+ ProfileSummaryInfo *PSI, TargetLibraryInfo &TLI,
+ bool InsertLifetime,
+ std::function<InlineCost(CallSite CS)> GetInlineCost,
+ std::function<AAResults &(Function &)> AARGetter) {
SmallPtrSet<Function*, 8> SCCFunctions;
DEBUG(dbgs() << "Inliner visiting SCC:");
for (CallGraphNode *Node : SCC) {
InlinedArrayAllocasTy InlinedArrayAllocas;
- InlineFunctionInfo InlineInfo(&CG, ACT);
+ InlineFunctionInfo InlineInfo(&CG, &GetAssumptionCache);
// Now that we have all of the call sites, loop over them and inline them if
// it looks profitable to do so.
// If the policy determines that we should inline this function,
// try to do so.
- if (!shouldInline(CS)) {
+ if (!shouldInline(CS, GetInlineCost)) {
emitOptimizationRemarkMissed(CallerCtx, DEBUG_TYPE, *Caller, DLoc,
Twine(Callee->getName() +
" will not be inlined into " +
}
// Attempt to inline the function.
- if (!InlineCallIfPossible(*this, CS, InlineInfo, InlinedArrayAllocas,
- InlineHistoryID, InsertLifetime)) {
+ if (!InlineCallIfPossible(CS, InlineInfo, InlinedArrayAllocas,
+ InlineHistoryID, InsertLifetime, AARGetter)) {
emitOptimizationRemarkMissed(CallerCtx, DEBUG_TYPE, *Caller, DLoc,
Twine(Callee->getName() +
" will not be inlined into " +
return Changed;
}
+bool Inliner::inlineCalls(CallGraphSCC &SCC) {
+ CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
+ ACT = &getAnalysis<AssumptionCacheTracker>();
+ PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(CG.getModule());
+ auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
+ // We compute dedicated AA results for each function in the SCC as needed. We
+ // use a lambda referencing external objects so that they live long enough to
+ // be queried, but we re-use them each time.
+ Optional<BasicAAResult> BAR;
+ Optional<AAResults> AAR;
+ auto AARGetter = [&](Function &F) -> AAResults & {
+ BAR.emplace(createLegacyPMBasicAAResult(*this, F));
+ AAR.emplace(createLegacyPMAAResults(*this, F, *BAR));
+ return *AAR;
+ };
+ auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
+ return ACT->getAssumptionCache(F);
+ };
+ return inlineCallsImpl(SCC, CG, GetAssumptionCache, PSI, TLI, InsertLifetime,
+ [this](CallSite CS) { return getInlineCost(CS); },
+ AARGetter);
+}
+
/// Remove now-dead linkonce functions at the end of
/// processing to avoid breaking the SCC traversal.
bool Inliner::doFinalization(CallGraph &CG) {
bool SampleProfileLoader::inlineHotFunctions(Function &F) {
bool Changed = false;
LLVMContext &Ctx = F.getContext();
+ std::function<AssumptionCache &(Function &)> GetAssumptionCache = [&](
+ Function &F) -> AssumptionCache & { return ACT->getAssumptionCache(F); };
while (true) {
bool LocalChanged = false;
SmallVector<CallInst *, 10> CIS;
}
}
for (auto CI : CIS) {
- InlineFunctionInfo IFI(nullptr, ACT);
+ InlineFunctionInfo IFI(nullptr, ACT ? &GetAssumptionCache : nullptr);
Function *CalledFunction = CI->getCalledFunction();
DebugLoc DLoc = CI->getDebugLoc();
uint64_t NumSamples = findCalleeFunctionSamples(*CI)->getTotalSamples();
// caller, then don't bother inserting the assumption.
Value *Arg = CS.getArgument(I->getArgNo());
if (getKnownAlignment(Arg, DL, CS.getInstruction(),
- &IFI.ACT->getAssumptionCache(*CS.getCaller()),
+ &(*IFI.GetAssumptionCache)(*CS.getCaller()),
&DT) >= Align)
continue;
// If the pointer is already known to be sufficiently aligned, or if we can
// round it up to a larger alignment, then we don't need a temporary.
if (getOrEnforceKnownAlignment(Arg, ByValAlignment, DL, TheCall,
- &IFI.ACT->getAssumptionCache(*Caller)) >=
+ &(*IFI.GetAssumptionCache)(*Caller)) >=
ByValAlignment)
return Arg;
// FIXME: We could register any cloned assumptions instead of clearing the
// whole function's cache.
- if (IFI.ACT)
- IFI.ACT->getAssumptionCache(*Caller).clear();
+ if (IFI.GetAssumptionCache)
+ (*IFI.GetAssumptionCache)(*Caller).clear();
}
// If there are any alloca instructions in the block that used to be the entry
if (PHI) {
auto &DL = Caller->getParent()->getDataLayout();
if (Value *V = SimplifyInstruction(PHI, DL, nullptr, nullptr,
- &IFI.ACT->getAssumptionCache(*Caller))) {
+ &(*IFI.GetAssumptionCache)(*Caller))) {
PHI->replaceAllUsesWith(V);
PHI->eraseFromParent();
}