class BasicBlock;
class CallBase;
class Function;
+class Module;
class OptimizationRemarkEmitter;
+class PreservedAnalyses;
+
+/// There are 3 scenarios we can use the InlineAdvisor:
+/// - Default - use manual heuristics.
+///
+/// - Release mode, the expected mode for production, day to day deployments.
+/// In this mode, when building the compiler, we also compile a pre-trained ML
+/// model to native code, and link it as a static library. This mode has low
+/// overhead and no additional dependencies for the compiler runtime.
+///
+/// - Development mode, for training new models.
+/// In this mode, we trade off runtime performance for flexibility. This mode
+/// requires the full C Tensorflow API library, and evaluates models
+/// dynamically. This mode also permits generating training logs, for offline
+/// training.
+enum class InliningAdvisorMode : int { Default, Release, Development };
+
+class InlineAdvisor;
+/// Capture state between an inlining decision having had been made, and
+/// its impact being observable. When collecting model training data, this
+/// allows recording features/decisions/partial reward data sets.
+///
+/// Derivations of this type are expected to be tightly coupled with their
+/// InliningAdvisors. The base type implements the minimal contractual
+/// obligations.
+class InlineAdvice {
+public:
+ InlineAdvice(InlineAdvice &&) = delete;
+ InlineAdvice(const InlineAdvice &) = delete;
+ virtual ~InlineAdvice() {
+ assert(Recorded && "InlineAdvice should have been informed of the "
+ "inliner's decision in all cases");
+ }
+
+ /// Exactly one of the record* APIs must be called. Implementers may extend
+ /// behavior by implementing the corresponding record*Impl.
+ ///
+ /// Call after inlining succeeded, and did not result in deleting the callee.
+ void recordInlining() {
+ markRecorded();
+ recordInliningImpl();
+ }
+
+ /// Call after inlining succeeded, and resulted in deleting the callee.
+ void recordInliningWithCalleeDeleted();
+
+ /// Call after the decision for a call site was to not inline.
+ void recordUnsuccessfulInlining(const InlineResult &Result) {
+ markRecorded();
+ recordUnsuccessfulInliningImpl(Result);
+ }
+
+ /// Call to indicate inlining was not attempted.
+ void recordUnattemptedInlining() {
+ markRecorded();
+ recordUnattemptedInliningImpl();
+ }
+
+ /// Get the inlining recommendation.
+ bool isInliningRecommended() const { return IsInliningRecommended; }
+
+protected:
+ InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
+ bool IsInliningRecommended);
+
+ virtual void recordInliningImpl() {}
+ virtual void recordInliningWithCalleeDeletedImpl() {}
+ virtual void recordUnsuccessfulInliningImpl(const InlineResult &Result) {}
+ virtual void recordUnattemptedInliningImpl() {}
+
+ InlineAdvisor *const Advisor;
+ /// Caller and Callee are pre-inlining.
+ Function *const Caller;
+ Function *const Callee;
+ const bool IsInliningRecommended;
+
+private:
+ void markRecorded() {
+ assert(!Recorded && "Recording should happen exactly once");
+ Recorded = true;
+ }
+
+ bool Recorded = false;
+};
+
+/// Interface for deciding whether to inline a call site or not.
+class InlineAdvisor {
+public:
+ InlineAdvisor(InlineAdvisor &&) = delete;
+ virtual ~InlineAdvisor() { freeDeletedFunctions(); }
+
+ /// Get an InlineAdvice containing a recommendation on whether to
+ /// inline or not. \p CB is assumed to be a direct call. \p FAM is assumed to
+ /// be up-to-date wrt previous inlining decisions.
+ /// Returns an InlineAdvice with the inlining recommendation.
+ virtual std::unique_ptr<InlineAdvice>
+ getAdvice(CallBase &CB, FunctionAnalysisManager &FAM) = 0;
+
+ /// This must be called when the Inliner pass is entered, to allow the
+ /// InlineAdvisor update internal state, as result of function passes run
+ /// between Inliner pass runs (for the same module).
+ virtual void OnPassEntry() {}
+
+ /// This must be called when the Inliner pass is exited, as function passes
+ /// may be run subsequently. This allows an implementation of InlineAdvisor
+ /// to prepare for a partial update.
+ virtual void OnPassExit() {}
+
+protected:
+ InlineAdvisor() = default;
+
+ /// We may want to defer deleting functions to after the inlining for a whole
+ /// module has finished. This allows us to reliably use function pointers as
+ /// unique identifiers, as an efficient implementation detail of the
+ /// InlineAdvisor. Otherwise, it is possible the memory allocator
+ /// re-allocate Function objects at the same address of a deleted Function;
+ /// and Functions are potentially created during the function passes called
+ /// after each SCC inlining (e.g. argument promotion does that).
+ void freeDeletedFunctions();
+
+ bool isFunctionDeleted(Function *F) const {
+ return DeletedFunctions.count(F);
+ }
+
+private:
+ friend class InlineAdvice;
+ void markFunctionAsDeleted(Function *F);
+ std::unordered_set<Function *> DeletedFunctions;
+};
+
+/// The default (manual heuristics) implementation of the InlineAdvisor. This
+/// implementation does not need to keep state between inliner pass runs, and is
+/// reusable as-is for inliner pass test scenarios, as well as for regular use.
+class DefaultInlineAdvisor : public InlineAdvisor {
+public:
+ DefaultInlineAdvisor(InlineParams Params) : Params(Params) {}
+
+private:
+ std::unique_ptr<InlineAdvice>
+ getAdvice(CallBase &CB, FunctionAnalysisManager &FAM) override;
+
+ void OnPassExit() override { freeDeletedFunctions(); }
+ InlineParams Params;
+};
+
+/// The InlineAdvisorAnalysis is a module pass because the InlineAdvisor
+/// needs to capture state right before inlining commences over a module.
+class InlineAdvisorAnalysis : public AnalysisInfoMixin<InlineAdvisorAnalysis> {
+public:
+ static AnalysisKey Key;
+ InlineAdvisorAnalysis() = default;
+ struct Result {
+ Result(Module &M, ModuleAnalysisManager &MAM) : M(M), MAM(MAM) {}
+ bool invalidate(Module &, const PreservedAnalyses &,
+ ModuleAnalysisManager::Invalidator &) {
+ // InlineAdvisor must be preserved across analysis invalidations.
+ return false;
+ }
+ bool tryCreate(InlineParams Params, InliningAdvisorMode Mode);
+ InlineAdvisor *getAdvisor() const { return Advisor.get(); }
+ void clear() { Advisor.reset(); }
+
+ private:
+ Module &M;
+ ModuleAnalysisManager &MAM;
+ std::unique_ptr<InlineAdvisor> Advisor;
+ };
+
+ Result run(Module &M, ModuleAnalysisManager &MAM) { return Result(M, MAM); }
+};
// Default (manual policy) decision making helper APIs. Shared with the legacy
// pass manager inliner.
#include "llvm/Analysis/CGSCCPassManager.h"
#include "llvm/IR/PassManager.h"
#include "llvm/Support/Error.h"
+#include "llvm/Transforms/IPO/Inliner.h"
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Scalar/LoopPassManager.h"
#include <vector>
/// Construct the module pipeline that performs inlining as well as
/// the inlining-driven cleanups.
- ModulePassManager buildInlinerPipeline(OptimizationLevel Level,
- ThinLTOPhase Phase,
- bool DebugLogging = false);
+ ModuleInlinerWrapperPass buildInlinerPipeline(OptimizationLevel Level,
+ ThinLTOPhase Phase,
+ bool DebugLogging = false);
/// Construct the core LLVM module optimization pipeline.
///
#include "llvm/Analysis/CGSCCPassManager.h"
#include "llvm/Analysis/CallGraphSCCPass.h"
+#include "llvm/Analysis/InlineAdvisor.h"
#include "llvm/Analysis/InlineCost.h"
#include "llvm/Analysis/LazyCallGraph.h"
#include "llvm/IR/PassManager.h"
/// passes be composed to achieve the same end result.
class InlinerPass : public PassInfoMixin<InlinerPass> {
public:
- InlinerPass(InlineParams Params = getInlineParams())
- : Params(std::move(Params)) {}
+ InlinerPass() = default;
~InlinerPass();
InlinerPass(InlinerPass &&Arg)
- : Params(std::move(Arg.Params)),
- ImportedFunctionsStats(std::move(Arg.ImportedFunctionsStats)) {}
+ : ImportedFunctionsStats(std::move(Arg.ImportedFunctionsStats)) {}
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
LazyCallGraph &CG, CGSCCUpdateResult &UR);
private:
- InlineParams Params;
+ InlineAdvisor &getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM,
+ Module &M);
std::unique_ptr<ImportedFunctionsInliningStatistics> ImportedFunctionsStats;
+ Optional<DefaultInlineAdvisor> OwnedDefaultAdvisor;
};
+/// Module pass, wrapping the inliner pass. This works in conjunction with the
+/// InlineAdvisorAnalysis to facilitate inlining decisions taking into account
+/// module-wide state, that need to keep track of inter-inliner pass runs, for
+/// a given module. An InlineAdvisor is configured and kept alive for the
+/// duration of the ModuleInlinerWrapperPass::run.
+class ModuleInlinerWrapperPass
+ : public PassInfoMixin<ModuleInlinerWrapperPass> {
+public:
+ ModuleInlinerWrapperPass(
+ InlineParams Params = getInlineParams(), bool Debugging = false,
+ InliningAdvisorMode Mode = InliningAdvisorMode::Default,
+ unsigned MaxDevirtIterations = 0);
+ ModuleInlinerWrapperPass(ModuleInlinerWrapperPass &&Arg) = default;
+
+ PreservedAnalyses run(Module &, ModuleAnalysisManager &);
+
+ /// Allow adding more CGSCC passes, besides inlining. This should be called
+ /// before run is called, as part of pass pipeline building.
+ CGSCCPassManager &getPM() { return PM; }
+
+ /// Allow adding module-level analyses benefiting the contained CGSCC passes.
+ template <class T> void addRequiredModuleAnalysis() {
+ MPM.addPass(RequireAnalysisPass<T, Module>());
+ }
+
+private:
+ const InlineParams Params;
+ const InliningAdvisorMode Mode;
+ const unsigned MaxDevirtIterations;
+ const bool Debugging;
+ CGSCCPassManager PM;
+ ModulePassManager MPM;
+};
} // end namespace llvm
#endif // LLVM_TRANSFORMS_IPO_INLINER_H
//
//===----------------------------------------------------------------------===//
//
-// This file implements inlining decision-making APIs.
+// This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and
+// related types.
//
//===----------------------------------------------------------------------===//
cl::desc("Scale to limit the cost of inline deferral"),
cl::init(-1), cl::Hidden);
+namespace {
+class DefaultInlineAdvice : public InlineAdvice {
+public:
+ DefaultInlineAdvice(DefaultInlineAdvisor *Advisor, CallBase &CB,
+ Optional<InlineCost> OIC, OptimizationRemarkEmitter &ORE)
+ : InlineAdvice(Advisor, CB, OIC.hasValue()), OriginalCB(&CB), OIC(OIC),
+ ORE(ORE), DLoc(CB.getDebugLoc()), Block(CB.getParent()) {}
+
+private:
+ void recordUnsuccessfulInliningImpl(const InlineResult &Result) override {
+ using namespace ore;
+ llvm::setInlineRemark(*OriginalCB, std::string(Result.getFailureReason()) +
+ "; " + inlineCostStr(*OIC));
+ ORE.emit([&]() {
+ return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block)
+ << NV("Callee", Callee) << " will not be inlined into "
+ << NV("Caller", Caller) << ": "
+ << NV("Reason", Result.getFailureReason());
+ });
+ }
+
+ void recordInliningWithCalleeDeletedImpl() override {
+ emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, *OIC);
+ }
+
+ void recordInliningImpl() override {
+ emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, *OIC);
+ }
+
+private:
+ CallBase *const OriginalCB;
+ Optional<InlineCost> OIC;
+ OptimizationRemarkEmitter &ORE;
+
+ // Capture the context of CB before inlining, as a successful inlining may
+ // change that context, and we want to report success or failure in the
+ // original context.
+ const DebugLoc DLoc;
+ const BasicBlock *const Block;
+};
+
+} // namespace
+
+std::unique_ptr<InlineAdvice>
+DefaultInlineAdvisor::getAdvice(CallBase &CB, FunctionAnalysisManager &FAM) {
+ Function &Callee = *CB.getCalledFunction();
+ Function &F = *CB.getCaller();
+ ProfileSummaryInfo *PSI = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(F)
+ .getCachedResult<ProfileSummaryAnalysis>(
+ *CB.getParent()->getParent()->getParent());
+
+ auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
+ // FIXME: make GetAssumptionCache's decl similar to the other 2 below. May
+ // need changing the type of getInlineCost parameters? Also see similar case
+ // in Inliner.cpp
+ std::function<AssumptionCache &(Function &)> GetAssumptionCache =
+ [&](Function &F) -> AssumptionCache & {
+ return FAM.getResult<AssumptionAnalysis>(Callee);
+ };
+ auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
+ return FAM.getResult<BlockFrequencyAnalysis>(F);
+ };
+ auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
+ return FAM.getResult<TargetLibraryAnalysis>(F);
+ };
+
+ auto GetInlineCost = [&](CallBase &CB) {
+ Function &Callee = *CB.getCalledFunction();
+ auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
+ bool RemarksEnabled =
+ Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
+ DEBUG_TYPE);
+ return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, {GetBFI},
+ GetTLI, PSI, RemarksEnabled ? &ORE : nullptr);
+ };
+ auto OIC = llvm::shouldInline(CB, GetInlineCost, ORE);
+ return std::make_unique<DefaultInlineAdvice>(this, CB, OIC, ORE);
+}
+
+InlineAdvice::InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
+ bool IsInliningRecommended)
+ : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),
+ IsInliningRecommended(IsInliningRecommended) {}
+
+void InlineAdvisor::markFunctionAsDeleted(Function *F) {
+ assert((!DeletedFunctions.count(F)) &&
+ "Cannot put cause a function to become dead twice!");
+ DeletedFunctions.insert(F);
+}
+
+void InlineAdvisor::freeDeletedFunctions() {
+ for (auto *F : DeletedFunctions)
+ delete F;
+ DeletedFunctions.clear();
+}
+
+void InlineAdvice::recordInliningWithCalleeDeleted() {
+ markRecorded();
+ Advisor->markFunctionAsDeleted(Callee);
+ recordInliningWithCalleeDeletedImpl();
+}
+
+AnalysisKey InlineAdvisorAnalysis::Key;
+
+bool InlineAdvisorAnalysis::Result::tryCreate(InlineParams Params,
+ InliningAdvisorMode Mode) {
+ switch (Mode) {
+ case InliningAdvisorMode::Default:
+ Advisor.reset(new DefaultInlineAdvisor(Params));
+ break;
+ case InliningAdvisorMode::Development:
+ // To be added subsequently under conditional compilation.
+ break;
+ case InliningAdvisorMode::Release:
+ // To be added subsequently under conditional compilation.
+ break;
+ }
+ return !!Advisor;
+}
+
/// Return true if inlining of CB can block the caller from being
/// inlined which is proved to be more beneficial. \p IC is the
/// estimated inline cost associated with callsite \p CB.
#include "llvm/Analysis/DominanceFrontier.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/IVUsers.h"
+#include "llvm/Analysis/InlineAdvisor.h"
#include "llvm/Analysis/LazyCallGraph.h"
#include "llvm/Analysis/LazyValueInfo.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
"enable-npm-gvn-hoist", cl::init(false), cl::Hidden,
cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
+static cl::opt<InliningAdvisorMode> UseInlineAdvisor(
+ "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden,
+ cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
+ cl::values(clEnumValN(InliningAdvisorMode::Default, "default",
+ "Heuristics-based inliner version."),
+ clEnumValN(InliningAdvisorMode::Development, "development",
+ "Use development mode (runtime-loadable model)."),
+ clEnumValN(InliningAdvisorMode::Release, "release",
+ "Use release mode (AOT-compiled model).")));
+
static cl::opt<bool> EnableGVNSink(
"enable-npm-gvn-sink", cl::init(false), cl::Hidden,
cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
// This should probably be lowered after performance testing.
// FIXME: this comment is cargo culted from the old pass manager, revisit).
IP.HintThreshold = 325;
-
- CGSCCPassManager CGPipeline(DebugLogging);
-
- CGPipeline.addPass(InlinerPass(IP));
+ ModuleInlinerWrapperPass MIWP(IP, DebugLogging);
+ CGSCCPassManager &CGPipeline = MIWP.getPM();
FunctionPassManager FPM;
FPM.addPass(SROA());
CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
- MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPipeline)));
+ MPM.addPass(std::move(MIWP));
// Delete anything that is now dead to make sure that we don't instrument
// dead code. Instrumentation can end up keeping dead code around and
return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel());
}
-ModulePassManager PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
- ThinLTOPhase Phase,
- bool DebugLogging) {
- ModulePassManager MPM(DebugLogging);
+ModuleInlinerWrapperPass
+PassBuilder::buildInlinerPipeline(OptimizationLevel Level, ThinLTOPhase Phase,
+ bool DebugLogging) {
+ InlineParams IP = getInlineParamsFromOptLevel(Level);
+ if (Phase == PassBuilder::ThinLTOPhase::PreLink && PGOOpt &&
+ PGOOpt->Action == PGOOptions::SampleUse)
+ IP.HotCallSiteThreshold = 0;
+
+ ModuleInlinerWrapperPass MIWP(IP, DebugLogging, UseInlineAdvisor,
+ MaxDevirtIterations);
// Require the GlobalsAA analysis for the module so we can query it within
// the CGSCC pipeline.
- MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
+ MIWP.addRequiredModuleAnalysis<GlobalsAA>();
// Require the ProfileSummaryAnalysis for the module so we can query it within
// the inliner pass.
- MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
+ MIWP.addRequiredModuleAnalysis<ProfileSummaryAnalysis>();
// Now begin the main postorder CGSCC pipeline.
// FIXME: The current CGSCC pipeline has its origins in the legacy pass
// manager and trying to emulate its precise behavior. Much of this doesn't
// make a lot of sense and we should revisit the core CGSCC structure.
- CGSCCPassManager MainCGPipeline(DebugLogging);
+ CGSCCPassManager &MainCGPipeline = MIWP.getPM();
// Note: historically, the PruneEH pass was run first to deduce nounwind and
// generally clean up exception handling overhead. It isn't clear this is
// valuable as the inliner doesn't currently care whether it is inlining an
// invoke or a call.
- // Run the inliner first. The theory is that we are walking bottom-up and so
- // the callees have already been fully optimized, and we want to inline them
- // into the callers so that our optimizations can reflect that.
- // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
- // because it makes profile annotation in the backend inaccurate.
- InlineParams IP = getInlineParamsFromOptLevel(Level);
- if (Phase == ThinLTOPhase::PreLink && PGOOpt &&
- PGOOpt->Action == PGOOptions::SampleUse)
- IP.HotCallSiteThreshold = 0;
- MainCGPipeline.addPass(InlinerPass(IP));
-
if (AttributorRun & AttributorRunOption::CGSCC)
MainCGPipeline.addPass(AttributorCGSCCPass());
for (auto &C : CGSCCOptimizerLateEPCallbacks)
C(MainCGPipeline, Level);
- // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
- // to detect when we devirtualize indirect calls and iterate the SCC passes
- // in that case to try and catch knock-on inlining or function attrs
- // opportunities. Then we add it to the module pipeline by walking the SCCs
- // in postorder (or bottom-up).
- MPM.addPass(
- createModuleToPostOrderCGSCCPassAdaptor(createDevirtSCCRepeatedPass(
- std::move(MainCGPipeline), MaxDevirtIterations)));
- return MPM;
+ return MIWP;
}
ModulePassManager PassBuilder::buildModuleSimplificationPipeline(
// valuable as the inliner doesn't currently care whether it is inlining an
// invoke or a call.
// Run the inliner now.
- MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
- InlinerPass(getInlineParamsFromOptLevel(Level))));
+ MPM.addPass(ModuleInlinerWrapperPass(getInlineParamsFromOptLevel(Level),
+ DebugLogging));
// Optimize globals again after we ran the inliner.
MPM.addPass(GlobalOptPass());
MODULE_ANALYSIS("verify", VerifierAnalysis())
MODULE_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC))
MODULE_ANALYSIS("asan-globals-md", ASanGlobalsMetadataAnalysis())
+MODULE_ANALYSIS("inline-advisor", InlineAdvisorAnalysis())
#ifndef MODULE_ALIAS_ANALYSIS
#define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
MODULE_PASS("hwasan", HWAddressSanitizerPass(false, false))
MODULE_PASS("khwasan", HWAddressSanitizerPass(true, true))
MODULE_PASS("inferattrs", InferFunctionAttrsPass())
+MODULE_PASS("inliner-wrapper", ModuleInlinerWrapperPass())
MODULE_PASS("insert-gcov-profiling", GCOVProfilerPass())
MODULE_PASS("instrorderfile", InstrOrderFilePass())
MODULE_PASS("instrprof", InstrProfiling())
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/CGSCCPassManager.h"
#include "llvm/Analysis/CallGraph.h"
+#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/InlineAdvisor.h"
#include "llvm/Analysis/InlineCost.h"
#include "llvm/Analysis/LazyCallGraph.h"
}
}
+InlineAdvisor &
+InlinerPass::getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM,
+ Module &M) {
+ auto *IAA = MAM.getCachedResult<InlineAdvisorAnalysis>(M);
+ if (!IAA) {
+ // It should still be possible to run the inliner as a stand-alone SCC pass,
+ // for test scenarios. In that case, we default to the
+ // DefaultInlineAdvisor, which doesn't need to keep state between SCC pass
+ // runs. It also uses just the default InlineParams.
+ OwnedDefaultAdvisor.emplace(getInlineParams());
+ return *OwnedDefaultAdvisor;
+ }
+ assert(IAA->getAdvisor() &&
+ "Expected a present InlineAdvisorAnalysis also have an "
+ "InlineAdvisor initialized");
+ return *IAA->getAdvisor();
+}
+
PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
CGSCCAnalysisManager &AM, LazyCallGraph &CG,
CGSCCUpdateResult &UR) {
Module &M = *InitialC.begin()->getFunction().getParent();
ProfileSummaryInfo *PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(M);
+ InlineAdvisor &Advisor = getAdvisor(MAMProxy, M);
+ Advisor.OnPassEntry();
+
+ auto AdvisorOnExit = make_scope_exit([&] { Advisor.OnPassExit(); });
+
if (!ImportedFunctionsStats &&
InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) {
ImportedFunctionsStats =
LLVM_DEBUG(dbgs() << "Inlining calls in: " << F.getName() << "\n");
- // Get the remarks emission analysis for the caller.
- auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
-
std::function<AssumptionCache &(Function &)> GetAssumptionCache =
[&](Function &F) -> AssumptionCache & {
return FAM.getResult<AssumptionAnalysis>(F);
};
- auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
- return FAM.getResult<BlockFrequencyAnalysis>(F);
- };
- auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
- return FAM.getResult<TargetLibraryAnalysis>(F);
- };
-
- auto GetInlineCost = [&](CallBase &CB) {
- Function &Callee = *CB.getCalledFunction();
- auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
- bool RemarksEnabled =
- Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
- DEBUG_TYPE);
- return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, {GetBFI},
- GetTLI, PSI, RemarksEnabled ? &ORE : nullptr);
- };
// Now process as many calls as we have within this caller in the sequnece.
// We bail out as soon as the caller has to change so we can update the
continue;
}
- auto OIC = shouldInline(*CB, GetInlineCost, ORE);
+ auto Advice = Advisor.getAdvice(*CB, FAM);
// Check whether we want to inline this callsite.
- if (!OIC)
+ if (!Advice->isInliningRecommended()) {
+ Advice->recordUnattemptedInlining();
continue;
- auto DoInline = [&]() -> InlineResult {
- // Setup the data structure used to plumb customization into the
- // `InlineFunction` routine.
- InlineFunctionInfo IFI(
- /*cg=*/nullptr, &GetAssumptionCache, PSI,
- &FAM.getResult<BlockFrequencyAnalysis>(*(CB->getCaller())),
- &FAM.getResult<BlockFrequencyAnalysis>(Callee));
-
- InlineResult IR = InlineFunction(*CB, IFI);
- if (!IR.isSuccess())
- return IR;
-
- DidInline = true;
- InlinedCallees.insert(&Callee);
- ++NumInlined;
+ }
- // Add any new callsites to defined functions to the worklist.
- if (!IFI.InlinedCallSites.empty()) {
- int NewHistoryID = InlineHistory.size();
- InlineHistory.push_back({&Callee, InlineHistoryID});
-
- for (CallBase *ICB : reverse(IFI.InlinedCallSites)) {
- Function *NewCallee = ICB->getCalledFunction();
- if (!NewCallee) {
- // Try to promote an indirect (virtual) call without waiting for
- // the post-inline cleanup and the next DevirtSCCRepeatedPass
- // iteration because the next iteration may not happen and we may
- // miss inlining it.
- if (tryPromoteCall(*ICB))
- NewCallee = ICB->getCalledFunction();
- }
- if (NewCallee)
- if (!NewCallee->isDeclaration())
- Calls.push_back({ICB, NewHistoryID});
- }
- }
+ // Setup the data structure used to plumb customization into the
+ // `InlineFunction` routine.
+ InlineFunctionInfo IFI(
+ /*cg=*/nullptr, &GetAssumptionCache, PSI,
+ &FAM.getResult<BlockFrequencyAnalysis>(*(CB->getCaller())),
+ &FAM.getResult<BlockFrequencyAnalysis>(Callee));
- if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No)
- ImportedFunctionsStats->recordInline(F, Callee);
-
- // Merge the attributes based on the inlining.
- AttributeFuncs::mergeAttributesForInlining(F, Callee);
-
- // For local functions, check whether this makes the callee trivially
- // dead. In that case, we can drop the body of the function eagerly
- // which may reduce the number of callers of other functions to one,
- // changing inline cost thresholds.
- if (Callee.hasLocalLinkage()) {
- // To check this we also need to nuke any dead constant uses (perhaps
- // made dead by this operation on other functions).
- Callee.removeDeadConstantUsers();
- if (Callee.use_empty() && !CG.isLibFunction(Callee)) {
- Calls.erase(
- std::remove_if(Calls.begin() + I + 1, Calls.end(),
- [&](const std::pair<CallBase *, int> &Call) {
- return Call.first->getCaller() == &Callee;
- }),
- Calls.end());
- // Clear the body and queue the function itself for deletion when we
- // finish inlining and call graph updates.
- // Note that after this point, it is an error to do anything other
- // than use the callee's address or delete it.
- Callee.dropAllReferences();
- assert(find(DeadFunctions, &Callee) == DeadFunctions.end() &&
- "Cannot put cause a function to become dead twice!");
- DeadFunctions.push_back(&Callee);
+ InlineResult IR = InlineFunction(*CB, IFI);
+ if (!IR.isSuccess()) {
+ Advice->recordUnsuccessfulInlining(IR);
+ continue;
+ }
+
+ DidInline = true;
+ InlinedCallees.insert(&Callee);
+ ++NumInlined;
+
+ // Add any new callsites to defined functions to the worklist.
+ if (!IFI.InlinedCallSites.empty()) {
+ int NewHistoryID = InlineHistory.size();
+ InlineHistory.push_back({&Callee, InlineHistoryID});
+
+ for (CallBase *ICB : reverse(IFI.InlinedCallSites)) {
+ Function *NewCallee = ICB->getCalledFunction();
+ if (!NewCallee) {
+ // Try to promote an indirect (virtual) call without waiting for
+ // the post-inline cleanup and the next DevirtSCCRepeatedPass
+ // iteration because the next iteration may not happen and we may
+ // miss inlining it.
+ if (tryPromoteCall(*ICB))
+ NewCallee = ICB->getCalledFunction();
}
+ if (NewCallee)
+ if (!NewCallee->isDeclaration())
+ Calls.push_back({ICB, NewHistoryID});
}
- return IR;
- };
- // Capture the context of CB before inlining, as a successful inlining may
- // change that context, and we want to report success or failure in the
- // original context.
- auto DLoc = CB->getDebugLoc();
- auto *Block = CB->getParent();
-
- auto Outcome = DoInline();
- if (!Outcome.isSuccess()) {
- using namespace ore;
- setInlineRemark(*CB, std::string(Outcome.getFailureReason()) + "; " +
- inlineCostStr(*OIC));
- ORE.emit([&]() {
- return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block)
- << NV("Callee", &Callee) << " will not be inlined into "
- << NV("Caller", &F) << ": "
- << NV("Reason", Outcome.getFailureReason());
- });
- continue;
}
- emitInlinedInto(ORE, DLoc, Block, Callee, F, *OIC);
+ if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No)
+ ImportedFunctionsStats->recordInline(F, Callee);
+
+ // Merge the attributes based on the inlining.
+ AttributeFuncs::mergeAttributesForInlining(F, Callee);
+
+ // For local functions, check whether this makes the callee trivially
+ // dead. In that case, we can drop the body of the function eagerly
+ // which may reduce the number of callers of other functions to one,
+ // changing inline cost thresholds.
+ bool CalleeWasDeleted = false;
+ if (Callee.hasLocalLinkage()) {
+ // To check this we also need to nuke any dead constant uses (perhaps
+ // made dead by this operation on other functions).
+ Callee.removeDeadConstantUsers();
+ if (Callee.use_empty() && !CG.isLibFunction(Callee)) {
+ Calls.erase(
+ std::remove_if(Calls.begin() + I + 1, Calls.end(),
+ [&](const std::pair<CallBase *, int> &Call) {
+ return Call.first->getCaller() == &Callee;
+ }),
+ Calls.end());
+ // Clear the body and queue the function itself for deletion when we
+ // finish inlining and call graph updates.
+ // Note that after this point, it is an error to do anything other
+ // than use the callee's address or delete it.
+ Callee.dropAllReferences();
+ assert(find(DeadFunctions, &Callee) == DeadFunctions.end() &&
+ "Cannot put cause a function to become dead twice!");
+ DeadFunctions.push_back(&Callee);
+ CalleeWasDeleted = true;
+ }
+ }
+ if (CalleeWasDeleted)
+ Advice->recordInliningWithCalleeDeleted();
+ else
+ Advice->recordInlining();
}
// Back the call index up by one to put us in a good position to go around
// sets.
for (Function *DeadF : DeadFunctions) {
// Get the necessary information out of the call graph and nuke the
- // function there. Also, cclear out any cached analyses.
+ // function there. Also, clear out any cached analyses.
auto &DeadC = *CG.lookupSCC(*CG.lookup(*DeadF));
FAM.clear(*DeadF, DeadF->getName());
AM.clear(DeadC, DeadC.getName());
UR.InvalidatedRefSCCs.insert(&DeadRC);
// And delete the actual function from the module.
- M.getFunctionList().erase(DeadF);
+ // The Advisor may use Function pointers to efficiently index various
+ // internal maps, e.g. for memoization. Function cleanup passes like
+ // argument promotion create new functions. It is possible for a new
+ // function to be allocated at the address of a deleted function. We could
+ // index using names, but that's inefficient. Alternatively, we let the
+ // Advisor free the functions when it sees fit.
+ DeadF->getBasicBlockList().clear();
+ M.getFunctionList().remove(DeadF);
+
++NumDeleted;
}
PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
return PA;
}
+
+ModuleInlinerWrapperPass::ModuleInlinerWrapperPass(InlineParams Params,
+ bool Debugging,
+ InliningAdvisorMode Mode,
+ unsigned MaxDevirtIterations)
+ : Params(Params), Mode(Mode), MaxDevirtIterations(MaxDevirtIterations),
+ Debugging(Debugging), PM(Debugging), MPM(Debugging) {
+ // Run the inliner first. The theory is that we are walking bottom-up and so
+ // the callees have already been fully optimized, and we want to inline them
+ // into the callers so that our optimizations can reflect that.
+ // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
+ // because it makes profile annotation in the backend inaccurate.
+ PM.addPass(InlinerPass());
+}
+
+PreservedAnalyses ModuleInlinerWrapperPass::run(Module &M,
+ ModuleAnalysisManager &MAM) {
+ auto &IAA = MAM.getResult<InlineAdvisorAnalysis>(M);
+ if (!IAA.tryCreate(Params, Mode)) {
+ M.getContext().emitError(
+ "Could not setup Inlining Advisor for the requested "
+ "mode and/or options");
+ return PreservedAnalyses::all();
+ }
+
+ // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
+ // to detect when we devirtualize indirect calls and iterate the SCC passes
+ // in that case to try and catch knock-on inlining or function attrs
+ // opportunities. Then we add it to the module pipeline by walking the SCCs
+ // in postorder (or bottom-up).
+ // If MaxDevirtIterations is 0, we just don't use the devirtualization
+ // wrapper.
+ if (MaxDevirtIterations == 0)
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(PM)));
+ else
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ createDevirtSCCRepeatedPass(std::move(PM), MaxDevirtIterations)));
+ auto Ret = MPM.run(M, MAM);
+
+ IAA.clear();
+ return Ret;
+}
; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass
; CHECK-O-NEXT: Running pass: SimplifyCFGPass
; CHECK-O-NEXT: Finished llvm::Function pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
; CHECK-O-NEXT: Starting llvm::Module pass manager run.
; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
; CHECK-O-NEXT: Running analysis: GlobalsAA
; CHECK-O2-NEXT: Running analysis: OuterAnalysisManagerProxy
; CHECK-EP-Peephole-NEXT: Running pass: NoOpFunctionPass
; CHECK-O2-NEXT: Finished llvm::Function pass manager run.
-; CHECK-O2-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}InlinerPass>
+; CHECK-O2-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O2-NEXT: Running analysis: InlineAdvisorAnalysis
+; CHECK-O2-NEXT: Starting llvm::Module pass manager run.
+; CHECK-O2-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}PassManager{{.*}}>
+; CHECK-O2-NEXT: Starting CGSCC pass manager run.
+; CHECK-O2-NEXT: Running pass: InlinerPass
+; CHECK-O2-NEXT: Finished CGSCC pass manager run.
+; CHECK-O2-NEXT: Finished llvm::Module pass manager run.
; CHECK-O2-NEXT: Running pass: GlobalOptPass
; CHECK-O2-NEXT: Running pass: GlobalDCEPass
; CHECK-O2-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy
; CHECK-O-NEXT: Running pass: SimplifyCFGPass
; CHECK-O-NEXT: Finished llvm::Function pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
; CHECK-O-NEXT: Starting llvm::Module pass manager run.
; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
; CHECK-O-NEXT: Running analysis: GlobalsAA
; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis on foo
; CHECK-O-NEXT: Running pass: SimplifyCFGPass
; CHECK-O-NEXT: Finished {{.*}}Function pass manager run.
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
; CHECK-O-NEXT: Running analysis: GlobalsAA
; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis on foo
; CHECK-O-NEXT: Running pass: SimplifyCFGPass on foo
; CHECK-O-NEXT: Finished {{.*}}Function pass manager run
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
; CHECK-O-NEXT: Running analysis: GlobalsAA
; Validate ThinLTO prelink pipeline when we have instrumentation PGO
;
; RUN: llvm-profdata merge %S/Inputs/new-pm-thinlto-prelink-pgo-defaults.proftext -o %t.profdata
-;
+;
; RUN: opt -disable-verify -debug-pass-manager \
; RUN: -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' \
; RUN: -passes='thinlto-pre-link<O1>,name-anon-globals' -S %s 2>&1 \
; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy
; CHECK-O-NEXT: Running pass: SimplifyCFGPass
; CHECK-O-NEXT: Finished {{.*}}Function pass manager run.
+; CHECK-O123-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O123-NEXT: Running analysis: InlineAdvisorAnalysis
+; CHECK-O123-NEXT: Starting {{.*}}Module pass manager run.
; CHECK-O123-NEXT: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}PassManager<{{.*}}LazyCallGraph::SCC
; CHECK-O123-NEXT: Running analysis: InnerAnalysisManagerProxy
; CHECK-O123-NEXT: Running analysis: LazyCallGraphAnalysis
; CHECK-O123-NEXT: Running pass: InlinerPass on (foo)
; CHECK-O123-NEXT: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
; CHECK-O123-NEXT: Finished CGSCC pass manager run.
+; CHECK-O123-NEXT: Finished {{.*}}Module pass manager run.
; CHECK-O123-NEXT: Running pass: GlobalDCEPass
; CHECK-O-NEXT: Running pass: PGOInstrumentationUse
; CHECK-O-NEXT: Running analysis: ProfileSummaryAnalysis
; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
; CHECK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis on foo
; CHECK-O-NEXT: Running analysis: PassInstrumentationAnalysis on foo
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-Os-NEXT: Running analysis: InlineAdvisorAnalysis
+; CHECK-Oz-NEXT: Running analysis: InlineAdvisorAnalysis
; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
; CHECK-O-NEXT: Running analysis: GlobalsAA
; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis on foo
; CHECK-O-NEXT: Running pass: SimplifyCFGPass on foo
; CHECK-O-NEXT: Finished {{.*}}Function pass manager run
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
; CHECK-O-NEXT: Running analysis: GlobalsAA
; RUN: opt < %s 2>&1 -disable-output \
; RUN: -passes=inline -print-before-all -print-after-all -print-module-scope | FileCheck %s -check-prefix=INL-MOD
+; RUN: opt < %s 2>&1 -disable-output \
+; RUN: -passes=inliner-wrapper -print-before-all -print-after-all | FileCheck %s -check-prefix=INL
+; RUN: opt < %s 2>&1 -disable-output \
+; RUN: -passes=inliner-wrapper -print-before-all -print-after-all -print-module-scope | FileCheck %s -check-prefix=INL-MOD
+
; INL: IR Dump Before {{InlinerPass .*scc: .tester, foo}}
; INL-NOT: IR Dump After {{InlinerPass}}
; INL: IR Dump Before {{InlinerPass .*scc: .tester}}
; RUN: opt < %s 2>&1 -disable-output \
; RUN: -passes=inline -print-after-all | FileCheck %s -check-prefix=INL
; RUN: opt < %s 2>&1 -disable-output \
+; RUN: -passes=inliner-wrapper -print-after-all | FileCheck %s -check-prefix=INL
+; RUN: opt < %s 2>&1 -disable-output \
; RUN: -inline -print-after-all -print-module-scope | FileCheck %s -check-prefix=INL-MOD
; RUN: opt < %s 2>&1 -disable-output \
; RUN: -passes=inline -print-after-all -print-module-scope | FileCheck %s -check-prefix=INL-MOD
+; RUN: opt < %s 2>&1 -disable-output \
+; RUN: -passes=inliner-wrapper -print-after-all -print-module-scope | FileCheck %s -check-prefix=INL-MOD
; INL: IR Dump After {{Function Integration/Inlining|InlinerPass .*scc: .bar, foo}}
; INL: define void @bar()
; RUN: opt -S -passes=inline -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=CHECK-BASIC -check-prefix=CHECK
; RUN: opt -S -passes=inline -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix="CHECK-VERBOSE" -check-prefix=CHECK
+; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=CHECK-BASIC -check-prefix=CHECK
+; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix="CHECK-VERBOSE" -check-prefix=CHECK
+
; CHECK: ------- Dumping inliner stats for [<stdin>] -------
; CHECK-BASIC-NOT: -- List of inlined functions:
; CHECK-BASIC-NOT: -- Inlined not imported function
--- /dev/null
+; Check that, in the absence of dependencies, we emit an error message when
+; trying to use ML-driven inlining.
+;
+; RUN: not opt -passes=scc-oz-module-inliner -enable-ml-inliner=development -S < %s 2>&1 | FileCheck %s
+; RUN: not opt -passes=scc-oz-module-inliner -enable-ml-inliner=release -S < %s 2>&1 | FileCheck %s
+
+declare i64 @f1()
+
+; CHECK: Could not setup Inlining Advisor for the requested mode and/or options
\ No newline at end of file
;
; RUN: opt < %s -S -inline | FileCheck %s
; RUN: opt < %s -S -passes=inline | FileCheck %s
+; RUN: opt < %s -S -passes=inliner-wrapper | FileCheck %s
; CHECK-LABEL: define internal void @test1_scc0()
; CHECK-NOT: call
; a 'ret 10'
;
; RUN: opt -passes=inline -S < %s | FileCheck %s --check-prefix=INLINE --check-prefix=CHECK
+; RUN: opt -passes=inliner-wrapper -S < %s | FileCheck %s --check-prefix=INLINE --check-prefix=CHECK
; RUN: opt -passes=scc-oz-module-inliner -S < %s | FileCheck %s --check-prefix=MODULE --check-prefix=CHECK
define void @modify_value({i32, float}* %v) {
;
; RUN: opt -S < %s -inline -inline-threshold=150 | FileCheck %s --check-prefixes=CHECK,OLD
; RUN: opt -S < %s -passes=inline -inline-threshold=150 | FileCheck %s --check-prefixes=CHECK,NEW
+; RUN: opt -S < %s -passes=inliner-wrapper -inline-threshold=150 | FileCheck %s --check-prefixes=CHECK,NEW
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
; RUN: -pass-remarks-with-hotness -pass-remarks-hotness-threshold=1 2>&1 | \
; RUN: FileCheck -allow-empty -check-prefix=THRESHOLD %s
+; RUN: opt < %s -S -passes=inliner-wrapper -pass-remarks-output=%t -pass-remarks=inline \
+; RUN: -pass-remarks-with-hotness -pass-remarks-hotness-threshold=1 2>&1 | \
+; RUN: FileCheck -allow-empty -check-prefix=THRESHOLD %s
+
; Check that when any threshold is specified we ignore remarks with no
; hotness -- these are blocks that have not been executed during training.
; RUN: -pass-remarks-with-hotness 2>&1 | FileCheck %s
; RUN: cat %t | FileCheck -check-prefix=YAML %s
+; RUN: opt < %s -S -passes=inliner-wrapper -pass-remarks-output=%t -pass-remarks=inline \
+; RUN: -pass-remarks-missed=inline -pass-remarks-analysis=inline \
+; RUN: -pass-remarks-with-hotness 2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+
; Check the YAML file for inliner-generated passed and analysis remarks. This
; is the input:
; RUN: opt < %s -passes=inline -pass-remarks=inline -pass-remarks-missed=inline \
; RUN: -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 \
; RUN: | FileCheck %s
+; RUN: opt < %s -passes=inliner-wrapper -pass-remarks=inline -pass-remarks-missed=inline \
+; RUN: -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 \
+; RUN: | FileCheck %s
; CHECK: foo inlined into bar with (cost=always): always inline attribute (hotness: 30)
; CHECK: foz not inlined into bar because it should never be inlined (cost=never): noinline function attribute (hotness: 30)
; RUN: opt < %s -S -passes=inline \
; RUN: -pass-remarks-with-hotness -pass-remarks-hotness-threshold 100 \
; RUN: -pass-remarks-output=%t.threshold
+
+; Inliner - Module Wrapper
+; RUN: opt < %s -S -passes=inliner-wrapper -pass-remarks-missed=inline \
+; RUN: -pass-remarks-with-hotness -pass-remarks-hotness-threshold 15 \
+; RUN: -pass-remarks-output=%t 2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+; RUN: opt < %s -S -passes=inliner-wrapper -pass-remarks-with-hotness -pass-remarks-output=%t
+; RUN: cat %t | FileCheck -check-prefix=YAML %s
+;
+; Verify that remarks that don't meet the hotness threshold are not output.
+; RUN: opt < %s -S -passes=inliner-wrapper -pass-remarks-missed=inline \
+; RUN: -pass-remarks-with-hotness -pass-remarks-hotness-threshold 100 \
+; RUN: -pass-remarks-output=%t.threshold 2>&1 | \
+; RUN: FileCheck -check-prefix=THRESHOLD %s
+; RUN: test ! -s %t.threshold
+; RUN: opt < %s -S -passes=inliner-wrapper \
+; RUN: -pass-remarks-with-hotness -pass-remarks-hotness-threshold 100 \
+; RUN: -pass-remarks-output=%t.threshold
+
; The remarks output file should be empty.
; RUN: test ! -s %t.threshold
; RUN: -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 | \
; RUN: FileCheck -check-prefix=CHECK -check-prefix=HOTNESS %s
+; RUN: opt < %s -passes=inliner-wrapper -pass-remarks=inline -pass-remarks-missed=inline \
+; RUN: -pass-remarks-analysis=inline -S 2>&1 | \
+; RUN: FileCheck -check-prefix=CHECK -check-prefix=NO_HOTNESS %s
+; RUN: opt < %s -passes=inliner-wrapper -pass-remarks=inline -pass-remarks-missed=inline \
+; RUN: -pass-remarks-analysis=inline -pass-remarks-with-hotness -S 2>&1 | \
+; RUN: FileCheck -check-prefix=CHECK -check-prefix=HOTNESS %s
+
; HOTNESS: fox will not be inlined into bar because its definition is unavailable
; NO_HOTNESS-NOT: fox will not be inlined into bar because its definition is unavailable
; CHECK: foo inlined into bar with (cost=always): always inline attribute