#include "llvm/Analysis/InlineCost.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/IR/PassManager.h"
+#include "llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h"
#include <memory>
#include <unordered_set>
/// behavior by implementing the corresponding record*Impl.
///
/// Call after inlining succeeded, and did not result in deleting the callee.
- void recordInlining() {
- markRecorded();
- recordInliningImpl();
- }
+ void recordInlining();
/// Call after inlining succeeded, and resulted in deleting the callee.
void recordInliningWithCalleeDeleted();
assert(!Recorded && "Recording should happen exactly once");
Recorded = true;
}
+ void recordInlineStatsIfNeeded();
bool Recorded = false;
};
class InlineAdvisor {
public:
InlineAdvisor(InlineAdvisor &&) = delete;
- virtual ~InlineAdvisor() { freeDeletedFunctions(); }
+ virtual ~InlineAdvisor();
/// Get an InlineAdvice containing a recommendation on whether to
/// inline or not. \p CB is assumed to be a direct call. \p FAM is assumed to
virtual void onPassExit() {}
protected:
- InlineAdvisor(FunctionAnalysisManager &FAM) : FAM(FAM) {}
+ InlineAdvisor(Module &M, FunctionAnalysisManager &FAM);
virtual std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) = 0;
virtual std::unique_ptr<InlineAdvice> getMandatoryAdvice(CallBase &CB,
bool Advice);
+ Module &M;
FunctionAnalysisManager &FAM;
+ std::unique_ptr<ImportedFunctionsInliningStatistics> ImportedFunctionsStats;
/// We may want to defer deleting functions to after the inlining for a whole
/// module has finished. This allows us to reliably use function pointers as
/// reusable as-is for inliner pass test scenarios, as well as for regular use.
class DefaultInlineAdvisor : public InlineAdvisor {
public:
- DefaultInlineAdvisor(FunctionAnalysisManager &FAM, InlineParams Params)
- : InlineAdvisor(FAM), Params(Params) {}
+ DefaultInlineAdvisor(Module &M, FunctionAnalysisManager &FAM,
+ InlineParams Params)
+ : InlineAdvisor(M, FAM), Params(Params) {}
private:
std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) override;
virtual std::unique_ptr<MLInlineAdvice>
getAdviceFromModel(CallBase &CB, OptimizationRemarkEmitter &ORE);
- Module &M;
std::unique_ptr<MLModelRunner> ModelRunner;
private:
/// previous build to guide current inlining. This is useful for inliner tuning.
class ReplayInlineAdvisor : public InlineAdvisor {
public:
- ReplayInlineAdvisor(FunctionAnalysisManager &FAM, LLVMContext &Context,
- StringRef RemarksFile, bool EmitRemarks);
+ ReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM,
+ LLVMContext &Context, StringRef RemarksFile,
+ bool EmitRemarks);
std::unique_ptr<InlineAdvice> getAdviceImpl(CallBase &CB) override;
bool areReplayRemarksLoaded() const { return HasReplayRemarks; }
class InlinerPass : public PassInfoMixin<InlinerPass> {
public:
InlinerPass(bool OnlyMandatory = false) : OnlyMandatory(OnlyMandatory) {}
- ~InlinerPass();
InlinerPass(InlinerPass &&Arg) = default;
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
private:
InlineAdvisor &getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM,
FunctionAnalysisManager &FAM, Module &M);
- std::unique_ptr<ImportedFunctionsInliningStatistics> ImportedFunctionsStats;
std::unique_ptr<DefaultInlineAdvisor> OwnedDefaultAdvisor;
const bool OnlyMandatory;
};
StringRef ModuleName;
};
+enum class InlinerFunctionImportStatsOpts {
+ No = 0,
+ Basic = 1,
+ Verbose = 2,
+};
+
} // llvm
#endif // LLVM_TRANSFORMS_UTILS_IMPORTEDFUNCTIONSINLININGSTATISTICS_H
cl::desc("Scale to limit the cost of inline deferral"),
cl::init(2), cl::Hidden);
+extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats;
+
void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(
const InlineResult &Result) {
using namespace ore;
DeletedFunctions.clear();
}
+void InlineAdvice::recordInlineStatsIfNeeded() {
+ if (Advisor->ImportedFunctionsStats)
+ Advisor->ImportedFunctionsStats->recordInline(*Caller, *Callee);
+}
+
+void InlineAdvice::recordInlining() {
+ markRecorded();
+ recordInlineStatsIfNeeded();
+ recordInliningImpl();
+}
+
void InlineAdvice::recordInliningWithCalleeDeleted() {
markRecorded();
+ recordInlineStatsIfNeeded();
Advisor->markFunctionAsDeleted(Callee);
recordInliningWithCalleeDeletedImpl();
}
auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
switch (Mode) {
case InliningAdvisorMode::Default:
- Advisor.reset(new DefaultInlineAdvisor(FAM, Params));
+ Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params));
break;
case InliningAdvisorMode::Development:
#ifdef LLVM_HAVE_TF_API
});
}
+InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM)
+ : M(M), FAM(FAM) {
+ if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) {
+ ImportedFunctionsStats =
+ std::make_unique<ImportedFunctionsInliningStatistics>();
+ ImportedFunctionsStats->setModuleInfo(M);
+ }
+}
+
+InlineAdvisor::~InlineAdvisor() {
+ if (ImportedFunctionsStats) {
+ assert(InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No);
+ ImportedFunctionsStats->dump(InlinerFunctionImportStats ==
+ InlinerFunctionImportStatsOpts::Verbose);
+ }
+
+ freeDeletedFunctions();
+}
+
std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB,
bool Advice) {
return std::make_unique<InlineAdvice>(this, CB, getCallerORE(CB), Advice);
MLInlineAdvisor::MLInlineAdvisor(Module &M, ModuleAnalysisManager &MAM,
std::unique_ptr<MLModelRunner> Runner)
: InlineAdvisor(
- MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager()),
- M(M), ModelRunner(std::move(Runner)), CG(new CallGraph(M)),
+ M, MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager()),
+ ModelRunner(std::move(Runner)), CG(new CallGraph(M)),
InitialIRSize(getModuleIRSize()), CurrentIRSize(InitialIRSize) {
assert(ModelRunner);
#define DEBUG_TYPE "inline-replay"
-ReplayInlineAdvisor::ReplayInlineAdvisor(FunctionAnalysisManager &FAM,
+ReplayInlineAdvisor::ReplayInlineAdvisor(Module &M,
+ FunctionAnalysisManager &FAM,
LLVMContext &Context,
StringRef RemarksFile,
bool EmitRemarks)
- : InlineAdvisor(FAM), HasReplayRemarks(false), EmitRemarks(EmitRemarks) {
+ : InlineAdvisor(M, FAM), HasReplayRemarks(false), EmitRemarks(EmitRemarks) {
auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(RemarksFile);
std::error_code EC = BufferOrErr.getError();
if (EC) {
DisableInlinedAllocaMerging("disable-inlined-alloca-merging",
cl::init(false), cl::Hidden);
-namespace {
-
-enum class InlinerFunctionImportStatsOpts {
- No = 0,
- Basic = 1,
- Verbose = 2,
-};
-
-} // end anonymous namespace
-
-static cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats(
- "inliner-function-import-stats",
- cl::init(InlinerFunctionImportStatsOpts::No),
- cl::values(clEnumValN(InlinerFunctionImportStatsOpts::Basic, "basic",
- "basic statistics"),
- clEnumValN(InlinerFunctionImportStatsOpts::Verbose, "verbose",
- "printing of statistics for each inlined function")),
- cl::Hidden, cl::desc("Enable inliner stats for imported functions"));
+extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats;
LegacyInlinerBase::LegacyInlinerBase(char &ID) : CallGraphSCCPass(ID) {}
return true;
}
-InlinerPass::~InlinerPass() {
- if (ImportedFunctionsStats) {
- assert(InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No);
- ImportedFunctionsStats->dump(InlinerFunctionImportStats ==
- InlinerFunctionImportStatsOpts::Verbose);
- }
-}
-
InlineAdvisor &
InlinerPass::getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM,
FunctionAnalysisManager &FAM, Module &M) {
+ if (OwnedDefaultAdvisor)
+ return *OwnedDefaultAdvisor;
+
auto *IAA = MAM.getCachedResult<InlineAdvisorAnalysis>(M);
if (!IAA) {
// It should still be possible to run the inliner as a stand-alone SCC pass,
// The one we would get from the MAM can be invalidated as a result of the
// inliner's activity.
OwnedDefaultAdvisor =
- std::make_unique<DefaultInlineAdvisor>(FAM, getInlineParams());
+ std::make_unique<DefaultInlineAdvisor>(M, FAM, getInlineParams());
return *OwnedDefaultAdvisor;
}
assert(IAA->getAdvisor() &&
auto AdvisorOnExit = make_scope_exit([&] { Advisor.onPassExit(); });
- if (!ImportedFunctionsStats &&
- InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) {
- ImportedFunctionsStats =
- std::make_unique<ImportedFunctionsInliningStatistics>();
- ImportedFunctionsStats->setModuleInfo(M);
- }
-
// We use a single common worklist for calls across the entire SCC. We
// process these in-order and append new calls introduced during inlining to
// the end.
}
}
- if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No)
- ImportedFunctionsStats->recordInline(F, Callee);
-
// Merge the attributes based on the inlining.
AttributeFuncs::mergeAttributesForInlining(F, Callee);
if (FAM && !ProfileInlineReplayFile.empty()) {
ExternalInlineAdvisor = std::make_unique<ReplayInlineAdvisor>(
- *FAM, Ctx, ProfileInlineReplayFile, /*EmitRemarks=*/false);
+ M, *FAM, Ctx, ProfileInlineReplayFile, /*EmitRemarks=*/false);
if (!ExternalInlineAdvisor->areReplayRemarksLoaded())
ExternalInlineAdvisor.reset();
}
#include "llvm/ADT/STLExtras.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Module.h"
+#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <sstream>
using namespace llvm;
+cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats(
+ "inliner-function-import-stats",
+ cl::init(InlinerFunctionImportStatsOpts::No),
+ cl::values(clEnumValN(InlinerFunctionImportStatsOpts::Basic, "basic",
+ "basic statistics"),
+ clEnumValN(InlinerFunctionImportStatsOpts::Verbose, "verbose",
+ "printing of statistics for each inlined function")),
+ cl::Hidden, cl::desc("Enable inliner stats for imported functions"));
+
ImportedFunctionsInliningStatistics::InlineGraphNode &
ImportedFunctionsInliningStatistics::createInlineGraphNode(const Function &F) {
; RUN: opt -S -passes=inliner-wrapper-no-mandatory-first -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-BASIC,CHECK
; RUN: opt -S -passes=inliner-wrapper-no-mandatory-first -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s --check-prefixes="CHECK-VERBOSE",CHECK
+; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-BASIC,CHECK
+; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-VERBOSE,CHECK
+
; CHECK: ------- Dumping inliner stats for [<stdin>] -------
; CHECK-BASIC-NOT: -- List of inlined functions:
; CHECK-BASIC-NOT: -- Inlined not imported function