From 5fe10263ab39be96e316f37272b85a72596a7928 Mon Sep 17 00:00:00 2001 From: Mircea Trofin Date: Mon, 16 Nov 2020 14:01:53 -0800 Subject: [PATCH] [llvm][inliner] Reuse the inliner pass to implement 'always inliner' Enable performing mandatory inlinings upfront, by reusing the same logic as the full inliner, instead of the AlwaysInliner. This has the following benefits: - reduce code duplication - one inliner codebase - open the opportunity to help the full inliner by performing additional function passes after the mandatory inlinings, but before th full inliner. Performing the mandatory inlinings first simplifies the problem the full inliner needs to solve: less call sites, more contextualization, and, depending on the additional function optimization passes run between the 2 inliners, higher accuracy of cost models / decision policies. Note that this patch does not yet enable much in terms of post-always inline function optimization. Differential Revision: https://reviews.llvm.org/D91567 --- clang/test/CodeGen/thinlto-distributed-newpm.ll | 13 +++++--- .../Frontend/optimization-remark-line-directive.c | 2 +- clang/test/Frontend/optimization-remark-new-pm.c | 4 +-- .../optimization-remark-with-hotness-new-pm.c | 16 ++++----- clang/test/Frontend/optimization-remark.c | 26 +++++++-------- llvm/include/llvm/Analysis/InlineAdvisor.h | 25 ++++++++++++-- llvm/include/llvm/Passes/PassBuilder.h | 3 +- llvm/lib/Analysis/InlineAdvisor.cpp | 38 ++++++++++++++++++++++ llvm/lib/Analysis/MLInlineAdvisor.cpp | 13 +++----- llvm/lib/Passes/PassBuilder.cpp | 24 +++++++++++--- llvm/lib/Passes/PassRegistry.def | 7 +++- llvm/lib/Transforms/IPO/Inliner.cpp | 8 ----- llvm/test/Other/new-pm-defaults.ll | 16 ++++++--- llvm/test/Other/new-pm-lto-defaults.ll | 7 +++- llvm/test/Other/new-pm-module-inliner-wrapper.ll | 5 +-- llvm/test/Other/new-pm-thinlto-defaults.ll | 16 ++++++--- .../Other/new-pm-thinlto-postlink-pgo-defaults.ll | 16 ++++++--- .../new-pm-thinlto-postlink-samplepgo-defaults.ll | 15 ++++++--- .../Other/new-pm-thinlto-prelink-pgo-defaults.ll | 18 ++++++---- .../new-pm-thinlto-prelink-samplepgo-defaults.ll | 14 +++++--- .../Transforms/Inline/ML/bounds-checks-rewards.ll | 6 ++-- llvm/test/Transforms/Inline/ML/bounds-checks.ll | 2 +- llvm/test/Transforms/Inline/inline_stats.ll | 7 ++-- llvm/test/Transforms/Inline/pr46945.ll | 20 ++++++++---- 24 files changed, 220 insertions(+), 101 deletions(-) diff --git a/clang/test/CodeGen/thinlto-distributed-newpm.ll b/clang/test/CodeGen/thinlto-distributed-newpm.ll index 3056bf4..75ea406 100644 --- a/clang/test/CodeGen/thinlto-distributed-newpm.ll +++ b/clang/test/CodeGen/thinlto-distributed-newpm.ll @@ -63,14 +63,19 @@ ; CHECK-O: Running analysis: OuterAnalysisManagerProxy ; CHECK-O: Running pass: SimplifyCFGPass on main ; CHECK-O: Finished {{.*}}Function pass manager run. -; CHECK-O: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA -; CHECK-O: Running analysis: GlobalsAA -; CHECK-O: Running analysis: CallGraphAnalysis -; CHECK-O: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis +; CHECK-O: Running pass: ModuleInlinerWrapperPass +; CHECK-O: Running analysis: InlineAdvisorAnalysis ; CHECK-O: Running analysis: InnerAnalysisManagerProxy ; CHECK-O: Running analysis: LazyCallGraphAnalysis ; CHECK-O: Running analysis: FunctionAnalysisManagerCGSCCProxy on (main) ; CHECK-O: Running analysis: OuterAnalysisManagerProxy +; CHECK-O: Running pass: InlinerPass on (main) +; CHECK-O: Finished {{.*}}Module pass manager run +; CHECK-O: Running pass: ModuleInlinerWrapperPass +; CHECK-O: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA +; CHECK-O: Running analysis: GlobalsAA +; CHECK-O: Running analysis: CallGraphAnalysis +; CHECK-O: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis ; CHECK-O: Starting CGSCC pass manager run. ; CHECK-O: Running pass: InlinerPass on (main) ; CHECK-O: Running pass: PostOrderFunctionAttrsPass on (main) diff --git a/clang/test/Frontend/optimization-remark-line-directive.c b/clang/test/Frontend/optimization-remark-line-directive.c index 095791e..5a2dc67 100644 --- a/clang/test/Frontend/optimization-remark-line-directive.c +++ b/clang/test/Frontend/optimization-remark-line-directive.c @@ -6,7 +6,7 @@ // The new PM inliner is not added to the default pipeline at O0, so we add // some optimizations to trigger it. -// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -debug-info-kind=line-tables-only -emit-llvm-only -verify +// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -debug-info-kind=line-tables-only -emit-llvm-only -verify -mllvm -mandatory-inlining-first=0 int foo(int x, int y) __attribute__((always_inline)); int foo(int x, int y) { return x + y; } diff --git a/clang/test/Frontend/optimization-remark-new-pm.c b/clang/test/Frontend/optimization-remark-new-pm.c index 885dc68..79e8301 100644 --- a/clang/test/Frontend/optimization-remark-new-pm.c +++ b/clang/test/Frontend/optimization-remark-new-pm.c @@ -1,8 +1,8 @@ // Verify that remarks for the inliner appear. The remarks under the new PM will // be slightly different than those emitted by the legacy PM. The new PM inliner // also doesnot appear to be added at O0, so we test at O1. -// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O1 -fexperimental-new-pass-manager -emit-llvm-only -verify -// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O1 -fexperimental-new-pass-manager -emit-llvm-only -debug-info-kind=line-tables-only -verify +// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O1 -fexperimental-new-pass-manager -emit-llvm-only -verify -mllvm -mandatory-inlining-first=0 +// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O1 -fexperimental-new-pass-manager -emit-llvm-only -debug-info-kind=line-tables-only -verify -mllvm -mandatory-inlining-first=0 int foo(int x, int y) __attribute__((always_inline)); int foo(int x, int y) { return x + y; } diff --git a/clang/test/Frontend/optimization-remark-with-hotness-new-pm.c b/clang/test/Frontend/optimization-remark-with-hotness-new-pm.c index 2a03bcb..76802bf 100644 --- a/clang/test/Frontend/optimization-remark-with-hotness-new-pm.c +++ b/clang/test/Frontend/optimization-remark-with-hotness-new-pm.c @@ -18,46 +18,46 @@ // RUN: -fprofile-instrument-use-path=%t.profdata -Rpass=inline \ // RUN: -fexperimental-new-pass-manager -O1 \ // RUN: -Rpass-analysis=inline -Rpass-missed=inline \ -// RUN: -fdiagnostics-show-hotness -verify +// RUN: -fdiagnostics-show-hotness -verify -mllvm -mandatory-inlining-first=0 // The clang version of the previous test. // RUN: %clang -target x86_64-apple-macosx10.9 %s -c -emit-llvm -o /dev/null \ // RUN: -fprofile-instr-use=%t.profdata -Rpass=inline \ // RUN: -fexperimental-new-pass-manager -O1 \ // RUN: -Rpass-analysis=inline -Rpass-missed=inline \ -// RUN: -fdiagnostics-show-hotness -Xclang -verify +// RUN: -fdiagnostics-show-hotness -Xclang -verify -mllvm -mandatory-inlining-first=0 // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \ // RUN: optimization-remark-with-hotness.c %s -emit-llvm-only \ // RUN: -fprofile-sample-use=%t-sample.profdata -Rpass=inline \ // RUN: -fexperimental-new-pass-manager -O1 \ // RUN: -Rpass-analysis=inline -Rpass-missed=inline \ // RUN: -fdiagnostics-show-hotness -fdiagnostics-hotness-threshold=10 \ -// RUN: -verify +// RUN: -verify -mllvm -mandatory-inlining-first=0 // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \ // RUN: optimization-remark-with-hotness.c %s -emit-llvm-only \ // RUN: -fprofile-instrument-use-path=%t.profdata -Rpass=inline \ // RUN: -fexperimental-new-pass-manager -O1 \ // RUN: -Rpass-analysis=inline -Rpass-missed=inline \ -// RUN: -fdiagnostics-show-hotness -fdiagnostics-hotness-threshold=10 -verify +// RUN: -fdiagnostics-show-hotness -fdiagnostics-hotness-threshold=10 -verify -mllvm -mandatory-inlining-first=0 // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \ // RUN: optimization-remark-with-hotness.c %s -emit-llvm-only \ // RUN: -fprofile-instrument-use-path=%t.profdata -Rpass=inline \ -// RUN: -fexperimental-new-pass-manager -O1 \ +// RUN: -fexperimental-new-pass-manager -O1 -mllvm -mandatory-inlining-first=0 \ // RUN: -Rpass-analysis=inline 2>&1 | FileCheck -check-prefix=HOTNESS_OFF %s // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \ // RUN: optimization-remark-with-hotness.c %s -emit-llvm-only \ // RUN: -fprofile-instrument-use-path=%t.profdata -Rpass=inline \ -// RUN: -fexperimental-new-pass-manager -O1 \ +// RUN: -fexperimental-new-pass-manager -O1 -mllvm -mandatory-inlining-first=0 \ // RUN: -Rpass-analysis=inline -Rno-pass-with-hotness 2>&1 | FileCheck \ // RUN: -check-prefix=HOTNESS_OFF %s // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \ // RUN: optimization-remark-with-hotness.c %s -emit-llvm-only \ // RUN: -fprofile-instrument-use-path=%t.profdata -Rpass=inline \ -// RUN: -Rpass-analysis=inline -fdiagnostics-show-hotness \ +// RUN: -Rpass-analysis=inline -fdiagnostics-show-hotness -mllvm -mandatory-inlining-first=0 \ // RUN: -fdiagnostics-hotness-threshold=100 2>&1 \ // RUN: | FileCheck -allow-empty -check-prefix=THRESHOLD %s // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \ // RUN: optimization-remark-with-hotness.c %s -emit-llvm-only \ -// RUN: -Rpass=inline -Rpass-analysis=inline \ +// RUN: -Rpass=inline -Rpass-analysis=inline -mllvm -mandatory-inlining-first=0 \ // RUN: -fdiagnostics-show-hotness -fdiagnostics-hotness-threshold=10 2>&1 \ // RUN: | FileCheck -check-prefix=NO_PGO %s diff --git a/clang/test/Frontend/optimization-remark.c b/clang/test/Frontend/optimization-remark.c index 0b5e9a1..9014169 100644 --- a/clang/test/Frontend/optimization-remark.c +++ b/clang/test/Frontend/optimization-remark.c @@ -7,30 +7,30 @@ // The inliner for the new PM does not seem to be enabled at O0, but we still // get the same remarks with at least O1. The remarks are also slightly // different and located in another test file. -// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O0 -fno-experimental-new-pass-manager -emit-llvm-only -verify -// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O0 -fno-experimental-new-pass-manager -emit-llvm-only -debug-info-kind=line-tables-only -verify -// RUN: %clang_cc1 %s -Rpass=inline -emit-llvm -o - 2>/dev/null | FileCheck %s +// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O0 -fno-experimental-new-pass-manager -emit-llvm-only -verify -mllvm -mandatory-inlining-first=0 +// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O0 -fno-experimental-new-pass-manager -emit-llvm-only -debug-info-kind=line-tables-only -verify -mllvm -mandatory-inlining-first=0 +// RUN: %clang_cc1 %s -Rpass=inline -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>/dev/null | FileCheck %s // // Check that we can override -Rpass= with -Rno-pass. -// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS -// RUN: %clang_cc1 %s -Rpass=inline -Rno-pass -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-NO-REMARKS -// RUN: %clang_cc1 %s -Rpass=inline -Rno-everything -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-NO-REMARKS -// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -Rno-everything -Reverything -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS +// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS +// RUN: %clang_cc1 %s -Rpass=inline -Rno-pass -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-NO-REMARKS +// RUN: %clang_cc1 %s -Rpass=inline -Rno-everything -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-NO-REMARKS +// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -Rno-everything -Reverything -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS // // The inliner for the new PM does not seem to be enabled at O0, but we still // get the same remarks with at least O1. -// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS -// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -Rno-everything -Reverything -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS +// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS +// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -Rno-everything -Reverything -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS // // Check that -w doesn't disable remarks. -// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -w -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS -// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -w -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS +// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -w -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS +// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -w -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS // // FIXME: -Reverything should imply -Rpass=.*. -// RUN: %clang_cc1 %s -Reverything -emit-llvm -o - 2>/dev/null | FileCheck %s --check-prefix=CHECK-NO-REMARKS +// RUN: %clang_cc1 %s -Reverything -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>/dev/null | FileCheck %s --check-prefix=CHECK-NO-REMARKS // // FIXME: -Rpass should either imply -Rpass=.* or should be rejected. -// RUN: %clang_cc1 %s -Rpass -emit-llvm -o - 2>/dev/null | FileCheck %s --check-prefix=CHECK-NO-REMARKS +// RUN: %clang_cc1 %s -Rpass -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>/dev/null | FileCheck %s --check-prefix=CHECK-NO-REMARKS // CHECK-REMARKS: remark: // CHECK-NO-REMARKS-NOT: remark: diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h index f57a609..4dbd5786 100644 --- a/llvm/include/llvm/Analysis/InlineAdvisor.h +++ b/llvm/include/llvm/Analysis/InlineAdvisor.h @@ -24,9 +24,11 @@ class Function; class Module; class OptimizationRemarkEmitter; -/// There are 3 scenarios we can use the InlineAdvisor: +/// There are 4 scenarios we can use the InlineAdvisor: /// - Default - use manual heuristics. /// +/// - MandatoryOnly - only mandatory inlinings (i.e. AlwaysInline). +/// /// - Release mode, the expected mode for production, day to day deployments. /// In this mode, when building the compiler, we also compile a pre-trained ML /// model to native code, and link it as a static library. This mode has low @@ -37,7 +39,12 @@ class OptimizationRemarkEmitter; /// requires the full C Tensorflow API library, and evaluates models /// dynamically. This mode also permits generating training logs, for offline /// training. -enum class InliningAdvisorMode : int { Default, Release, Development }; +enum class InliningAdvisorMode : int { + Default, + MandatoryOnly, + Release, + Development +}; class InlineAdvisor; /// Capture state between an inlining decision having had been made, and @@ -178,6 +185,20 @@ private: InlineParams Params; }; +/// Advisor recommending only mandatory (AlwaysInline) cases. +class MandatoryInlineAdvisor final : public InlineAdvisor { + std::unique_ptr getAdvice(CallBase &CB) override; + +public: + MandatoryInlineAdvisor(FunctionAnalysisManager &FAM) : InlineAdvisor(FAM) {} + + enum class MandatoryInliningKind { NotMandatory, Always, Never }; + + static MandatoryInliningKind getMandatoryKind(CallBase &CB, + FunctionAnalysisManager &FAM, + OptimizationRemarkEmitter &ORE); +}; + /// The InlineAdvisorAnalysis is a module pass because the InlineAdvisor /// needs to capture state right before inlining commences over a module. class InlineAdvisorAnalysis : public AnalysisInfoMixin { diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index 91e5af0..7a5ead6 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -350,7 +350,8 @@ public: /// Construct the module pipeline that performs inlining as well as /// the inlining-driven cleanups. ModuleInlinerWrapperPass buildInlinerPipeline(OptimizationLevel Level, - ThinLTOPhase Phase); + ThinLTOPhase Phase, + bool MandatoryOnly); /// Construct the core LLVM module optimization pipeline. /// diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp index 960141d..4be17ae 100644 --- a/llvm/lib/Analysis/InlineAdvisor.cpp +++ b/llvm/lib/Analysis/InlineAdvisor.cpp @@ -158,6 +158,9 @@ bool InlineAdvisorAnalysis::Result::tryCreate(InlineParams Params, case InliningAdvisorMode::Default: Advisor.reset(new DefaultInlineAdvisor(FAM, Params)); break; + case InliningAdvisorMode::MandatoryOnly: + Advisor.reset(new MandatoryInlineAdvisor(FAM)); + break; case InliningAdvisorMode::Development: #ifdef LLVM_HAVE_TF_API Advisor = @@ -437,3 +440,38 @@ void llvm::emitInlinedInto(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, return Remark; }); } + +std::unique_ptr MandatoryInlineAdvisor::getAdvice(CallBase &CB) { + auto &Caller = *CB.getCaller(); + auto &Callee = *CB.getCalledFunction(); + auto &ORE = FAM.getResult(Caller); + + bool Advice = MandatoryInliningKind::Always == + MandatoryInlineAdvisor::getMandatoryKind(CB, FAM, ORE) && + &Caller != &Callee; + return std::make_unique(this, CB, ORE, Advice); +} + +MandatoryInlineAdvisor::MandatoryInliningKind +MandatoryInlineAdvisor::getMandatoryKind(CallBase &CB, + FunctionAnalysisManager &FAM, + OptimizationRemarkEmitter &ORE) { + auto &Callee = *CB.getCalledFunction(); + + auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & { + return FAM.getResult(F); + }; + + auto &TIR = FAM.getResult(Callee); + + auto TrivialDecision = + llvm::getAttributeBasedInliningDecision(CB, &Callee, TIR, GetTLI); + + if (TrivialDecision.hasValue()) { + if (TrivialDecision->isSuccess()) + return MandatoryInliningKind::Always; + else + return MandatoryInliningKind::Never; + } + return MandatoryInliningKind::NotMandatory; +} diff --git a/llvm/lib/Analysis/MLInlineAdvisor.cpp b/llvm/lib/Analysis/MLInlineAdvisor.cpp index e697af9..1e8833f 100644 --- a/llvm/lib/Analysis/MLInlineAdvisor.cpp +++ b/llvm/lib/Analysis/MLInlineAdvisor.cpp @@ -175,25 +175,20 @@ std::unique_ptr MLInlineAdvisor::getAdvice(CallBase &CB) { auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { return FAM.getResult(F); }; - auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & { - return FAM.getResult(F); - }; - auto &TIR = FAM.getResult(Callee); auto &ORE = FAM.getResult(Caller); - auto TrivialDecision = - llvm::getAttributeBasedInliningDecision(CB, &Callee, TIR, GetTLI); - + auto MandatoryKind = MandatoryInlineAdvisor::getMandatoryKind(CB, FAM, ORE); // If this is a "never inline" case, there won't be any changes to internal // state we need to track, so we can just return the base InlineAdvice, which // will do nothing interesting. // Same thing if this is a recursive case. - if ((TrivialDecision.hasValue() && !TrivialDecision->isSuccess()) || + if (MandatoryKind == MandatoryInlineAdvisor::MandatoryInliningKind::Never || &Caller == &Callee) return std::make_unique(this, CB, ORE, false); - bool Mandatory = TrivialDecision.hasValue() && TrivialDecision->isSuccess(); + bool Mandatory = + MandatoryKind == MandatoryInlineAdvisor::MandatoryInliningKind::Always; // If we need to stop, we won't want to track anymore any state changes, so // we just return the base InlineAdvice, which acts as a noop. diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 0dfa573..d9710bb 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -264,6 +264,11 @@ static cl::opt EnableMemProfiler("enable-mem-prof", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Enable memory profiler")); +static cl::opt PerformMandatoryInliningsFirst( + "mandatory-inlining-first", cl::init(true), cl::Hidden, cl::ZeroOrMore, + cl::desc("Perform mandatory inlinings module-wide, before performing " + "inlining.")); + PipelineTuningOptions::PipelineTuningOptions() { LoopInterleaving = true; LoopVectorization = true; @@ -886,7 +891,8 @@ getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) { } ModuleInlinerWrapperPass -PassBuilder::buildInlinerPipeline(OptimizationLevel Level, ThinLTOPhase Phase) { +PassBuilder::buildInlinerPipeline(OptimizationLevel Level, ThinLTOPhase Phase, + bool MandatoryOnly) { InlineParams IP = getInlineParamsFromOptLevel(Level); if (Phase == PassBuilder::ThinLTOPhase::PreLink && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) @@ -895,8 +901,13 @@ PassBuilder::buildInlinerPipeline(OptimizationLevel Level, ThinLTOPhase Phase) { if (PGOOpt) IP.EnableDeferral = EnablePGOInlineDeferral; - ModuleInlinerWrapperPass MIWP(IP, DebugLogging, UseInlineAdvisor, - MaxDevirtIterations); + ModuleInlinerWrapperPass MIWP( + IP, DebugLogging, + (MandatoryOnly ? InliningAdvisorMode::MandatoryOnly : UseInlineAdvisor), + MaxDevirtIterations); + + if (MandatoryOnly) + return MIWP; // Require the GlobalsAA analysis for the module so we can query it within // the CGSCC pipeline. @@ -1093,7 +1104,9 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, if (EnableSyntheticCounts && !PGOOpt) MPM.addPass(SyntheticCountsPropagation()); - MPM.addPass(buildInlinerPipeline(Level, Phase)); + if (PerformMandatoryInliningsFirst) + MPM.addPass(buildInlinerPipeline(Level, Phase, /*MandatoryOnly=*/true)); + MPM.addPass(buildInlinerPipeline(Level, Phase, /*MandatoryOnly=*/false)); if (EnableMemProfiler && Phase != ThinLTOPhase::PreLink) { MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass())); @@ -1572,6 +1585,9 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM))); + MPM.addPass(ModuleInlinerWrapperPass(getInlineParamsFromOptLevel(Level), + DebugLogging, + InliningAdvisorMode::MandatoryOnly)); // Note: historically, the PruneEH pass was run first to deduce nounwind and // generally clean up exception handling overhead. It isn't clear this is // valuable as the inliner doesn't currently care whether it is inlining an diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 2871282..9c5e97e 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -62,6 +62,10 @@ MODULE_PASS("hwasan", HWAddressSanitizerPass(false, false)) MODULE_PASS("khwasan", HWAddressSanitizerPass(true, true)) MODULE_PASS("inferattrs", InferFunctionAttrsPass()) MODULE_PASS("inliner-wrapper", ModuleInlinerWrapperPass()) +MODULE_PASS("always-inliner-wrapper", ModuleInlinerWrapperPass( + getInlineParams(), + DebugLogging, + InliningAdvisorMode::MandatoryOnly)) MODULE_PASS("insert-gcov-profiling", GCOVProfilerPass()) MODULE_PASS("instrorderfile", InstrOrderFilePass()) MODULE_PASS("instrprof", InstrProfiling()) @@ -95,7 +99,8 @@ MODULE_PASS("rewrite-symbols", RewriteSymbolPass()) MODULE_PASS("rpo-function-attrs", ReversePostOrderFunctionAttrsPass()) MODULE_PASS("sample-profile", SampleProfileLoaderPass()) MODULE_PASS("scc-oz-module-inliner", - buildInlinerPipeline(OptimizationLevel::Oz, ThinLTOPhase::None)) + buildInlinerPipeline(OptimizationLevel::Oz, ThinLTOPhase::None, + /*MandatoryOnly=*/false)) MODULE_PASS("loop-extract-single", LoopExtractorPass(1)) MODULE_PASS("oz-module-optimizer", buildModuleOptimizationPipeline(OptimizationLevel::Oz, /*LTOPreLink*/false)) diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp index 9be4ce0..5b6afc4 100644 --- a/llvm/lib/Transforms/IPO/Inliner.cpp +++ b/llvm/lib/Transforms/IPO/Inliner.cpp @@ -58,7 +58,6 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/Utils/CallPromotionUtils.h" #include "llvm/Transforms/Utils/Cloning.h" #include "llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h" @@ -92,11 +91,6 @@ static cl::opt DisableInlinedAllocaMerging("disable-inlined-alloca-merging", cl::init(false), cl::Hidden); -/// Flag to disable adding AlwaysInlinerPass to ModuleInlinerWrapperPass. -/// TODO: remove this once this has is baked in for long enough. -static cl::opt DisableAlwaysInlinerInModuleWrapper( - "disable-always-inliner-in-module-wrapper", cl::init(false), cl::Hidden); - namespace { enum class InlinerFunctionImportStatsOpts { @@ -1046,8 +1040,6 @@ PreservedAnalyses ModuleInlinerWrapperPass::run(Module &M, return PreservedAnalyses::all(); } - if (!DisableAlwaysInlinerInModuleWrapper) - MPM.addPass(AlwaysInlinerPass()); // We wrap the CGSCC pipeline in a devirtualization repeater. This will try // to detect when we devirtualize indirect calls and iterate the SCC passes // in that case to try and catch knock-on inlining or function attrs diff --git a/llvm/test/Other/new-pm-defaults.ll b/llvm/test/Other/new-pm-defaults.ll index 261dbe0..6377aa0 100644 --- a/llvm/test/Other/new-pm-defaults.ll +++ b/llvm/test/Other/new-pm-defaults.ll @@ -105,16 +105,22 @@ ; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass ; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis ; CHECK-O-NEXT: Starting llvm::Module pass manager run. +; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy +; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis +; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy +; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}> +; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass +; CHECK-O-NEXT: Starting CGSCC pass manager run. +; CHECK-O-NEXT: Running pass: InlinerPass +; CHECK-O-NEXT: Finished CGSCC pass manager run. +; CHECK-O-NEXT: Finished llvm::Module pass manager run. +; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass +; CHECK-O-NEXT: Starting llvm::Module pass manager run. ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA ; CHECK-O-NEXT: Running analysis: GlobalsAA ; CHECK-O-NEXT: Running analysis: CallGraphAnalysis ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis ; CHECK-O-NEXT: Running analysis: ProfileSummaryAnalysis -; CHECK-O-NEXT: Running pass: AlwaysInlinerPass -; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy -; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis -; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy -; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}> ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass ; CHECK-O-NEXT: Starting CGSCC pass manager run. ; CHECK-O-NEXT: Running pass: InlinerPass diff --git a/llvm/test/Other/new-pm-lto-defaults.ll b/llvm/test/Other/new-pm-lto-defaults.ll index 0fc5808..236b93d 100644 --- a/llvm/test/Other/new-pm-lto-defaults.ll +++ b/llvm/test/Other/new-pm-lto-defaults.ll @@ -67,7 +67,12 @@ ; CHECK-O2-NEXT: Running pass: ModuleInlinerWrapperPass ; CHECK-O2-NEXT: Running analysis: InlineAdvisorAnalysis ; CHECK-O2-NEXT: Starting llvm::Module pass manager run. -; CHECK-O2-NEXT: Running pass: AlwaysInlinerPass +; CHECK-O2-NEXT: Starting CGSCC pass manager run. +; CHECK-O2-NEXT: Running pass: InlinerPass +; CHECK-O2-NEXT: Finished CGSCC pass manager run. +; CHECK-O2-NEXT: Finished llvm::Module pass manager run. +; CHECK-O2-NEXT: Running pass: ModuleInlinerWrapperPass +; CHECK-O2-NEXT: Starting llvm::Module pass manager run. ; CHECK-O2-NEXT: Starting CGSCC pass manager run. ; CHECK-O2-NEXT: Running pass: InlinerPass ; CHECK-O2-NEXT: Finished CGSCC pass manager run. diff --git a/llvm/test/Other/new-pm-module-inliner-wrapper.ll b/llvm/test/Other/new-pm-module-inliner-wrapper.ll index becee55..111be03 100644 --- a/llvm/test/Other/new-pm-module-inliner-wrapper.ll +++ b/llvm/test/Other/new-pm-module-inliner-wrapper.ll @@ -1,8 +1,5 @@ -; RUN: opt -disable-verify -debug-pass-manager -passes='inliner-wrapper' -S %s 2>&1 | FileCheck %s --check-prefixes=CHECK,ALWAYS -; RUN: opt -disable-verify -disable-always-inliner-in-module-wrapper -debug-pass-manager -passes='inliner-wrapper' -S %s 2>&1 | FileCheck %s --check-prefixes=CHECK,DISABLEALWAYS +; RUN: opt -disable-verify -debug-pass-manager -passes='always-inliner-wrapper' -S %s 2>&1 | FileCheck %s -; DISABLEALWAYS-NOT: Running pass: AlwaysInlinerPass -; ALWAYS: Running pass: AlwaysInlinerPass ; CHECK: Running pass: InlinerPass define void @foo() { diff --git a/llvm/test/Other/new-pm-thinlto-defaults.ll b/llvm/test/Other/new-pm-thinlto-defaults.ll index a62f27f..45af638 100644 --- a/llvm/test/Other/new-pm-thinlto-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-defaults.ll @@ -90,16 +90,22 @@ ; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass ; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis ; CHECK-O-NEXT: Starting llvm::Module pass manager run. +; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy +; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis +; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy +; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy +; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass +; CHECK-O-NEXT: Starting CGSCC pass manager run. +; CHECK-O-NEXT: Running pass: InlinerPass +; CHECK-O-NEXT: Finished CGSCC pass manager run. +; CHECK-O-NEXT: Finished llvm::Module pass manager run. +; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass +; CHECK-O-NEXT: Starting llvm::Module pass manager run. ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA ; CHECK-O-NEXT: Running analysis: GlobalsAA ; CHECK-O-NEXT: Running analysis: CallGraphAnalysis ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis ; CHECK-PRELINK-O-NEXT: Running analysis: ProfileSummaryAnalysis -; CHECK-O-NEXT: Running pass: AlwaysInlinerPass -; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy -; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis -; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy -; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass ; CHECK-O-NEXT: Starting CGSCC pass manager run. ; CHECK-O-NEXT: Running pass: InlinerPass diff --git a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll index 996f5f4..0c0e73e 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll @@ -65,11 +65,6 @@ ; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass ; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run. -; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA -; CHECK-O-NEXT: Running analysis: GlobalsAA -; CHECK-O-NEXT: Running analysis: CallGraphAnalysis -; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis -; CHECK-O-NEXT: Running pass: AlwaysInlinerPass ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy @@ -77,6 +72,17 @@ ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass ; CHECK-O-NEXT: Starting CGSCC pass manager run. ; CHECK-O-NEXT: Running pass: InlinerPass +; CHECK-O-NEXT: Finished CGSCC pass manager run. +; CHECK-O-NEXT: Finished {{.*}}Module pass manager run. +; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass +; CHECK-O-NEXT: Starting {{.*}}Module pass manager run. +; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA +; CHECK-O-NEXT: Running analysis: GlobalsAA +; CHECK-O-NEXT: Running analysis: CallGraphAnalysis +; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis +; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass +; CHECK-O-NEXT: Starting CGSCC pass manager run. +; CHECK-O-NEXT: Running pass: InlinerPass ; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass ; CHECK-O3-NEXT: Running pass: ArgumentPromotionPass ; CHECK-O2-NEXT: Running pass: OpenMPOptPass diff --git a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll index 2587d07..3dd9009 100644 --- a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll @@ -71,13 +71,10 @@ ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis on foo ; CHECK-O-NEXT: Running pass: SimplifyCFGPass on foo ; CHECK-O-NEXT: Finished {{.*}}Function pass manager run + ; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass ; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run. -; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA -; CHECK-O-NEXT: Running analysis: GlobalsAA -; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis -; CHECK-O-NEXT: Running pass: AlwaysInlinerPass ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy @@ -85,6 +82,16 @@ ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass ; CHECK-O-NEXT: Starting CGSCC pass manager run. ; CHECK-O-NEXT: Running pass: InlinerPass +; CHECK-O-NEXT: Finished CGSCC pass manager run. +; CHECK-O-NEXT: Finished {{.*}}Module pass manager run. +; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass +; CHECK-O-NEXT: Starting {{.*}}Module pass manager run. +; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA +; CHECK-O-NEXT: Running analysis: GlobalsAA +; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis +; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass +; CHECK-O-NEXT: Starting CGSCC pass manager run. +; CHECK-O-NEXT: Running pass: InlinerPass ; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass ; CHECK-O3-NEXT: Running pass: ArgumentPromotionPass ; CHECK-O2-NEXT: Running pass: OpenMPOptPass diff --git a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll index d1a9b00..66694b1 100644 --- a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll @@ -60,8 +60,6 @@ ; CHECK-O123-NEXT: Running pass: ModuleInlinerWrapperPass ; CHECK-O123-NEXT: Running analysis: InlineAdvisorAnalysis ; CHECK-O123-NEXT: Starting {{.*}}Module pass manager run. -; CHECK-O123-NEXT: Running pass: AlwaysInlinerPass -; CHECK-O123-NEXT: Running analysis: ProfileSummaryAnalysis ; CHECK-O123-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O123-NEXT: Running analysis: LazyCallGraphAnalysis ; CHECK-O123-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy on (foo) @@ -93,11 +91,6 @@ ; CHECK-Os-NEXT: Running analysis: InlineAdvisorAnalysis ; CHECK-Oz-NEXT: Running analysis: InlineAdvisorAnalysis ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run. -; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA -; CHECK-O-NEXT: Running analysis: GlobalsAA -; CHECK-O-NEXT: Running analysis: CallGraphAnalysis -; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis -; CHECK-O-NEXT: Running pass: AlwaysInlinerPass ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis on foo @@ -106,6 +99,17 @@ ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass ; CHECK-O-NEXT: Starting CGSCC pass manager run. ; CHECK-O-NEXT: Running pass: InlinerPass +; CHECK-O-NEXT: Finished CGSCC pass manager run. +; CHECK-O-NEXT: Finished {{.*}}Module pass manager run. +; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass +; CHECK-O-NEXT: Starting {{.*}}Module pass manager run. +; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA +; CHECK-O-NEXT: Running analysis: GlobalsAA +; CHECK-O-NEXT: Running analysis: CallGraphAnalysis +; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis +; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass +; CHECK-O-NEXT: Starting CGSCC pass manager run. +; CHECK-O-NEXT: Running pass: InlinerPass ; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass ; CHECK-O-NEXT: Running analysis: AAManager on foo ; CHECK-O3-NEXT: Running pass: ArgumentPromotionPass diff --git a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll index 7078c6d..25b277f 100644 --- a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll +++ b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll @@ -70,10 +70,6 @@ ; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass ; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run. -; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA -; CHECK-O-NEXT: Running analysis: GlobalsAA -; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis -; CHECK-O-NEXT: Running pass: AlwaysInlinerPass ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy @@ -81,6 +77,16 @@ ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass ; CHECK-O-NEXT: Starting CGSCC pass manager run. ; CHECK-O-NEXT: Running pass: InlinerPass +; CHECK-O-NEXT: Finished CGSCC pass manager run. +; CHECK-O-NEXT: Finished llvm::Module pass manager run. +; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass +; CHECK-O-NEXT: Starting llvm::Module pass manager run. +; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA +; CHECK-O-NEXT: Running analysis: GlobalsAA +; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis +; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass +; CHECK-O-NEXT: Starting CGSCC pass manager run. +; CHECK-O-NEXT: Running pass: InlinerPass ; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass ; CHECK-O3-NEXT: Running pass: ArgumentPromotionPass ; CHECK-O2-NEXT: Running pass: OpenMPOptPass diff --git a/llvm/test/Transforms/Inline/ML/bounds-checks-rewards.ll b/llvm/test/Transforms/Inline/ML/bounds-checks-rewards.ll index c5edf1a..abcc27e 100644 --- a/llvm/test/Transforms/Inline/ML/bounds-checks-rewards.ll +++ b/llvm/test/Transforms/Inline/ML/bounds-checks-rewards.ll @@ -7,18 +7,18 @@ ; REQUIRES: have_tf_api ; ; When the bounds are very wide ("no bounds"), all inlinings happen. -; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-model-under-training=%S/../../../../lib/Analysis/models/inliner -training-log=- -enable-ml-inliner=development -ml-advisor-size-increase-threshold=10.0 -disable-always-inliner-in-module-wrapper -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=NOBOUNDS +; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-model-under-training=%S/../../../../lib/Analysis/models/inliner -training-log=- -enable-ml-inliner=development -ml-advisor-size-increase-threshold=10.0 -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=NOBOUNDS ; ; When the bounds are very restrictive, the first inlining happens but it's ; considered as "bad" (since it trips over the bounds) and its reward is a ; penalty. However, the mandatory inlining, which is considered next, happens. ; No other inlinings happend. -; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-model-under-training=%S/../../../../lib/Analysis/models/inliner -training-log=- -enable-ml-inliner=development -ml-advisor-size-increase-threshold=1.0 -disable-always-inliner-in-module-wrapper -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=BOUNDS +; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-model-under-training=%S/../../../../lib/Analysis/models/inliner -training-log=- -enable-ml-inliner=development -ml-advisor-size-increase-threshold=1.0 -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=BOUNDS ; ; With more restrictive bounds, the first inlining happens and is OK. The ; mandatory inlining happens next, and it trips over the bounds, which then ; forces no further inlinings. -; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-model-under-training=%S/../../../../lib/Analysis/models/inliner -training-log=- -enable-ml-inliner=development -ml-advisor-size-increase-threshold=1.1 -disable-always-inliner-in-module-wrapper -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=RELAXED-BOUNDS +; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-model-under-training=%S/../../../../lib/Analysis/models/inliner -training-log=- -enable-ml-inliner=development -ml-advisor-size-increase-threshold=1.1 -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=RELAXED-BOUNDS target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-grtev4-linux-gnu" diff --git a/llvm/test/Transforms/Inline/ML/bounds-checks.ll b/llvm/test/Transforms/Inline/ML/bounds-checks.ll index 986f54c..b98e23a 100644 --- a/llvm/test/Transforms/Inline/ML/bounds-checks.ll +++ b/llvm/test/Transforms/Inline/ML/bounds-checks.ll @@ -4,7 +4,7 @@ ; factor, we don't inline anymore. ; REQUIRES: have_tf_aot ; RUN: opt -passes=scc-oz-module-inliner -enable-ml-inliner=release -ml-advisor-size-increase-threshold=10.0 -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=NOBOUNDS -; RUN: opt -passes=scc-oz-module-inliner -enable-ml-inliner=release -ml-advisor-size-increase-threshold=1.0 -disable-always-inliner-in-module-wrapper -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=BOUNDS +; RUN: opt -passes=scc-oz-module-inliner -enable-ml-inliner=release -ml-advisor-size-increase-threshold=1.0 -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=BOUNDS target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-grtev4-linux-gnu" diff --git a/llvm/test/Transforms/Inline/inline_stats.ll b/llvm/test/Transforms/Inline/inline_stats.ll index 65d8ad2..3010b56 100644 --- a/llvm/test/Transforms/Inline/inline_stats.ll +++ b/llvm/test/Transforms/Inline/inline_stats.ll @@ -6,8 +6,11 @@ ; RUN: opt -S -passes=inline -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=CHECK-BASIC -check-prefix=CHECK ; RUN: opt -S -passes=inline -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix="CHECK-VERBOSE" -check-prefix=CHECK -; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=WRAPPER-BASIC -check-prefix=WRAPPER -; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix=WRAPPER-VERBOSE -check-prefix=WRAPPER +; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=CHECK-BASIC -check-prefix=CHECK +; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix="CHECK-VERBOSE" -check-prefix=CHECK + +; RUN: opt -S -passes=always-inliner-wrapper,inliner-wrapper -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=WRAPPER-BASIC -check-prefix=WRAPPER +; RUN: opt -S -passes=always-inliner-wrapper,inliner-wrapper -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix=WRAPPER-VERBOSE -check-prefix=WRAPPER ; CHECK: ------- Dumping inliner stats for [] ------- ; CHECK-BASIC-NOT: -- List of inlined functions: diff --git a/llvm/test/Transforms/Inline/pr46945.ll b/llvm/test/Transforms/Inline/pr46945.ll index 540a33c..2d27ba2 100644 --- a/llvm/test/Transforms/Inline/pr46945.ll +++ b/llvm/test/Transforms/Inline/pr46945.ll @@ -1,12 +1,13 @@ -; RUN: opt %s -o - -S -passes=inliner-wrapper | FileCheck %s +; RUN: opt %s -o - -S -passes=always-inliner-wrapper | FileCheck %s +; RUN: opt %s -o - -S -passes='default' | FileCheck %s +; RUN: opt %s -o - -S -passes=inliner-wrapper | FileCheck %s -check-prefix=BASELINE -; CHECK-NOT: call void @b() -define void @a() { -entry: - call void @b() - ret void -} +; In the baseline case, a will be first inlined into b, which makes c recursive, +; and, thus, un-inlinable. We need a baseline case to make sure intra-SCC order +; is as expected: b first, then a. +; BASELINE: call void @b() +; CHECK-NOT: call void @b() define void @b() alwaysinline { entry: br label %for.cond @@ -16,3 +17,8 @@ for.cond: br label %for.cond } +define void @a() { +entry: + call void @b() + ret void +} -- 2.7.4