From 7175886a0f612aded1430ae240ca7ffd53d260dd Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Wed, 3 Nov 2021 15:45:30 -0700 Subject: [PATCH] [NewPM] Make eager analysis invalidation per-adaptor Follow-up change to D111575. We don't need eager invalidation on every adaptor. Most notably, adaptors running passes that use very few analyses, or passes that purely invalidate specific analyses. Also allow testing of this via a pipeline string "function()". The compile time/memory impact of this is very comparable to D111575. https://llvm-compile-time-tracker.com/compare.php?from=9a2eec512a29df45c90c2fcb741e9d5c693b1383&to=b9f20bcdea138060967d95a98eab87ce725b22bb&stat=instructions Reviewed By: nikic Differential Revision: https://reviews.llvm.org/D113196 --- llvm/include/llvm/Analysis/CGSCCPassManager.h | 19 ++++++++++----- llvm/include/llvm/IR/PassManager.h | 12 ++++++--- llvm/include/llvm/Passes/PassBuilder.h | 9 +++++++ llvm/lib/Analysis/CGSCCPassManager.cpp | 5 +--- llvm/lib/IR/PassManager.cpp | 19 ++++----------- llvm/lib/Passes/PassBuilder.cpp | 16 ++++++------ llvm/lib/Passes/PassBuilderPipelines.cpp | 35 +++++++++++++++++++-------- llvm/test/Other/new-pm-eager-invalidate.ll | 29 ++++++++++++++++++---- llvm/test/Other/new-pm-print-pipeline.ll | 3 +++ 9 files changed, 97 insertions(+), 50 deletions(-) diff --git a/llvm/include/llvm/Analysis/CGSCCPassManager.h b/llvm/include/llvm/Analysis/CGSCCPassManager.h index 9acb7c0..8560732 100644 --- a/llvm/include/llvm/Analysis/CGSCCPassManager.h +++ b/llvm/include/llvm/Analysis/CGSCCPassManager.h @@ -477,11 +477,12 @@ class CGSCCToFunctionPassAdaptor public: using PassConceptT = detail::PassConcept; - explicit CGSCCToFunctionPassAdaptor(std::unique_ptr Pass) - : Pass(std::move(Pass)) {} + explicit CGSCCToFunctionPassAdaptor(std::unique_ptr Pass, + bool EagerlyInvalidate) + : Pass(std::move(Pass)), EagerlyInvalidate(EagerlyInvalidate) {} CGSCCToFunctionPassAdaptor(CGSCCToFunctionPassAdaptor &&Arg) - : Pass(std::move(Arg.Pass)) {} + : Pass(std::move(Arg.Pass)), EagerlyInvalidate(Arg.EagerlyInvalidate) {} friend void swap(CGSCCToFunctionPassAdaptor &LHS, CGSCCToFunctionPassAdaptor &RHS) { @@ -499,7 +500,10 @@ public: void printPipeline(raw_ostream &OS, function_ref MapClassName2PassName) { - OS << "function("; + OS << "function"; + if (EagerlyInvalidate) + OS << ""; + OS << "("; Pass->printPipeline(OS, MapClassName2PassName); OS << ")"; } @@ -508,13 +512,15 @@ public: private: std::unique_ptr Pass; + bool EagerlyInvalidate; }; /// A function to deduce a function pass type and wrap it in the /// templated adaptor. template CGSCCToFunctionPassAdaptor -createCGSCCToFunctionPassAdaptor(FunctionPassT &&Pass) { +createCGSCCToFunctionPassAdaptor(FunctionPassT &&Pass, + bool EagerlyInvalidate = false) { using PassModelT = detail::PassModel; @@ -522,7 +528,8 @@ createCGSCCToFunctionPassAdaptor(FunctionPassT &&Pass) { // causing terrible compile times. return CGSCCToFunctionPassAdaptor( std::unique_ptr( - new PassModelT(std::forward(Pass)))); + new PassModelT(std::forward(Pass))), + EagerlyInvalidate); } /// A helper that repeats an SCC pass each time an indirect call is refined to diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h index d4f0bb3..e88d223 100644 --- a/llvm/include/llvm/IR/PassManager.h +++ b/llvm/include/llvm/IR/PassManager.h @@ -1204,8 +1204,9 @@ class ModuleToFunctionPassAdaptor public: using PassConceptT = detail::PassConcept; - explicit ModuleToFunctionPassAdaptor(std::unique_ptr Pass) - : Pass(std::move(Pass)) {} + explicit ModuleToFunctionPassAdaptor(std::unique_ptr Pass, + bool EagerlyInvalidate) + : Pass(std::move(Pass)), EagerlyInvalidate(EagerlyInvalidate) {} /// Runs the function pass across every function in the module. PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); @@ -1216,13 +1217,15 @@ public: private: std::unique_ptr Pass; + bool EagerlyInvalidate; }; /// A function to deduce a function pass type and wrap it in the /// templated adaptor. template ModuleToFunctionPassAdaptor -createModuleToFunctionPassAdaptor(FunctionPassT &&Pass) { +createModuleToFunctionPassAdaptor(FunctionPassT &&Pass, + bool EagerlyInvalidate = false) { using PassModelT = detail::PassModel; @@ -1230,7 +1233,8 @@ createModuleToFunctionPassAdaptor(FunctionPassT &&Pass) { // causing terrible compile times. return ModuleToFunctionPassAdaptor( std::unique_ptr( - new PassModelT(std::forward(Pass)))); + new PassModelT(std::forward(Pass))), + EagerlyInvalidate); } /// A utility pass template to force an analysis result to be available. diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index 943ad31..4cbda10 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -73,6 +73,15 @@ public: /// Tuning option to enable/disable function merging. Its default value is /// false. bool MergeFunctions; + + // Experimental option to eagerly invalidate more analyses. This has the + // potential to decrease max memory usage in exchange for more compile time. + // This may affect codegen due to either passes using analyses only when + // cached, or invalidating and recalculating an analysis that was + // stale/imprecise but still valid. Currently this invalidates all function + // analyses after various module->function or cgscc->function adaptors in the + // default pipelines. + bool EagerlyInvalidateAnalyses; }; /// This class provides access to building LLVM's passes. diff --git a/llvm/lib/Analysis/CGSCCPassManager.cpp b/llvm/lib/Analysis/CGSCCPassManager.cpp index c2ba3d6..ae60b9c 100644 --- a/llvm/lib/Analysis/CGSCCPassManager.cpp +++ b/llvm/lib/Analysis/CGSCCPassManager.cpp @@ -38,8 +38,6 @@ using namespace llvm; // Explicit template instantiations and specialization definitions for core // template typedefs. namespace llvm { -extern cl::opt EagerlyInvalidateAnalyses; - static cl::opt AbortOnMaxDevirtIterationsReached( "abort-on-max-devirt-iterations-reached", cl::desc("Abort when the max iterations for devirtualization CGSCC repeat " @@ -557,8 +555,7 @@ PreservedAnalyses CGSCCToFunctionPassAdaptor::run(LazyCallGraph::SCC &C, // We know that the function pass couldn't have invalidated any other // function's analyses (that's the contract of a function pass), so // directly handle the function analysis manager's invalidation here. - FAM.invalidate(F, EagerlyInvalidateAnalyses ? PreservedAnalyses::none() - : PassPA); + FAM.invalidate(F, EagerlyInvalidate ? PreservedAnalyses::none() : PassPA); // Then intersect the preserved set so that invalidation of module // analyses will eventually occur when the module pass completes. diff --git a/llvm/lib/IR/PassManager.cpp b/llvm/lib/IR/PassManager.cpp index 4965a7d..d933003 100644 --- a/llvm/lib/IR/PassManager.cpp +++ b/llvm/lib/IR/PassManager.cpp @@ -15,17 +15,6 @@ using namespace llvm; namespace llvm { -// Experimental option to eagerly invalidate more analyses. This has the -// potential to decrease max memory usage in exchange for more compile time. -// This may affect codegen due to either passes using analyses only when -// cached, or invalidating and recalculating an analysis that was -// stale/imprecise but still valid. Currently this invalidates all function -// analyses after a module->function or cgscc->function adaptor. -// TODO: make this a PipelineTuningOption. -cl::opt EagerlyInvalidateAnalyses( - "eagerly-invalidate-analyses", cl::init(false), cl::Hidden, - cl::desc("Eagerly invalidate more analyses in default pipelines")); - // Explicit template instantiations and specialization defininitions for core // template typedefs. template class AllAnalysesOn; @@ -105,7 +94,10 @@ bool FunctionAnalysisManagerModuleProxy::Result::invalidate( void ModuleToFunctionPassAdaptor::printPipeline( raw_ostream &OS, function_ref MapClassName2PassName) { - OS << "function("; + OS << "function"; + if (EagerlyInvalidate) + OS << ""; + OS << "("; Pass->printPipeline(OS, MapClassName2PassName); OS << ")"; } @@ -141,8 +133,7 @@ PreservedAnalyses ModuleToFunctionPassAdaptor::run(Module &M, // We know that the function pass couldn't have invalidated any other // function's analyses (that's the contract of a function pass), so // directly handle the function analysis manager's invalidation here. - FAM.invalidate(F, EagerlyInvalidateAnalyses ? PreservedAnalyses::none() - : PassPA); + FAM.invalidate(F, EagerlyInvalidate ? PreservedAnalyses::none() : PassPA); // Then intersect the preserved set so that invalidation of module // analyses will eventually occur when the module pass completes. diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 74d96c7..40a885f 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -832,7 +832,7 @@ static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) { return true; if (Name == "cgscc") return true; - if (Name == "function") + if (Name == "function" || Name == "function") return true; // Explicitly handle custom-parsed pass names. @@ -858,7 +858,7 @@ static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) { // Explicitly handle pass manager names. if (Name == "cgscc") return true; - if (Name == "function") + if (Name == "function" || Name == "function") return true; // Explicitly handle custom-parsed pass names. @@ -884,7 +884,7 @@ static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) { template static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) { // Explicitly handle pass manager names. - if (Name == "function") + if (Name == "function" || Name == "function") return true; if (Name == "loop" || Name == "loop-mssa") return true; @@ -1013,11 +1013,12 @@ Error PassBuilder::parseModulePass(ModulePassManager &MPM, MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM))); return Error::success(); } - if (Name == "function") { + if (Name == "function" || Name == "function") { FunctionPassManager FPM; if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline)) return Err; - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM), + Name != "function")); return Error::success(); } if (auto Count = parseRepeatPassName(Name)) { @@ -1180,12 +1181,13 @@ Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM, CGPM.addPass(std::move(NestedCGPM)); return Error::success(); } - if (Name == "function") { + if (Name == "function" || Name == "function") { FunctionPassManager FPM; if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline)) return Err; // Add the nested pass manager with the appropriate adaptor. - CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM))); + CGPM.addPass( + createCGSCCToFunctionPassAdaptor(std::move(FPM), Name != "function")); return Error::success(); } if (auto Count = parseRepeatPassName(Name)) { diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index 8274aee..8b95b4c 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -162,6 +162,10 @@ static cl::opt EnableO3NonTrivialUnswitching( "enable-npm-O3-nontrivial-unswitch", cl::init(true), cl::Hidden, cl::ZeroOrMore, cl::desc("Enable non-trivial loop unswitching for -O3")); +static cl::opt EnableEagerlyInvalidateAnalyses( + "eagerly-invalidate-analyses", cl::init(false), cl::Hidden, + cl::desc("Eagerly invalidate more analyses in default pipelines")); + PipelineTuningOptions::PipelineTuningOptions() { LoopInterleaving = true; LoopVectorization = true; @@ -172,6 +176,7 @@ PipelineTuningOptions::PipelineTuningOptions() { LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap; CallGraphProfile = true; MergeFunctions = false; + EagerlyInvalidateAnalyses = EnableEagerlyInvalidateAnalyses; } namespace llvm { @@ -596,7 +601,8 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, FPM.addPass(InstCombinePass()); // Combine silly sequences. invokePeepholeEPCallbacks(FPM, Level); - CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM))); + CGPipeline.addPass(createCGSCCToFunctionPassAdaptor( + std::move(FPM), PTO.EagerlyInvalidateAnalyses)); MPM.addPass(std::move(MIWP)); @@ -623,7 +629,8 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, FPM.addPass(createFunctionToLoopPassAdaptor( LoopRotatePass(Level != OptimizationLevel::Oz), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false)); - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM), + PTO.EagerlyInvalidateAnalyses)); // Add the profile lowering pass. InstrProfOptions Options; @@ -723,7 +730,8 @@ PassBuilder::buildInlinerPipeline(OptimizationLevel Level, // Lastly, add the core function simplification pipeline nested inside the // CGSCC walk. MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor( - buildFunctionSimplificationPipeline(Level, Phase))); + buildFunctionSimplificationPipeline(Level, Phase), + PTO.EagerlyInvalidateAnalyses)); MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0)); @@ -792,7 +800,8 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured. if (LoadSampleProfile) EarlyFPM.addPass(InstCombinePass()); - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM))); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM), + PTO.EagerlyInvalidateAnalyses)); if (LoadSampleProfile) { // Annotate sample profile right after early FPM to ensure freshness of @@ -866,7 +875,8 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, invokePeepholeEPCallbacks(GlobalCleanupPM, Level); GlobalCleanupPM.addPass(SimplifyCFGPass()); - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM))); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM), + PTO.EagerlyInvalidateAnalyses)); // Add all the requested passes for instrumentation PGO, if requested. if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink && @@ -1154,7 +1164,8 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level, OptimizePM.addPass(CoroCleanupPass()); // Add the core optimizing pipeline. - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM))); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM), + PTO.EagerlyInvalidateAnalyses)); for (auto &C : OptimizerLastEPCallbacks) C(MPM, Level); @@ -1397,7 +1408,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, if (Level.getSpeedupLevel() > 1) { FunctionPassManager EarlyFPM; EarlyFPM.addPass(CallSiteSplittingPass()); - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM))); + MPM.addPass(createModuleToFunctionPassAdaptor( + std::move(EarlyFPM), PTO.EagerlyInvalidateAnalyses)); // Indirect call promotion. This should promote all the targets that are // left by the earlier promotion pass that promotes intra-module targets. @@ -1473,7 +1485,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, PeepholeFPM.addPass(InstCombinePass()); invokePeepholeEPCallbacks(PeepholeFPM, Level); - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM))); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM), + PTO.EagerlyInvalidateAnalyses)); // Note: historically, the PruneEH pass was run first to deduce nounwind and // generally clean up exception handling overhead. It isn't clear this is @@ -1520,7 +1533,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, FPM.addPass(TailCallElimPass()); // Run a few AA driver optimizations here and now to cleanup the code. - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM), + PTO.EagerlyInvalidateAnalyses)); MPM.addPass( createModuleToPostOrderCGSCCPassAdaptor(PostOrderFunctionAttrsPass())); @@ -1577,7 +1591,8 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, invokePeepholeEPCallbacks(MainFPM, Level); MainFPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true)); - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM))); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM), + PTO.EagerlyInvalidateAnalyses)); // Lower type metadata and the type.test intrinsic. This pass supports // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs diff --git a/llvm/test/Other/new-pm-eager-invalidate.ll b/llvm/test/Other/new-pm-eager-invalidate.ll index 188ac9b..cabc977 100644 --- a/llvm/test/Other/new-pm-eager-invalidate.ll +++ b/llvm/test/Other/new-pm-eager-invalidate.ll @@ -1,8 +1,27 @@ -; RUN: opt -disable-verify -debug-pass-manager -passes='function(require)' -disable-output -eagerly-invalidate-analyses %s 2>&1 | FileCheck %s -; RUN: opt -disable-verify -debug-pass-manager -passes='cgscc(function(require))' -disable-output -eagerly-invalidate-analyses %s 2>&1 | FileCheck %s +; RUN: opt -disable-verify -debug-pass-manager -passes='function(require)' -disable-output %s 2>&1 | FileCheck %s --check-prefix=NORMAL +; RUN: opt -disable-verify -debug-pass-manager -passes='cgscc(function(require))' -disable-output %s 2>&1 | FileCheck %s --check-prefix=NORMAL +; RUN: opt -disable-verify -debug-pass-manager -passes='function(require)' -disable-output %s 2>&1 | FileCheck %s --check-prefix=EAGER +; RUN: opt -disable-verify -debug-pass-manager -passes='cgscc(function(require))' -disable-output %s 2>&1 | FileCheck %s --check-prefix=EAGER -; CHECK: Invalidating analysis: NoOpFunctionAnalysis +; RUN: opt -disable-verify -debug-pass-manager -passes='default' -disable-output %s 2>&1 | FileCheck %s --check-prefix=PIPELINE +; RUN: opt -disable-verify -debug-pass-manager -passes='default' -eagerly-invalidate-analyses -disable-output %s 2>&1 | FileCheck %s --check-prefix=PIPELINE-EAGER -define void @foo() { - unreachable +; NORMAL-NOT: Invalidating analysis: NoOpFunctionAnalysis +; EAGER: Invalidating analysis: NoOpFunctionAnalysis +; PIPELINE-NOT: Invalidating analysis: DominatorTreeAnalysis +; PIPELINE-EAGER: Invalidating analysis: DominatorTreeAnalysis + +declare void @bar() local_unnamed_addr + +define void @foo(i32 %n) local_unnamed_addr { +entry: + br label %loop +loop: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] + %iv.next = add i32 %iv, 1 + tail call void @bar() + %cmp = icmp eq i32 %iv, %n + br i1 %cmp, label %exit, label %loop +exit: + ret void } diff --git a/llvm/test/Other/new-pm-print-pipeline.ll b/llvm/test/Other/new-pm-print-pipeline.ll index 11fa916..da06e93 100644 --- a/llvm/test/Other/new-pm-print-pipeline.ll +++ b/llvm/test/Other/new-pm-print-pipeline.ll @@ -66,3 +66,6 @@ ; RUN: opt -disable-output -disable-verify -print-pipeline-passes -passes='scc-oz-module-inliner' < %s | FileCheck %s --match-full-lines --check-prefixes=CHECK-21 ; CHECK-21: require,function(invalidate),require,cgscc(devirt<4>(inline,inline,{{.*}},instcombine{{.*}})) + +; RUN: opt -disable-output -disable-verify -print-pipeline-passes -passes='cgscc(function(no-op-function)),function(no-op-function)' < %s | FileCheck %s --match-full-lines --check-prefixes=CHECK-22 +; CHECK-22: cgscc(function(no-op-function)),function(no-op-function) -- 2.7.4