From 33817296c6003f6b3bb7eed5fbcd64a2385fe425 Mon Sep 17 00:00:00 2001 From: Prem Chintalapudi Date: Mon, 17 Apr 2023 17:14:30 -0700 Subject: [PATCH] Expose PassBuilder extension point callbacks This patch allows access to callbacks registered by TargetMachines to allow custom pipelines to run those callbacks. Reviewed By: aeubanks Differential Revision: https://reviews.llvm.org/D148561 --- llvm/include/llvm/Passes/PassBuilder.h | 29 ++++++- llvm/lib/Passes/PassBuilderPipelines.cpp | 139 ++++++++++++++++++------------- 2 files changed, 111 insertions(+), 57 deletions(-) diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index 589bde2..585c335 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -555,6 +555,34 @@ public: return PIC; } + // Invoke the callbacks registered for the various extension points. + // Custom pipelines should use these to invoke the callbacks registered + // by TargetMachines and other clients. + void invokePeepholeEPCallbacks(FunctionPassManager &FPM, + OptimizationLevel Level); + void invokeLateLoopOptimizationsEPCallbacks(LoopPassManager &LPM, + OptimizationLevel Level); + void invokeLoopOptimizerEndEPCallbacks(LoopPassManager &LPM, + OptimizationLevel Level); + void invokeScalarOptimizerLateEPCallbacks(FunctionPassManager &FPM, + OptimizationLevel Level); + void invokeCGSCCOptimizerLateEPCallbacks(CGSCCPassManager &CGPM, + OptimizationLevel Level); + void invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM, + OptimizationLevel Level); + void invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level); + void invokeOptimizerLastEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level); + void invokeFullLinkTimeOptimizationEarlyEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level); + void invokeFullLinkTimeOptimizationLastEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level); + void invokePipelineStartEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level); + void invokePipelineEarlySimplificationEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level); + private: // O1 pass pipeline FunctionPassManager @@ -589,7 +617,6 @@ private: std::string ProfileRemappingFile, ThinOrFullLTOPhase LTOPhase, IntrusiveRefCntPtr FS); - void invokePeepholeEPCallbacks(FunctionPassManager &, OptimizationLevel); // Extension Point callbacks SmallVector, 2> diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index 52b2304..3ef8b26 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -300,6 +300,61 @@ void PassBuilder::invokePeepholeEPCallbacks(FunctionPassManager &FPM, for (auto &C : PeepholeEPCallbacks) C(FPM, Level); } +void PassBuilder::invokeLateLoopOptimizationsEPCallbacks( + LoopPassManager &LPM, OptimizationLevel Level) { + for (auto &C : LateLoopOptimizationsEPCallbacks) + C(LPM, Level); +} +void PassBuilder::invokeLoopOptimizerEndEPCallbacks(LoopPassManager &LPM, + OptimizationLevel Level) { + for (auto &C : LoopOptimizerEndEPCallbacks) + C(LPM, Level); +} +void PassBuilder::invokeScalarOptimizerLateEPCallbacks( + FunctionPassManager &FPM, OptimizationLevel Level) { + for (auto &C : ScalarOptimizerLateEPCallbacks) + C(FPM, Level); +} +void PassBuilder::invokeCGSCCOptimizerLateEPCallbacks(CGSCCPassManager &CGPM, + OptimizationLevel Level) { + for (auto &C : CGSCCOptimizerLateEPCallbacks) + C(CGPM, Level); +} +void PassBuilder::invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM, + OptimizationLevel Level) { + for (auto &C : VectorizerStartEPCallbacks) + C(FPM, Level); +} +void PassBuilder::invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level) { + for (auto &C : OptimizerEarlyEPCallbacks) + C(MPM, Level); +} +void PassBuilder::invokeOptimizerLastEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level) { + for (auto &C : OptimizerLastEPCallbacks) + C(MPM, Level); +} +void PassBuilder::invokeFullLinkTimeOptimizationEarlyEPCallbacks( + ModulePassManager &MPM, OptimizationLevel Level) { + for (auto &C : FullLinkTimeOptimizationEarlyEPCallbacks) + C(MPM, Level); +} +void PassBuilder::invokeFullLinkTimeOptimizationLastEPCallbacks( + ModulePassManager &MPM, OptimizationLevel Level) { + for (auto &C : FullLinkTimeOptimizationLastEPCallbacks) + C(MPM, Level); +} +void PassBuilder::invokePipelineStartEPCallbacks(ModulePassManager &MPM, + OptimizationLevel Level) { + for (auto &C : PipelineStartEPCallbacks) + C(MPM, Level); +} +void PassBuilder::invokePipelineEarlySimplificationEPCallbacks( + ModulePassManager &MPM, OptimizationLevel Level) { + for (auto &C : PipelineEarlySimplificationEPCallbacks) + C(MPM, Level); +} // Helper to add AnnotationRemarksPass. static void addAnnotationRemarksPass(ModulePassManager &MPM) { @@ -384,8 +439,7 @@ PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level, LPM2.addPass(LoopIdiomRecognizePass()); LPM2.addPass(IndVarSimplifyPass()); - for (auto &C : LateLoopOptimizationsEPCallbacks) - C(LPM2, Level); + invokeLateLoopOptimizationsEPCallbacks(LPM2, Level); LPM2.addPass(LoopDeletionPass()); @@ -403,8 +457,7 @@ PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level, /* OnlyWhenForced= */ !PTO.LoopUnrolling, PTO.ForgetAllSCEVInLoopUnroll)); - for (auto &C : LoopOptimizerEndEPCallbacks) - C(LPM2, Level); + invokeLoopOptimizerEndEPCallbacks(LPM2, Level); // We provide the opt remark emitter pass for LICM to use. We only need to do // this once as it is immutable. @@ -445,8 +498,7 @@ PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level, FPM.addPass(CoroElidePass()); - for (auto &C : ScalarOptimizerLateEPCallbacks) - C(FPM, Level); + invokeScalarOptimizerLateEPCallbacks(FPM, Level); // Finally, do an expensive DCE pass to catch all the dead code exposed by // the simplifications and basic cleanup after all the simplifications. @@ -570,8 +622,7 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, LPM2.addPass(LoopIdiomRecognizePass()); LPM2.addPass(IndVarSimplifyPass()); - for (auto &C : LateLoopOptimizationsEPCallbacks) - C(LPM2, Level); + invokeLateLoopOptimizationsEPCallbacks(LPM2, Level); LPM2.addPass(LoopDeletionPass()); @@ -589,8 +640,7 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, /* OnlyWhenForced= */ !PTO.LoopUnrolling, PTO.ForgetAllSCEVInLoopUnroll)); - for (auto &C : LoopOptimizerEndEPCallbacks) - C(LPM2, Level); + invokeLoopOptimizerEndEPCallbacks(LPM2, Level); // We provide the opt remark emitter pass for LICM to use. We only need to do // this once as it is immutable. @@ -662,8 +712,7 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, FPM.addPass(CoroElidePass()); - for (auto &C : ScalarOptimizerLateEPCallbacks) - C(FPM, Level); + invokeScalarOptimizerLateEPCallbacks(FPM, Level); FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions() .convertSwitchRangeToICmp(true) @@ -849,8 +898,7 @@ PassBuilder::buildInlinerPipeline(OptimizationLevel Level, if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3) MainCGPipeline.addPass(OpenMPOptCGSCCPass()); - for (auto &C : CGSCCOptimizerLateEPCallbacks) - C(MainCGPipeline, Level); + invokeCGSCCOptimizerLateEPCallbacks(MainCGPipeline, Level); // Add the core function simplification pipeline nested inside the // CGSCC walk. @@ -1007,8 +1055,7 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink) MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true)); - for (auto &C : PipelineEarlySimplificationEPCallbacks) - C(MPM, Level); + invokePipelineEarlySimplificationEPCallbacks(MPM, Level); // Interprocedural constant propagation now that basic cleanup has occurred // and prior to optimizing globals. @@ -1279,8 +1326,7 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level, // memory operations. MPM.addPass(RecomputeGlobalsAAPass()); - for (auto &C : OptimizerEarlyEPCallbacks) - C(MPM, Level); + invokeOptimizerEarlyEPCallbacks(MPM, Level); FunctionPassManager OptimizePM; OptimizePM.addPass(Float2IntPass()); @@ -1303,8 +1349,7 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level, // rather than on each loop in an inside-out manner, and so they are actually // function passes. - for (auto &C : VectorizerStartEPCallbacks) - C(OptimizePM, Level); + invokeVectorizerStartEPCallbacks(OptimizePM, Level); LoopPassManager LPM; // First rotate loops that may have been un-rotated by prior passes. @@ -1356,8 +1401,7 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level, MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM), PTO.EagerlyInvalidateAnalyses)); - for (auto &C : OptimizerLastEPCallbacks) - C(MPM, Level); + invokeOptimizerLastEPCallbacks(MPM, Level); // Split out cold code. Splitting is done late to avoid hiding context from // other optimizations and inadvertently regressing performance. The tradeoff @@ -1413,8 +1457,7 @@ PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level, MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); // Apply module pipeline start EP callback. - for (auto &C : PipelineStartEPCallbacks) - C(MPM, Level); + invokePipelineStartEPCallbacks(MPM, Level); const ThinOrFullLTOPhase LTOPhase = LTOPreLink ? ThinOrFullLTOPhase::FullLTOPreLink @@ -1455,8 +1498,7 @@ PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) { MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); // Apply module pipeline start EP callback. - for (auto &C : PipelineStartEPCallbacks) - C(MPM, Level); + invokePipelineStartEPCallbacks(MPM, Level); // If we are planning to perform ThinLTO later, we don't bloat the code with // unrolling/vectorization/... now. Just simplify the module as much as we @@ -1481,10 +1523,8 @@ PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) { // Handle Optimizer{Early,Last}EPCallbacks added by clang on PreLink. Actual // optimization is going to be done in PostLink stage, but clang can't add // callbacks there in case of in-process ThinLTO called by linker. - for (auto &C : OptimizerEarlyEPCallbacks) - C(MPM, Level); - for (auto &C : OptimizerLastEPCallbacks) - C(MPM, Level); + invokeOptimizerEarlyEPCallbacks(MPM, Level); + invokeOptimizerLastEPCallbacks(MPM, Level); // Emit annotation remarks. addAnnotationRemarksPass(MPM); @@ -1559,8 +1599,7 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, ModuleSummaryIndex *ExportSummary) { ModulePassManager MPM; - for (auto &C : FullLinkTimeOptimizationEarlyEPCallbacks) - C(MPM, Level); + invokeFullLinkTimeOptimizationEarlyEPCallbacks(MPM, Level); // Create a function that performs CFI checks for cross-DSO calls with targets // in the current module. @@ -1575,8 +1614,7 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, // in ICP. MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true)); - for (auto &C : FullLinkTimeOptimizationLastEPCallbacks) - C(MPM, Level); + invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level); // Emit annotation remarks. addAnnotationRemarksPass(MPM); @@ -1657,8 +1695,7 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, // pipeline). MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true)); - for (auto &C : FullLinkTimeOptimizationLastEPCallbacks) - C(MPM, Level); + invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level); // Emit annotation remarks. addAnnotationRemarksPass(MPM); @@ -1867,8 +1904,7 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, if (PTO.CallGraphProfile) MPM.addPass(CGProfilePass()); - for (auto &C : FullLinkTimeOptimizationLastEPCallbacks) - C(MPM, Level); + invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level); // Emit annotation remarks. addAnnotationRemarksPass(MPM); @@ -1898,14 +1934,12 @@ ModulePassManager PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level, /* IsCS */ false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile, PGOOpt->FS); - for (auto &C : PipelineStartEPCallbacks) - C(MPM, Level); + invokePipelineStartEPCallbacks(MPM, Level); if (PGOOpt && PGOOpt->DebugInfoForProfiling) MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass())); - for (auto &C : PipelineEarlySimplificationEPCallbacks) - C(MPM, Level); + invokePipelineEarlySimplificationEPCallbacks(MPM, Level); // Build a minimal pipeline based on the semantics required by LLVM, // which is just that always inlining occurs. Further, disable generating @@ -1923,15 +1957,13 @@ ModulePassManager PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level, if (!CGSCCOptimizerLateEPCallbacks.empty()) { CGSCCPassManager CGPM; - for (auto &C : CGSCCOptimizerLateEPCallbacks) - C(CGPM, Level); + invokeCGSCCOptimizerLateEPCallbacks(CGPM, Level); if (!CGPM.isEmpty()) MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM))); } if (!LateLoopOptimizationsEPCallbacks.empty()) { LoopPassManager LPM; - for (auto &C : LateLoopOptimizationsEPCallbacks) - C(LPM, Level); + invokeLateLoopOptimizationsEPCallbacks(LPM, Level); if (!LPM.isEmpty()) { MPM.addPass(createModuleToFunctionPassAdaptor( createFunctionToLoopPassAdaptor(std::move(LPM)))); @@ -1939,8 +1971,7 @@ ModulePassManager PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level, } if (!LoopOptimizerEndEPCallbacks.empty()) { LoopPassManager LPM; - for (auto &C : LoopOptimizerEndEPCallbacks) - C(LPM, Level); + invokeLoopOptimizerEndEPCallbacks(LPM, Level); if (!LPM.isEmpty()) { MPM.addPass(createModuleToFunctionPassAdaptor( createFunctionToLoopPassAdaptor(std::move(LPM)))); @@ -1948,19 +1979,16 @@ ModulePassManager PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level, } if (!ScalarOptimizerLateEPCallbacks.empty()) { FunctionPassManager FPM; - for (auto &C : ScalarOptimizerLateEPCallbacks) - C(FPM, Level); + invokeScalarOptimizerLateEPCallbacks(FPM, Level); if (!FPM.isEmpty()) MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } - for (auto &C : OptimizerEarlyEPCallbacks) - C(MPM, Level); + invokeOptimizerEarlyEPCallbacks(MPM, Level); if (!VectorizerStartEPCallbacks.empty()) { FunctionPassManager FPM; - for (auto &C : VectorizerStartEPCallbacks) - C(FPM, Level); + invokeVectorizerStartEPCallbacks(FPM, Level); if (!FPM.isEmpty()) MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } @@ -1974,8 +2002,7 @@ ModulePassManager PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level, CoroPM.addPass(GlobalDCEPass()); MPM.addPass(CoroConditionalWrapper(std::move(CoroPM))); - for (auto &C : OptimizerLastEPCallbacks) - C(MPM, Level); + invokeOptimizerLastEPCallbacks(MPM, Level); if (LTOPreLink) addRequiredLTOPreLinkPasses(MPM); -- 2.7.4