bool formLCSSARecursively(Loop &L, DominatorTree &DT, LoopInfo *LI,
ScalarEvolution *SE);
+struct SinkAndHoistLICMFlags {
+ bool NoOfMemAccTooLarge;
+ unsigned LicmMssaOptCounter;
+ unsigned LicmMssaOptCap;
+ unsigned LicmMssaNoAccForPromotionCap;
+};
+
/// Walk the specified region of the CFG (defined by all blocks
/// dominated by the specified block, and that are in the current loop) in
/// reverse depth first order w.r.t the DominatorTree. This allows us to visit
bool sinkRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
TargetLibraryInfo *, TargetTransformInfo *, Loop *,
AliasSetTracker *, MemorySSAUpdater *, ICFLoopSafetyInfo *,
- bool, int &, OptimizationRemarkEmitter *);
+ SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *);
/// Walk the specified region of the CFG (defined by all blocks
/// dominated by the specified block, and that are in the current loop) in depth
/// ORE. It returns changed status.
bool hoistRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
TargetLibraryInfo *, Loop *, AliasSetTracker *,
- MemorySSAUpdater *, ICFLoopSafetyInfo *, bool, int &,
- OptimizationRemarkEmitter *);
+ MemorySSAUpdater *, ICFLoopSafetyInfo *,
+ SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *);
/// This function deletes dead loops. The caller of this function needs to
/// guarantee that the loop is infact dead.
bool canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
Loop *CurLoop, AliasSetTracker *CurAST,
MemorySSAUpdater *MSSAU, bool TargetExecutesOncePerLoop,
- bool NoOfMemAccessesTooLarge,
- int *LicmMssaOptCap = nullptr,
+ SinkAndHoistLICMFlags *LICMFlags = nullptr,
OptimizationRemarkEmitter *ORE = nullptr);
/// Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Scalar/GVN.h"
#include "llvm/Transforms/Scalar/InstSimplifyPass.h"
+#include "llvm/Transforms/Scalar/LICM.h"
#include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
#include "llvm/Transforms/Utils.h"
#include "llvm/Transforms/Vectorize.h"
// the LoopVectorize pass, to be consistent with the new pass manager.
RerollLoops = RunLoopRerolling;
NewGVN = RunNewGVN;
+ LicmMssaOptCap = SetLicmMssaOptCap;
+ LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
DisableGVNLoadPRE = false;
ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
VerifyInput = false;
}
// Rotate Loop - disable header duplication at -Oz
MPM.add(createLoopRotatePass(SizeLevel == 2 ? 0 : -1));
- MPM.add(createLICMPass()); // Hoist loop invariants
+ MPM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
if (EnableSimpleLoopUnswitch)
MPM.add(createSimpleLoopUnswitchLegacyPass());
else
MPM.add(createJumpThreadingPass()); // Thread jumps
MPM.add(createCorrelatedValuePropagationPass());
MPM.add(createDeadStoreEliminationPass()); // Delete dead stores
- MPM.add(createLICMPass());
+ MPM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
addExtensionsToPM(EP_ScalarOptimizerLate, MPM);
// later might get benefit of no-alias assumption in clone loop.
if (UseLoopVersioningLICM) {
MPM.add(createLoopVersioningLICMPass()); // Do LoopVersioningLICM
- MPM.add(createLICMPass()); // Hoist loop invariants
+ MPM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
}
// We add a fresh GlobalsModRef run at this point. This is particularly
MPM.add(createEarlyCSEPass());
MPM.add(createCorrelatedValuePropagationPass());
addInstructionCombiningPass(MPM);
- MPM.add(createLICMPass());
+ MPM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
MPM.add(createLoopUnswitchPass(SizeLevel || OptLevel < 3, DivergentTarget));
MPM.add(createCFGSimplificationPass());
addInstructionCombiningPass(MPM);
// unrolled loop is a inner loop, then the prologue will be inside the
// outer loop. LICM pass can help to promote the runtime check out if the
// checked value is loop invariant.
- MPM.add(createLICMPass());
+ MPM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
}
MPM.add(createWarnMissedTransformationsPass());
PM.add(createPostOrderFunctionAttrsLegacyPass()); // Add nocapture.
PM.add(createGlobalsAAWrapperPass()); // IP alias analysis.
- PM.add(createLICMPass()); // Hoist loop invariants.
+ PM.add(createLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap));
PM.add(createMergedLoadStoreMotionPass()); // Merge ld/st in diamonds.
PM.add(NewGVN ? createNewGVNPass()
: createGVNPass(DisableGVNLoadPRE)); // Remove redundancies.
// which may not be precise, since optimizeUses is capped. The result is
// correct, but we may not get as "far up" as possible to get which access is
// clobbering the one queried.
-static cl::opt<int> LicmMssaOptCap(
+cl::opt<unsigned> llvm::SetLicmMssaOptCap(
"licm-mssa-optimization-cap", cl::init(100), cl::Hidden,
cl::desc("Enable imprecision in LICM in pathological cases, in exchange "
"for faster compile. Caps the MemorySSA clobbering calls."));
// Experimentally, memory promotion carries less importance than sinking and
// hoisting. Limit when we do promotion when using MemorySSA, in order to save
// compile time.
-static cl::opt<unsigned> AccessCapForMSSAPromotion(
- "max-acc-licm-promotion", cl::init(250), cl::Hidden,
+cl::opt<unsigned> llvm::SetLicmMssaNoAccForPromotionCap(
+ "licm-mssa-max-acc-promotion", cl::init(250), cl::Hidden,
cl::desc("[LICM & MemorySSA] When MSSA in LICM is disabled, this has no "
"effect. When MSSA in LICM is enabled, then this is the maximum "
"number of accesses allowed to be present in a loop in order to "
AliasAnalysis *AA);
static bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
Loop *CurLoop,
- int &LicmMssaOptCounter);
+ SinkAndHoistLICMFlags &Flags);
static Instruction *CloneInstructionInExitBlock(
Instruction &I, BasicBlock &ExitBlock, PHINode &PN, const LoopInfo *LI,
const LoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU);
OptimizationRemarkEmitter *ORE, bool DeleteAST);
ASTrackerMapTy &getLoopToAliasSetMap() { return LoopToAliasSetMap; }
+ LoopInvariantCodeMotion(unsigned LicmMssaOptCap,
+ unsigned LicmMssaNoAccForPromotionCap)
+ : LicmMssaOptCap(LicmMssaOptCap),
+ LicmMssaNoAccForPromotionCap(LicmMssaNoAccForPromotionCap) {}
private:
ASTrackerMapTy LoopToAliasSetMap;
+ unsigned LicmMssaOptCap;
+ unsigned LicmMssaNoAccForPromotionCap;
std::unique_ptr<AliasSetTracker>
collectAliasInfoForLoop(Loop *L, LoopInfo *LI, AliasAnalysis *AA);
struct LegacyLICMPass : public LoopPass {
static char ID; // Pass identification, replacement for typeid
- LegacyLICMPass() : LoopPass(ID) {
+ LegacyLICMPass(
+ unsigned LicmMssaOptCap = SetLicmMssaOptCap,
+ unsigned LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap)
+ : LoopPass(ID), LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap) {
initializeLegacyLICMPassPass(*PassRegistry::getPassRegistry());
}
report_fatal_error("LICM: OptimizationRemarkEmitterAnalysis not "
"cached at a higher level");
- LoopInvariantCodeMotion LICM;
+ LoopInvariantCodeMotion LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
if (!LICM.runOnLoop(&L, &AR.AA, &AR.LI, &AR.DT, &AR.TLI, &AR.TTI, &AR.SE,
AR.MSSA, ORE, true))
return PreservedAnalyses::all();
false)
Pass *llvm::createLICMPass() { return new LegacyLICMPass(); }
+Pass *llvm::createLICMPass(unsigned LicmMssaOptCap,
+ unsigned LicmMssaNoAccForPromotionCap) {
+ return new LegacyLICMPass(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
+}
/// Hoist expressions out of the specified loop. Note, alias info for inner
/// loop is not preserved so it is not a good idea to run LICM multiple
std::unique_ptr<AliasSetTracker> CurAST;
std::unique_ptr<MemorySSAUpdater> MSSAU;
bool NoOfMemAccTooLarge = false;
- int LicmMssaOptCounter = 0;
+ unsigned LicmMssaOptCounter = 0;
if (!MSSA) {
LLVM_DEBUG(dbgs() << "LICM: Using Alias Set Tracker.\n");
for (const auto &MA : *Accesses) {
(void)MA;
AccessCapCount++;
- if (AccessCapCount > AccessCapForMSSAPromotion) {
+ if (AccessCapCount > LicmMssaNoAccForPromotionCap) {
NoOfMemAccTooLarge = true;
break;
}
// that we are guaranteed to see definitions before we see uses. This allows
// us to sink instructions in one pass, without iteration. After sinking
// instructions, we perform another pass to hoist them out of the loop.
- //
+ SinkAndHoistLICMFlags Flags = {NoOfMemAccTooLarge, LicmMssaOptCounter,
+ LicmMssaOptCap, LicmMssaNoAccForPromotionCap};
if (L->hasDedicatedExits())
Changed |= sinkRegion(DT->getNode(L->getHeader()), AA, LI, DT, TLI, TTI, L,
- CurAST.get(), MSSAU.get(), &SafetyInfo,
- NoOfMemAccTooLarge, LicmMssaOptCounter, ORE);
+ CurAST.get(), MSSAU.get(), &SafetyInfo, Flags, ORE);
if (Preheader)
Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, TLI, L,
- CurAST.get(), MSSAU.get(), &SafetyInfo,
- NoOfMemAccTooLarge, LicmMssaOptCounter, ORE);
+ CurAST.get(), MSSAU.get(), &SafetyInfo, Flags, ORE);
// Now that all loop invariants have been removed from the loop, promote any
// memory references to scalars that we can.
DominatorTree *DT, TargetLibraryInfo *TLI,
TargetTransformInfo *TTI, Loop *CurLoop,
AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
- ICFLoopSafetyInfo *SafetyInfo, bool NoOfMemAccTooLarge,
- int &LicmMssaOptCounter, OptimizationRemarkEmitter *ORE) {
+ ICFLoopSafetyInfo *SafetyInfo,
+ SinkAndHoistLICMFlags &Flags,
+ OptimizationRemarkEmitter *ORE) {
// Verify inputs.
assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&
//
bool FreeInLoop = false;
if (isNotUsedOrFreeInLoop(I, CurLoop, SafetyInfo, TTI, FreeInLoop) &&
- canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true,
- NoOfMemAccTooLarge, &LicmMssaOptCounter, ORE) &&
+ canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true, &Flags,
+ ORE) &&
!I.mayHaveSideEffects()) {
if (sink(I, LI, DT, CurLoop, SafetyInfo, MSSAU, ORE)) {
if (!FreeInLoop) {
bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
DominatorTree *DT, TargetLibraryInfo *TLI, Loop *CurLoop,
AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
- ICFLoopSafetyInfo *SafetyInfo, bool NoOfMemAccTooLarge,
- int &LicmMssaOptCounter,
+ ICFLoopSafetyInfo *SafetyInfo,
+ SinkAndHoistLICMFlags &Flags,
OptimizationRemarkEmitter *ORE) {
// Verify inputs.
assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&
// and we have accurately duplicated the control flow from the loop header
// to that block.
if (CurLoop->hasLoopInvariantOperands(&I) &&
- canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true,
- NoOfMemAccTooLarge, &LicmMssaOptCounter, ORE) &&
+ canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true, &Flags,
+ ORE) &&
isSafeToExecuteUnconditionally(
I, DT, CurLoop, SafetyInfo, ORE,
CurLoop->getLoopPreheader()->getTerminator())) {
Loop *CurLoop, AliasSetTracker *CurAST,
MemorySSAUpdater *MSSAU,
bool TargetExecutesOncePerLoop,
- bool NoOfMemAccTooLarge, int *LicmMssaOptCounter,
+ SinkAndHoistLICMFlags *Flags,
OptimizationRemarkEmitter *ORE) {
// If we don't understand the instruction, bail early.
if (!isHoistableAndSinkableInst(I))
MemorySSA *MSSA = MSSAU ? MSSAU->getMemorySSA() : nullptr;
if (MSSA)
- assert(LicmMssaOptCounter != nullptr && "Counter cannot be null.");
+ assert(Flags != nullptr && "Flags cannot be null.");
// Loads have extra constraints we have to verify before we can hoist them.
if (LoadInst *LI = dyn_cast<LoadInst>(&I)) {
CurLoop, AA);
else
Invalidated = pointerInvalidatedByLoopWithMSSA(
- MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(LI)), CurLoop,
- *LicmMssaOptCounter);
+ MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(LI)), CurLoop, *Flags);
// Check loop-invariant address because this may also be a sinkable load
// whose address is not necessarily loop-invariant.
if (ORE && Invalidated && CurLoop->isLoopInvariant(LI->getPointerOperand()))
else
Invalidated = pointerInvalidatedByLoopWithMSSA(
MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(CI)), CurLoop,
- *LicmMssaOptCounter);
+ *Flags);
if (Invalidated)
return false;
}
return true;
// If there are more accesses than the Promotion cap, give up, we're not
// walking a list that long.
- if (NoOfMemAccTooLarge)
+ if (Flags->NoOfMemAccTooLarge)
return false;
// Check store only if there's still "quota" to check clobber.
- if (*LicmMssaOptCounter >= LicmMssaOptCap)
+ if (Flags->LicmMssaOptCounter >= Flags->LicmMssaOptCap)
return false;
// If there are interfering Uses (i.e. their defining access is in the
// loop), or ordered loads (stored as Defs!), don't move this store.
}
auto *Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(SI);
- (*LicmMssaOptCounter)++;
+ Flags->LicmMssaOptCounter++;
// If there are no clobbering Defs in the loop, store is safe to hoist.
return MSSA->isLiveOnEntryDef(Source) ||
!CurLoop->contains(Source->getBlock());
static bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
Loop *CurLoop,
- int &LicmMssaOptCounter) {
+ SinkAndHoistLICMFlags &Flags) {
MemoryAccess *Source;
- // See declaration of LicmMssaOptCap for usage details.
- if (LicmMssaOptCounter >= LicmMssaOptCap)
+ // See declaration of SetLicmMssaOptCap for usage details.
+ if (Flags.LicmMssaOptCounter >= Flags.LicmMssaOptCap)
Source = MU->getDefiningAccess();
else {
Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(MU);
- LicmMssaOptCounter++;
+ Flags.LicmMssaOptCounter++;
}
return !MSSA->isLiveOnEntryDef(Source) &&
CurLoop->contains(Source->getBlock());