// Filter the candidates further.
SmallVector<StoreToLoadForwardingCandidate, 4> Candidates;
- unsigned NumForwarding = 0;
for (const StoreToLoadForwardingCandidate &Cand : StoreToLoadDependences) {
LLVM_DEBUG(dbgs() << "Candidate " << Cand);
if (!Cand.isDependenceDistanceOfOne(PSE, L))
continue;
- ++NumForwarding;
+ assert(isa<SCEVAddRecExpr>(PSE.getSCEV(Cand.Load->getPointerOperand())) &&
+ "Loading from something other than indvar?");
+ assert(
+ isa<SCEVAddRecExpr>(PSE.getSCEV(Cand.Store->getPointerOperand())) &&
+ "Storing to something other than indvar?");
+
+ Candidates.push_back(Cand);
LLVM_DEBUG(
dbgs()
- << NumForwarding
+ << Candidates.size()
<< ". Valid store-to-load forwarding across the loop backedge\n");
- Candidates.push_back(Cand);
}
if (Candidates.empty())
return false;
LV.setAliasChecks(std::move(Checks));
LV.setSCEVChecks(LAI.getPSE().getUnionPredicate());
LV.versionLoop();
+
+ // After versioning, some of the candidates' pointers could stop being
+ // SCEVAddRecs. We need to filter them out.
+ auto NoLongerGoodCandidate = [this](
+ const StoreToLoadForwardingCandidate &Cand) {
+ return !isa<SCEVAddRecExpr>(
+ PSE.getSCEV(Cand.Load->getPointerOperand())) ||
+ !isa<SCEVAddRecExpr>(
+ PSE.getSCEV(Cand.Store->getPointerOperand()));
+ };
+ llvm::erase_if(Candidates, NoLongerGoodCandidate);
}
// Next, propagate the value stored by the store to the users of the load.
"storeforward");
for (const auto &Cand : Candidates)
propagateStoredValueToLoadUsers(Cand, SEE);
- NumLoopLoadEliminted += NumForwarding;
+ NumLoopLoadEliminted += Candidates.size();
return true;
}