return BlockMaskCache[BB] = BlockMask;
}
-VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(Instruction *I, VFRange &Range,
+VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(Instruction *I,
+ ArrayRef<VPValue *> Operands,
+ VFRange &Range,
VPlanPtr &Plan) {
assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
"Must be called with either a load or store");
if (Legal->isMaskRequired(I))
Mask = createBlockInMask(I->getParent(), Plan);
- VPValue *Addr = Plan->getOrAddVPValue(getLoadStorePointerOperand(I));
if (LoadInst *Load = dyn_cast<LoadInst>(I))
- return new VPWidenMemoryInstructionRecipe(*Load, Addr, Mask);
+ return new VPWidenMemoryInstructionRecipe(*Load, Operands[0], Mask);
StoreInst *Store = cast<StoreInst>(I);
- VPValue *StoredValue = Plan->getOrAddVPValue(Store->getValueOperand());
- return new VPWidenMemoryInstructionRecipe(*Store, Addr, StoredValue, Mask);
+ return new VPWidenMemoryInstructionRecipe(*Store, Operands[1], Operands[0],
+ Mask);
}
VPWidenIntOrFpInductionRecipe *
-VPRecipeBuilder::tryToOptimizeInductionPHI(PHINode *Phi, VPlan &Plan) const {
+VPRecipeBuilder::tryToOptimizeInductionPHI(PHINode *Phi,
+ ArrayRef<VPValue *> Operands) const {
// Check if this is an integer or fp induction. If so, build the recipe that
// produces its scalar and vector values.
InductionDescriptor II = Legal->getInductionVars().lookup(Phi);
if (II.getKind() == InductionDescriptor::IK_IntInduction ||
II.getKind() == InductionDescriptor::IK_FpInduction) {
- VPValue *Start = Plan.getOrAddVPValue(II.getStartValue());
+ assert(II.getStartValue() ==
+ Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()));
const SmallVectorImpl<Instruction *> &Casts = II.getCastInsts();
return new VPWidenIntOrFpInductionRecipe(
- Phi, Start, Casts.empty() ? nullptr : Casts.front());
+ Phi, Operands[0], Casts.empty() ? nullptr : Casts.front());
}
return nullptr;
}
-VPWidenIntOrFpInductionRecipe *
-VPRecipeBuilder::tryToOptimizeInductionTruncate(TruncInst *I, VFRange &Range,
- VPlan &Plan) const {
+VPWidenIntOrFpInductionRecipe *VPRecipeBuilder::tryToOptimizeInductionTruncate(
+ TruncInst *I, ArrayRef<VPValue *> Operands, VFRange &Range,
+ VPlan &Plan) const {
// Optimize the special case where the source is a constant integer
// induction variable. Notice that we can only optimize the 'trunc' case
// because (a) FP conversions lose precision, (b) sext/zext may wrap, and
return nullptr;
}
-VPRecipeOrVPValueTy VPRecipeBuilder::tryToBlend(PHINode *Phi, VPlanPtr &Plan) {
+VPRecipeOrVPValueTy VPRecipeBuilder::tryToBlend(PHINode *Phi,
+ ArrayRef<VPValue *> Operands,
+ VPlanPtr &Plan) {
// If all incoming values are equal, the incoming VPValue can be used directly
// instead of creating a new VPBlendRecipe.
- Value *FirstIncoming = Phi->getIncomingValue(0);
- if (all_of(Phi->incoming_values(), [FirstIncoming](const Value *Inc) {
+ VPValue *FirstIncoming = Operands[0];
+ if (all_of(Operands, [FirstIncoming](const VPValue *Inc) {
return FirstIncoming == Inc;
})) {
- return Plan->getOrAddVPValue(Phi->getIncomingValue(0));
+ return Operands[0];
}
// We know that all PHIs in non-header blocks are converted into selects, so
// builder. At this point we generate the predication tree. There may be
// duplications since this is a simple recursive scan, but future
// optimizations will clean it up.
- SmallVector<VPValue *, 2> Operands;
+ SmallVector<VPValue *, 2> OperandsWithMask;
unsigned NumIncoming = Phi->getNumIncomingValues();
for (unsigned In = 0; In < NumIncoming; In++) {
createEdgeMask(Phi->getIncomingBlock(In), Phi->getParent(), Plan);
assert((EdgeMask || NumIncoming == 1) &&
"Multiple predecessors with one having a full mask");
- Operands.push_back(Plan->getOrAddVPValue(Phi->getIncomingValue(In)));
+ OperandsWithMask.push_back(Operands[In]);
if (EdgeMask)
- Operands.push_back(EdgeMask);
+ OperandsWithMask.push_back(EdgeMask);
}
- return toVPRecipeResult(new VPBlendRecipe(Phi, Operands));
+ return toVPRecipeResult(new VPBlendRecipe(Phi, OperandsWithMask));
}
-VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI, VFRange &Range,
- VPlan &Plan) const {
+VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
+ ArrayRef<VPValue *> Operands,
+ VFRange &Range) const {
bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(
[this, CI](ElementCount VF) {
if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range))
return nullptr;
- return new VPWidenCallRecipe(*CI, Plan.mapToVPValues(CI->arg_operands()));
+ ArrayRef<VPValue *> Ops = Operands.take_front(CI->getNumArgOperands());
+ return new VPWidenCallRecipe(*CI, make_range(Ops.begin(), Ops.end()));
}
bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const {
Range);
}
-VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VPlan &Plan) const {
+VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
+ ArrayRef<VPValue *> Operands) const {
auto IsVectorizableOpcode = [](unsigned Opcode) {
switch (Opcode) {
case Instruction::Add:
return nullptr;
// Success: widen this instruction.
- return new VPWidenRecipe(*I, Plan.mapToVPValues(I->operands()));
+ return new VPWidenRecipe(*I, make_range(Operands.begin(), Operands.end()));
}
VPBasicBlock *VPRecipeBuilder::handleReplication(
return Region;
}
-VPRecipeOrVPValueTy VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
- VFRange &Range,
- VPlanPtr &Plan) {
+VPRecipeOrVPValueTy
+VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
+ ArrayRef<VPValue *> Operands,
+ VFRange &Range, VPlanPtr &Plan) {
// First, check for specific widening recipes that deal with calls, memory
// operations, inductions and Phi nodes.
if (auto *CI = dyn_cast<CallInst>(Instr))
- return toVPRecipeResult(tryToWidenCall(CI, Range, *Plan));
+ return toVPRecipeResult(tryToWidenCall(CI, Operands, Range));
if (isa<LoadInst>(Instr) || isa<StoreInst>(Instr))
- return toVPRecipeResult(tryToWidenMemory(Instr, Range, Plan));
+ return toVPRecipeResult(tryToWidenMemory(Instr, Operands, Range, Plan));
VPRecipeBase *Recipe;
if (auto Phi = dyn_cast<PHINode>(Instr)) {
if (Phi->getParent() != OrigLoop->getHeader())
- return tryToBlend(Phi, Plan);
- if ((Recipe = tryToOptimizeInductionPHI(Phi, *Plan)))
+ return tryToBlend(Phi, Operands, Plan);
+ if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands)))
return toVPRecipeResult(Recipe);
if (Legal->isReductionVariable(Phi)) {
RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi];
- VPValue *StartV =
- Plan->getOrAddVPValue(RdxDesc.getRecurrenceStartValue());
+ assert(RdxDesc.getRecurrenceStartValue() ==
+ Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()));
+ VPValue *StartV = Operands[0];
return toVPRecipeResult(new VPWidenPHIRecipe(Phi, RdxDesc, *StartV));
}
return toVPRecipeResult(new VPWidenPHIRecipe(Phi));
}
- if (isa<TruncInst>(Instr) && (Recipe = tryToOptimizeInductionTruncate(
- cast<TruncInst>(Instr), Range, *Plan)))
+ if (isa<TruncInst>(Instr) &&
+ (Recipe = tryToOptimizeInductionTruncate(cast<TruncInst>(Instr), Operands,
+ Range, *Plan)))
return toVPRecipeResult(Recipe);
if (!shouldWiden(Instr, Range))
if (auto GEP = dyn_cast<GetElementPtrInst>(Instr))
return toVPRecipeResult(new VPWidenGEPRecipe(
- GEP, Plan->mapToVPValues(GEP->operands()), OrigLoop));
+ GEP, make_range(Operands.begin(), Operands.end()), OrigLoop));
if (auto *SI = dyn_cast<SelectInst>(Instr)) {
bool InvariantCond =
PSE.getSE()->isLoopInvariant(PSE.getSCEV(SI->getOperand(0)), OrigLoop);
return toVPRecipeResult(new VPWidenSelectRecipe(
- *SI, Plan->mapToVPValues(SI->operands()), InvariantCond));
+ *SI, make_range(Operands.begin(), Operands.end()), InvariantCond));
}
- return toVPRecipeResult(tryToWiden(Instr, *Plan));
+ return toVPRecipeResult(tryToWiden(Instr, Operands));
}
void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
if (isa<BranchInst>(Instr) || DeadInstructions.count(Instr))
continue;
- if (auto RecipeOrValue =
- RecipeBuilder.tryToCreateWidenRecipe(Instr, Range, Plan)) {
+ SmallVector<VPValue *, 4> Operands;
+ auto *Phi = dyn_cast<PHINode>(Instr);
+ if (Phi && Phi->getParent() == OrigLoop->getHeader()) {
+ Operands.push_back(Plan->getOrAddVPValue(
+ Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader())));
+ } else {
+ auto OpRange = Plan->mapToVPValues(Instr->operands());
+ Operands = {OpRange.begin(), OpRange.end()};
+ }
+ if (auto RecipeOrValue = RecipeBuilder.tryToCreateWidenRecipe(
+ Instr, Operands, Range, Plan)) {
// If Instr can be simplified to an existing VPValue, use it.
if (RecipeOrValue.is<VPValue *>()) {
Plan->addVPValue(Instr, RecipeOrValue.get<VPValue *>());
/// Check if the load or store instruction \p I should widened for \p
/// Range.Start and potentially masked. Such instructions are handled by a
/// recipe that takes an additional VPInstruction for the mask.
- VPRecipeBase *tryToWidenMemory(Instruction *I, VFRange &Range,
- VPlanPtr &Plan);
+ VPRecipeBase *tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
+ VFRange &Range, VPlanPtr &Plan);
/// Check if an induction recipe should be constructed for \I. If so build and
/// return it. If not, return null.
- VPWidenIntOrFpInductionRecipe *tryToOptimizeInductionPHI(PHINode *Phi,
- VPlan &Plan) const;
+ VPWidenIntOrFpInductionRecipe *
+ tryToOptimizeInductionPHI(PHINode *Phi, ArrayRef<VPValue *> Operands) const;
/// Optimize the special case where the operand of \p I is a constant integer
/// induction variable.
VPWidenIntOrFpInductionRecipe *
- tryToOptimizeInductionTruncate(TruncInst *I, VFRange &Range,
- VPlan &Plan) const;
+ tryToOptimizeInductionTruncate(TruncInst *I, ArrayRef<VPValue *> Operands,
+ VFRange &Range, VPlan &Plan) const;
/// Handle non-loop phi nodes. Return a VPValue, if all incoming values match
/// or a new VPBlendRecipe otherwise. Currently all such phi nodes are turned
/// into a sequence of select instructions as the vectorizer currently
/// performs full if-conversion.
- VPRecipeOrVPValueTy tryToBlend(PHINode *Phi, VPlanPtr &Plan);
+ VPRecipeOrVPValueTy tryToBlend(PHINode *Phi, ArrayRef<VPValue *> Operands,
+ VPlanPtr &Plan);
/// Handle call instructions. If \p CI can be widened for \p Range.Start,
/// return a new VPWidenCallRecipe. Range.End may be decreased to ensure same
/// decision from \p Range.Start to \p Range.End.
- VPWidenCallRecipe *tryToWidenCall(CallInst *CI, VFRange &Range,
- VPlan &Plan) const;
+ VPWidenCallRecipe *tryToWidenCall(CallInst *CI, ArrayRef<VPValue *> Operands,
+ VFRange &Range) const;
/// Check if \p I has an opcode that can be widened and return a VPWidenRecipe
/// if it can. The function should only be called if the cost-model indicates
/// that widening should be performed.
- VPWidenRecipe *tryToWiden(Instruction *I, VPlan &Plan) const;
+ VPWidenRecipe *tryToWiden(Instruction *I, ArrayRef<VPValue *> Operands) const;
/// Return a VPRecipeOrValueTy with VPRecipeBase * being set. This can be used to force the use as VPRecipeBase* for recipe sub-types that also inherit from VPValue.
VPRecipeOrVPValueTy toVPRecipeResult(VPRecipeBase *R) const { return R; }
/// create for \p I withing the given VF \p Range. If an existing VPValue can
/// be used or if a recipe can be created, return it. Otherwise return a
/// VPRecipeOrVPValueTy with nullptr.
- VPRecipeOrVPValueTy tryToCreateWidenRecipe(Instruction *Instr, VFRange &Range,
- VPlanPtr &Plan);
+ VPRecipeOrVPValueTy tryToCreateWidenRecipe(Instruction *Instr,
+ ArrayRef<VPValue *> Operands,
+ VFRange &Range, VPlanPtr &Plan);
/// Set the recipe created for given ingredient. This operation is a no-op for
/// ingredients that were not marked using a nullptr entry in the map.
projects / platform / upstream / llvm.git / commitdiff