if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range))
return false;
- // If this ingredient's recipe is to be recorded, keep its recipe a singleton
- // to avoid having to split recipes later.
- bool IsSingleton = Ingredient2Recipe.count(I);
// Success: widen this instruction.
-
- // Use the default widening recipe. We optimize the common case where
- // consecutive instructions can be represented by a single recipe.
- if (!IsSingleton && !VPBB->empty() && LastExtensibleRecipe == &VPBB->back() &&
- LastExtensibleRecipe->appendInstruction(I))
- return true;
-
- VPWidenRecipe *WidenRecipe = new VPWidenRecipe(I);
- if (!IsSingleton)
- LastExtensibleRecipe = WidenRecipe;
+ VPWidenRecipe *WidenRecipe = new VPWidenRecipe(*I);
setRecipe(I, WidenRecipe);
VPBB->appendRecipe(WidenRecipe);
return true;
}
void VPWidenRecipe::execute(VPTransformState &State) {
- for (auto &Instr : make_range(Begin, End))
- State.ILV->widenInstruction(Instr);
+ State.ILV->widenInstruction(Ingredient);
}
void VPWidenGEPRecipe::execute(VPTransformState &State) {
// VPlan-VPlan transformations support: Hold a mapping from ingredients to
// their recipe. To save on memory, only do so for selected ingredients,
- // marked by having a nullptr entry in this map. If those ingredients get a
- // VPWidenRecipe, also avoid compressing other ingredients into it to avoid
- // having to split such recipes later.
+ // marked by having a nullptr entry in this map.
DenseMap<Instruction *, VPRecipeBase *> Ingredient2Recipe;
- VPWidenRecipe *LastExtensibleRecipe = nullptr;
/// Set the recipe created for given ingredient. This operation is a no-op for
/// ingredients that were not marked using a nullptr entry in the map.
void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << " +\n" << Indent << "\"WIDEN\\l\"";
- for (auto &Instr : make_range(Begin, End))
- O << " +\n" << Indent << "\" " << VPlanIngredient(&Instr) << "\\l\"";
+ O << "\" " << VPlanIngredient(&Ingredient) << "\\l\"";
}
void VPWidenIntOrFpInductionRecipe::print(raw_ostream &O, const Twine &Indent,
}
};
-/// VPWidenRecipe is a recipe for producing a copy of vector type for each
-/// Instruction in its ingredients independently, in order. This recipe covers
-/// most of the traditional vectorization cases where each ingredient transforms
-/// into a vectorized version of itself.
+/// VPWidenRecipe is a recipe for producing a copy of vector type its
+/// ingredient. This recipe covers most of the traditional vectorization cases
+/// where each ingredient transforms into a vectorized version of itself.
class VPWidenRecipe : public VPRecipeBase {
private:
- /// Hold the ingredients by pointing to their original BasicBlock location.
- BasicBlock::iterator Begin;
- BasicBlock::iterator End;
+ /// Hold the instruction to be widened.
+ Instruction &Ingredient;
public:
- VPWidenRecipe(Instruction *I) : VPRecipeBase(VPWidenSC) {
- End = I->getIterator();
- Begin = End++;
- }
+ VPWidenRecipe(Instruction &I) : VPRecipeBase(VPWidenSC), Ingredient(I) {}
~VPWidenRecipe() override = default;
/// Produce widened copies of all Ingredients.
void execute(VPTransformState &State) override;
- /// Augment the recipe to include Instr, if it lies at its End.
- bool appendInstruction(Instruction *Instr) {
- if (End != Instr->getIterator())
- return false;
- End++;
- return true;
- }
-
/// Print the recipe.
void print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const override;
continue;
VPBasicBlock *VPBB = Base->getEntryBasicBlock();
- VPRecipeBase *LastRecipe = nullptr;
// Introduce each ingredient into VPlan.
for (auto I = VPBB->begin(), E = VPBB->end(); I != E;) {
VPRecipeBase *Ingredient = &*I++;
NewRecipe = new VPWidenPHIRecipe(Phi);
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
NewRecipe = new VPWidenGEPRecipe(GEP, OrigLoop);
- } else {
- // If the last recipe is a VPWidenRecipe, add Inst to it instead of
- // creating a new recipe.
- if (VPWidenRecipe *WidenRecipe =
- dyn_cast_or_null<VPWidenRecipe>(LastRecipe)) {
- WidenRecipe->appendInstruction(Inst);
- Ingredient->eraseFromParent();
- continue;
- }
- NewRecipe = new VPWidenRecipe(Inst);
- }
+ } else
+ NewRecipe = new VPWidenRecipe(*Inst);
NewRecipe->insertBefore(Ingredient);
- LastRecipe = NewRecipe;
Ingredient->eraseFromParent();
}
}
EXPECT_EQ(1u, Entry->getNumSuccessors());
VPBasicBlock *VecBB = Entry->getSingleSuccessor()->getEntryBasicBlock();
- EXPECT_EQ(6u, VecBB->size());
+ EXPECT_EQ(7u, VecBB->size());
EXPECT_EQ(2u, VecBB->getNumPredecessors());
EXPECT_EQ(2u, VecBB->getNumSuccessors());
auto Iter = VecBB->begin();
- auto *Phi = dyn_cast<VPWidenPHIRecipe>(&*Iter++);
- EXPECT_NE(nullptr, Phi);
-
- auto *Idx = dyn_cast<VPWidenGEPRecipe>(&*Iter++);
- EXPECT_NE(nullptr, Idx);
-
- auto *Load = dyn_cast<VPWidenMemoryInstructionRecipe>(&*Iter++);
- EXPECT_NE(nullptr, Load);
-
- auto *Add = dyn_cast<VPWidenRecipe>(&*Iter++);
- EXPECT_NE(nullptr, Add);
-
- auto *Store = dyn_cast<VPWidenMemoryInstructionRecipe>(&*Iter++);
- EXPECT_NE(nullptr, Store);
-
- auto *LastWiden = dyn_cast<VPWidenRecipe>(&*Iter++);
- EXPECT_NE(nullptr, LastWiden);
+ EXPECT_NE(nullptr, dyn_cast<VPWidenPHIRecipe>(&*Iter++));
+ EXPECT_NE(nullptr, dyn_cast<VPWidenGEPRecipe>(&*Iter++));
+ EXPECT_NE(nullptr, dyn_cast<VPWidenMemoryInstructionRecipe>(&*Iter++));
+ EXPECT_NE(nullptr, dyn_cast<VPWidenRecipe>(&*Iter++));
+ EXPECT_NE(nullptr, dyn_cast<VPWidenMemoryInstructionRecipe>(&*Iter++));
+ EXPECT_NE(nullptr, dyn_cast<VPWidenRecipe>(&*Iter++));
+ EXPECT_NE(nullptr, dyn_cast<VPWidenRecipe>(&*Iter++));
EXPECT_EQ(VecBB->end(), Iter);
}