/// values in the vectorized loop.
void vectorizeInterleaveGroup(const InterleaveGroup<Instruction> *Group,
VPTransformState &State, VPValue *Addr,
+ ArrayRef<VPValue *> StoredValues,
VPValue *BlockInMask = nullptr);
/// Vectorize Load and Store instructions with the base address given in \p
// store <12 x i32> %interleaved.vec ; Write 4 tuples of R,G,B
void InnerLoopVectorizer::vectorizeInterleaveGroup(
const InterleaveGroup<Instruction> *Group, VPTransformState &State,
- VPValue *Addr, VPValue *BlockInMask) {
+ VPValue *Addr, ArrayRef<VPValue *> StoredValues, VPValue *BlockInMask) {
Instruction *Instr = Group->getInsertPos();
const DataLayout &DL = Instr->getModule()->getDataLayout();
Instruction *Member = Group->getMember(i);
assert(Member && "Fail to get a member from an interleaved store group");
- Value *StoredVec = getOrCreateVectorValue(
- cast<StoreInst>(Member)->getValueOperand(), Part);
+ Value *StoredVec = State.get(StoredValues[i], Part);
+
if (Group->isReverse())
StoredVec = reverseVector(StoredVec);
for (auto IG : InterleaveGroups) {
auto *Recipe = cast<VPWidenMemoryInstructionRecipe>(
RecipeBuilder.getRecipe(IG->getInsertPos()));
- (new VPInterleaveRecipe(IG, Recipe->getAddr(), Recipe->getMask()))
+ SmallVector<VPValue *, 4> StoredValues;
+ for (unsigned i = 0; i < IG->getFactor(); ++i)
+ if (auto *SI = dyn_cast_or_null<StoreInst>(IG->getMember(i)))
+ StoredValues.push_back(Plan->getOrAddVPValue(SI->getOperand(0)));
+
+ (new VPInterleaveRecipe(IG, Recipe->getAddr(), StoredValues,
+ Recipe->getMask()))
->insertBefore(Recipe);
for (unsigned i = 0; i < IG->getFactor(); ++i)
void VPInterleaveRecipe::execute(VPTransformState &State) {
assert(!State.Instance && "Interleave group being replicated.");
- State.ILV->vectorizeInterleaveGroup(IG, State, getAddr(), getMask());
+ State.ILV->vectorizeInterleaveGroup(IG, State, getAddr(), getStoredValues(),
+ getMask());
}
void VPReductionRecipe::execute(VPTransformState &State) {
};
/// VPInterleaveRecipe is a recipe for transforming an interleave group of load
-/// or stores into one wide load/store and shuffles.
+/// or stores into one wide load/store and shuffles. The first operand of a
+/// VPInterleave recipe is the address, followed by the stored values, followed
+/// by an optional mask.
class VPInterleaveRecipe : public VPRecipeBase, public VPUser {
const InterleaveGroup<Instruction> *IG;
+ bool HasMask = false;
+
public:
VPInterleaveRecipe(const InterleaveGroup<Instruction> *IG, VPValue *Addr,
- VPValue *Mask)
+ ArrayRef<VPValue *> StoredValues, VPValue *Mask)
: VPRecipeBase(VPInterleaveSC), VPUser({Addr}), IG(IG) {
- if (Mask)
+ for (auto *SV : StoredValues)
+ addOperand(SV);
+ if (Mask) {
+ HasMask = true;
addOperand(Mask);
+ }
}
~VPInterleaveRecipe() override = default;
/// by a nullptr.
VPValue *getMask() const {
// Mask is optional and therefore the last, currently 2nd operand.
- return getNumOperands() == 2 ? getOperand(1) : nullptr;
+ return HasMask ? getOperand(getNumOperands() - 1) : nullptr;
+ }
+
+ /// Return the VPValues stored by this interleave group. If it is a load
+ /// interleave group, return an empty ArrayRef.
+ ArrayRef<VPValue *> getStoredValues() const {
+ // The first operand is the address, followed by the stored values, followed
+ // by an optional mask.
+ return ArrayRef<VPValue *>(op_begin(), getNumOperands())
+ .slice(1, getNumOperands() - (HasMask ? 2 : 1));
}
/// Generate the wide load or store, and shuffles.