From a813090072c0527eb6ed51dd2ea4f54cb6bc72a0 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sun, 29 Nov 2020 15:05:50 +0000 Subject: [PATCH] [VPlan] Manage stored values of interleave groups using VPUser (NFC) Interleave groups also depend on the values they store. Manage the stored values as VPUser operands. This is currently a NFC, but is required to allow VPlan transforms and to manage generated vector values exclusively in VPTransformState. --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 18 +++++++++++----- llvm/lib/Transforms/Vectorize/VPlan.h | 25 +++++++++++++++++++---- llvm/unittests/Transforms/Vectorize/VPlanTest.cpp | 2 +- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 0f519d1..1b716e1 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -550,6 +550,7 @@ public: /// values in the vectorized loop. void vectorizeInterleaveGroup(const InterleaveGroup *Group, VPTransformState &State, VPValue *Addr, + ArrayRef StoredValues, VPValue *BlockInMask = nullptr); /// Vectorize Load and Store instructions with the base address given in \p @@ -2322,7 +2323,7 @@ static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI) { // store <12 x i32> %interleaved.vec ; Write 4 tuples of R,G,B void InnerLoopVectorizer::vectorizeInterleaveGroup( const InterleaveGroup *Group, VPTransformState &State, - VPValue *Addr, VPValue *BlockInMask) { + VPValue *Addr, ArrayRef StoredValues, VPValue *BlockInMask) { Instruction *Instr = Group->getInsertPos(); const DataLayout &DL = Instr->getModule()->getDataLayout(); @@ -2467,8 +2468,8 @@ void InnerLoopVectorizer::vectorizeInterleaveGroup( Instruction *Member = Group->getMember(i); assert(Member && "Fail to get a member from an interleaved store group"); - Value *StoredVec = getOrCreateVectorValue( - cast(Member)->getValueOperand(), Part); + Value *StoredVec = State.get(StoredValues[i], Part); + if (Group->isReverse()) StoredVec = reverseVector(StoredVec); @@ -7828,7 +7829,13 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes( for (auto IG : InterleaveGroups) { auto *Recipe = cast( RecipeBuilder.getRecipe(IG->getInsertPos())); - (new VPInterleaveRecipe(IG, Recipe->getAddr(), Recipe->getMask())) + SmallVector StoredValues; + for (unsigned i = 0; i < IG->getFactor(); ++i) + if (auto *SI = dyn_cast_or_null(IG->getMember(i))) + StoredValues.push_back(Plan->getOrAddVPValue(SI->getOperand(0))); + + (new VPInterleaveRecipe(IG, Recipe->getAddr(), StoredValues, + Recipe->getMask())) ->insertBefore(Recipe); for (unsigned i = 0; i < IG->getFactor(); ++i) @@ -8068,7 +8075,8 @@ void VPBlendRecipe::execute(VPTransformState &State) { void VPInterleaveRecipe::execute(VPTransformState &State) { assert(!State.Instance && "Interleave group being replicated."); - State.ILV->vectorizeInterleaveGroup(IG, State, getAddr(), getMask()); + State.ILV->vectorizeInterleaveGroup(IG, State, getAddr(), getStoredValues(), + getMask()); } void VPReductionRecipe::execute(VPTransformState &State) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index b4d84ed..4abb088 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1035,16 +1035,24 @@ public: }; /// VPInterleaveRecipe is a recipe for transforming an interleave group of load -/// or stores into one wide load/store and shuffles. +/// or stores into one wide load/store and shuffles. The first operand of a +/// VPInterleave recipe is the address, followed by the stored values, followed +/// by an optional mask. class VPInterleaveRecipe : public VPRecipeBase, public VPUser { const InterleaveGroup *IG; + bool HasMask = false; + public: VPInterleaveRecipe(const InterleaveGroup *IG, VPValue *Addr, - VPValue *Mask) + ArrayRef StoredValues, VPValue *Mask) : VPRecipeBase(VPInterleaveSC), VPUser({Addr}), IG(IG) { - if (Mask) + for (auto *SV : StoredValues) + addOperand(SV); + if (Mask) { + HasMask = true; addOperand(Mask); + } } ~VPInterleaveRecipe() override = default; @@ -1062,7 +1070,16 @@ public: /// by a nullptr. VPValue *getMask() const { // Mask is optional and therefore the last, currently 2nd operand. - return getNumOperands() == 2 ? getOperand(1) : nullptr; + return HasMask ? getOperand(getNumOperands() - 1) : nullptr; + } + + /// Return the VPValues stored by this interleave group. If it is a load + /// interleave group, return an empty ArrayRef. + ArrayRef getStoredValues() const { + // The first operand is the address, followed by the stored values, followed + // by an optional mask. + return ArrayRef(op_begin(), getNumOperands()) + .slice(1, getNumOperands() - (HasMask ? 2 : 1)); } /// Generate the wide load or store, and shuffles. diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index d5c1139..f89abc8 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -470,7 +470,7 @@ TEST(VPRecipeTest, CastVPInterleaveRecipeToVPUser) { VPValue Addr; VPValue Mask; - VPInterleaveRecipe Recipe(nullptr, &Addr, &Mask); + VPInterleaveRecipe Recipe(nullptr, &Addr, {}, &Mask); EXPECT_TRUE(isa(&Recipe)); VPRecipeBase *BaseR = &Recipe; EXPECT_TRUE(isa(BaseR)); -- 2.7.4