From: Benjamin Kramer Date: Fri, 17 Apr 2020 13:28:00 +0000 (+0200) Subject: [VectorUtils] Create shufflevector masks as int vectors instead of Constants X-Git-Tag: llvmorg-12-init~8695 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=166467e8221d202ad3e7b8c156e99f7a6def35e0;p=platform%2Fupstream%2Fllvm.git [VectorUtils] Create shufflevector masks as int vectors instead of Constants No functionality change intended. --- diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h index 6571c87..2b680e8 100644 --- a/llvm/include/llvm/Analysis/VectorUtils.h +++ b/llvm/include/llvm/Analysis/VectorUtils.h @@ -450,8 +450,8 @@ Constant *createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, /// For example, the mask for \p ReplicationFactor=3 and \p VF=4 is: /// /// <0,0,0,1,1,1,2,2,2,3,3,3> -Constant *createReplicatedMask(IRBuilderBase &Builder, - unsigned ReplicationFactor, unsigned VF); +llvm::SmallVector createReplicatedMask(unsigned ReplicationFactor, + unsigned VF); /// Create an interleave shuffle mask. /// @@ -464,8 +464,7 @@ Constant *createReplicatedMask(IRBuilderBase &Builder, /// For example, the mask for VF = 4 and NumVecs = 2 is: /// /// <0, 4, 1, 5, 2, 6, 3, 7>. -Constant *createInterleaveMask(IRBuilderBase &Builder, unsigned VF, - unsigned NumVecs); +llvm::SmallVector createInterleaveMask(unsigned VF, unsigned NumVecs); /// Create a stride shuffle mask. /// @@ -479,8 +478,8 @@ Constant *createInterleaveMask(IRBuilderBase &Builder, unsigned VF, /// For example, the mask for Start = 0, Stride = 2, and VF = 4 is: /// /// <0, 2, 4, 6> -Constant *createStrideMask(IRBuilderBase &Builder, unsigned Start, - unsigned Stride, unsigned VF); +llvm::SmallVector createStrideMask(unsigned Start, unsigned Stride, + unsigned VF); /// Create a sequential shuffle mask. /// @@ -493,8 +492,8 @@ Constant *createStrideMask(IRBuilderBase &Builder, unsigned Start, /// For example, the mask for Start = 0, NumInsts = 4, and NumUndefs = 4 is: /// /// <0, 1, 2, 3, undef, undef, undef, undef> -Constant *createSequentialMask(IRBuilderBase &Builder, unsigned Start, - unsigned NumInts, unsigned NumUndefs); +llvm::SmallVector +createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs); /// Concatenate a list of vectors. /// diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp index 31b600b..0d41148 100644 --- a/llvm/lib/Analysis/VectorUtils.cpp +++ b/llvm/lib/Analysis/VectorUtils.cpp @@ -761,46 +761,46 @@ llvm::createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, return ConstantVector::get(Mask); } -Constant *llvm::createReplicatedMask(IRBuilderBase &Builder, - unsigned ReplicationFactor, unsigned VF) { - SmallVector MaskVec; +llvm::SmallVector +llvm::createReplicatedMask(unsigned ReplicationFactor, unsigned VF) { + SmallVector MaskVec; for (unsigned i = 0; i < VF; i++) for (unsigned j = 0; j < ReplicationFactor; j++) - MaskVec.push_back(Builder.getInt32(i)); + MaskVec.push_back(i); - return ConstantVector::get(MaskVec); + return MaskVec; } -Constant *llvm::createInterleaveMask(IRBuilderBase &Builder, unsigned VF, - unsigned NumVecs) { - SmallVector Mask; +llvm::SmallVector llvm::createInterleaveMask(unsigned VF, + unsigned NumVecs) { + SmallVector Mask; for (unsigned i = 0; i < VF; i++) for (unsigned j = 0; j < NumVecs; j++) - Mask.push_back(Builder.getInt32(j * VF + i)); + Mask.push_back(j * VF + i); - return ConstantVector::get(Mask); + return Mask; } -Constant *llvm::createStrideMask(IRBuilderBase &Builder, unsigned Start, - unsigned Stride, unsigned VF) { - SmallVector Mask; +llvm::SmallVector +llvm::createStrideMask(unsigned Start, unsigned Stride, unsigned VF) { + SmallVector Mask; for (unsigned i = 0; i < VF; i++) - Mask.push_back(Builder.getInt32(Start + i * Stride)); + Mask.push_back(Start + i * Stride); - return ConstantVector::get(Mask); + return Mask; } -Constant *llvm::createSequentialMask(IRBuilderBase &Builder, unsigned Start, - unsigned NumInts, unsigned NumUndefs) { - SmallVector Mask; +llvm::SmallVector llvm::createSequentialMask(unsigned Start, + unsigned NumInts, + unsigned NumUndefs) { + SmallVector Mask; for (unsigned i = 0; i < NumInts; i++) - Mask.push_back(Builder.getInt32(Start + i)); + Mask.push_back(Start + i); - Constant *Undef = UndefValue::get(Builder.getInt32Ty()); for (unsigned i = 0; i < NumUndefs; i++) - Mask.push_back(Undef); + Mask.push_back(-1); - return ConstantVector::get(Mask); + return Mask; } /// A helper function for concatenating vectors. This function concatenates two @@ -820,13 +820,13 @@ static Value *concatenateTwoVectors(IRBuilderBase &Builder, Value *V1, if (NumElts1 > NumElts2) { // Extend with UNDEFs. - Constant *ExtMask = - createSequentialMask(Builder, 0, NumElts2, NumElts1 - NumElts2); - V2 = Builder.CreateShuffleVector(V2, UndefValue::get(VecTy2), ExtMask); + V2 = Builder.CreateShuffleVector( + V2, UndefValue::get(VecTy2), + createSequentialMask(0, NumElts2, NumElts1 - NumElts2)); } - Constant *Mask = createSequentialMask(Builder, 0, NumElts1 + NumElts2, 0); - return Builder.CreateShuffleVector(V1, V2, Mask); + return Builder.CreateShuffleVector( + V1, V2, createSequentialMask(0, NumElts1 + NumElts2, 0)); } Value *llvm::concatenateVectors(IRBuilderBase &Builder, diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 6311137..f285b4b 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -9557,7 +9557,7 @@ bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI, unsigned IdxI = StoreCount * LaneLen * Factor + i; if (Mask[IdxI] >= 0) { Ops.push_back(Builder.CreateShuffleVector( - Op0, Op1, createSequentialMask(Builder, Mask[IdxI], LaneLen, 0))); + Op0, Op1, createSequentialMask(Mask[IdxI], LaneLen, 0))); } else { unsigned StartMask = 0; for (unsigned j = 1; j < LaneLen; j++) { @@ -9573,7 +9573,7 @@ bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI, // Note: StartMask cannot be negative, it's checked in // isReInterleaveMask Ops.push_back(Builder.CreateShuffleVector( - Op0, Op1, createSequentialMask(Builder, StartMask, LaneLen, 0))); + Op0, Op1, createSequentialMask(StartMask, LaneLen, 0))); } } diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index 4807148..e9ecc05 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -18016,7 +18016,7 @@ bool ARMTargetLowering::lowerInterleavedStore(StoreInst *SI, unsigned IdxI = StoreCount * LaneLen * Factor + i; if (Mask[IdxI] >= 0) { Shuffles.push_back(Builder.CreateShuffleVector( - Op0, Op1, createSequentialMask(Builder, Mask[IdxI], LaneLen, 0))); + Op0, Op1, createSequentialMask(Mask[IdxI], LaneLen, 0))); } else { unsigned StartMask = 0; for (unsigned j = 1; j < LaneLen; j++) { @@ -18033,7 +18033,7 @@ bool ARMTargetLowering::lowerInterleavedStore(StoreInst *SI, // Note: StartMask cannot be negative, it's checked in // isReInterleaveMask Shuffles.push_back(Builder.CreateShuffleVector( - Op0, Op1, createSequentialMask(Builder, StartMask, LaneLen, 0))); + Op0, Op1, createSequentialMask(StartMask, LaneLen, 0))); } } diff --git a/llvm/lib/Target/X86/X86InterleavedAccess.cpp b/llvm/lib/Target/X86/X86InterleavedAccess.cpp index 81879b59..aa25cb9 100644 --- a/llvm/lib/Target/X86/X86InterleavedAccess.cpp +++ b/llvm/lib/Target/X86/X86InterleavedAccess.cpp @@ -186,8 +186,8 @@ void X86InterleavedAccessGroup::decompose( DecomposedVectors.push_back( cast(Builder.CreateShuffleVector( Op0, Op1, - createSequentialMask(Builder, Indices[i], - SubVecTy->getVectorNumElements(), 0)))); + createSequentialMask(Indices[i], SubVecTy->getVectorNumElements(), + 0)))); return; } diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp index a2ddf85..86c5600 100644 --- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp +++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp @@ -328,9 +328,9 @@ class LowerMatrixIntrinsics { IRBuilder<> &Builder) const { Value *Vec = isColumnMajor() ? getColumn(J) : getRow(I); Value *Undef = UndefValue::get(Vec->getType()); - Constant *Mask = - createSequentialMask(Builder, isColumnMajor() ? I : J, NumElts, 0); - return Builder.CreateShuffleVector(Vec, Undef, Mask, "block"); + return Builder.CreateShuffleVector( + Vec, Undef, createSequentialMask(isColumnMajor() ? I : J, NumElts, 0), + "block"); } }; @@ -442,9 +442,9 @@ public: Value *Undef = UndefValue::get(VType); for (unsigned MaskStart = 0; MaskStart < VType->getNumElements(); MaskStart += SI.getStride()) { - Constant *Mask = - createSequentialMask(Builder, MaskStart, SI.getStride(), 0); - Value *V = Builder.CreateShuffleVector(MatrixVal, Undef, Mask, "split"); + Value *V = Builder.CreateShuffleVector( + MatrixVal, Undef, createSequentialMask(MaskStart, SI.getStride(), 0), + "split"); SplitVecs.push_back(V); } @@ -909,10 +909,10 @@ public: unsigned NumElts = cast(Col->getType())->getNumElements(); assert(NumElts >= BlockNumElts && "Too few elements for current block"); - Value *ExtendMask = - createSequentialMask(Builder, 0, BlockNumElts, NumElts - BlockNumElts); Value *Undef = UndefValue::get(Block->getType()); - Block = Builder.CreateShuffleVector(Block, Undef, ExtendMask); + Block = Builder.CreateShuffleVector( + Block, Undef, + createSequentialMask(0, BlockNumElts, NumElts - BlockNumElts)); // If Col is 7 long and I is 2 and BlockNumElts is 2 the mask is: 0, 1, 7, // 8, 4, 5, 6 diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 35ed4c0..7eaab8b 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2253,9 +2253,9 @@ void InnerLoopVectorizer::vectorizeInterleaveGroup( if (BlockInMask) { Value *BlockInMaskPart = State.get(BlockInMask, Part); auto *Undefs = UndefValue::get(BlockInMaskPart->getType()); - auto *RepMask = createReplicatedMask(Builder, InterleaveFactor, VF); Value *ShuffledMask = Builder.CreateShuffleVector( - BlockInMaskPart, Undefs, RepMask, "interleaved.mask"); + BlockInMaskPart, Undefs, + createReplicatedMask(InterleaveFactor, VF), "interleaved.mask"); GroupMask = MaskForGaps ? Builder.CreateBinOp(Instruction::And, ShuffledMask, MaskForGaps) @@ -2281,7 +2281,7 @@ void InnerLoopVectorizer::vectorizeInterleaveGroup( if (!Member) continue; - Constant *StrideMask = createStrideMask(Builder, I, InterleaveFactor, VF); + auto StrideMask = createStrideMask(I, InterleaveFactor, VF); for (unsigned Part = 0; Part < UF; Part++) { Value *StridedVec = Builder.CreateShuffleVector( NewLoads[Part], UndefVec, StrideMask, "strided.vec"); @@ -2330,17 +2330,17 @@ void InnerLoopVectorizer::vectorizeInterleaveGroup( Value *WideVec = concatenateVectors(Builder, StoredVecs); // Interleave the elements in the wide vector. - Constant *IMask = createInterleaveMask(Builder, VF, InterleaveFactor); - Value *IVec = Builder.CreateShuffleVector(WideVec, UndefVec, IMask, - "interleaved.vec"); + Value *IVec = Builder.CreateShuffleVector( + WideVec, UndefVec, createInterleaveMask(VF, InterleaveFactor), + "interleaved.vec"); Instruction *NewStoreInstr; if (BlockInMask) { Value *BlockInMaskPart = State.get(BlockInMask, Part); auto *Undefs = UndefValue::get(BlockInMaskPart->getType()); - auto *RepMask = createReplicatedMask(Builder, InterleaveFactor, VF); Value *ShuffledMask = Builder.CreateShuffleVector( - BlockInMaskPart, Undefs, RepMask, "interleaved.mask"); + BlockInMaskPart, Undefs, createReplicatedMask(InterleaveFactor, VF), + "interleaved.mask"); NewStoreInstr = Builder.CreateMaskedStore( IVec, AddrParts[Part], Group->getAlign(), ShuffledMask); }