// Shuffle vector to get vec3.
V = Builder.CreateShuffleVector(V, llvm::UndefValue::get(vec4Ty),
- {0, 1, 2}, "extractVec");
+ ArrayRef<int>{0, 1, 2}, "extractVec");
return EmitFromMemory(V, Ty);
}
}
return LHS.equals(RHS);
}
- template<typename T>
+ template <typename T>
+ inline bool operator==(SmallVectorImpl<T> &LHS, ArrayRef<T> RHS) {
+ return ArrayRef<T>(LHS).equals(RHS);
+ }
+
+ template <typename T>
inline bool operator!=(ArrayRef<T> LHS, ArrayRef<T> RHS) {
return !(LHS == RHS);
}
+ template <typename T>
+ inline bool operator!=(SmallVectorImpl<T> &LHS, ArrayRef<T> RHS) {
+ return !(LHS == RHS);
+ }
+
/// @}
template <typename T> hash_code hash_value(ArrayRef<T> S) {
Constant *ConstantFoldExtractElementInstruction(Constant *Val, Constant *Idx);
/// Attempt to constant fold a shufflevector instruction with the
-/// specified operands and indices. The constant result is returned if
-/// successful; if not, null is returned.
+/// specified operands and mask. See class ShuffleVectorInst for a description
+/// of the mask representation. The constant result is returned if successful;
+/// if not, null is returned.
Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
- Constant *Mask);
+ ArrayRef<int> Mask);
/// ConstantFoldLoadFromConstPtr - Return the value that a load from C would
/// produce if it is constant and determinable. If this is not determinable,
const SimplifyQuery &Q);
/// Given operands for a ShuffleVectorInst, fold the result or return null.
-Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
+/// See class ShuffleVectorInst for a description of the mask representation.
+Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef<int> Mask,
Type *RetTy, const SimplifyQuery &Q);
//=== Helper functions for higher up the class hierarchy.
}
Constant *CreateShuffleVector(Constant *V1, Constant *V2,
- Constant *Mask) const override {
+ ArrayRef<int> Mask) const override {
return Fold(ConstantExpr::getShuffleVector(V1, V2, Mask));
}
}
Constant *CreateShuffleVector(Constant *V1, Constant *V2,
- Constant *Mask) const override {
+ ArrayRef<int> Mask) const override {
return ConstantExpr::getShuffleVector(V1, V2, Mask);
}
Type *OnlyIfReducedTy = nullptr);
static Constant *getInsertElement(Constant *Vec, Constant *Elt, Constant *Idx,
Type *OnlyIfReducedTy = nullptr);
- static Constant *getShuffleVector(Constant *V1, Constant *V2, Constant *Mask,
+ static Constant *getShuffleVector(Constant *V1, Constant *V2,
+ ArrayRef<int> Mask,
Type *OnlyIfReducedTy = nullptr);
static Constant *getExtractValue(Constant *Agg, ArrayRef<unsigned> Idxs,
Type *OnlyIfReducedTy = nullptr);
/// expression and return the list of indices.
ArrayRef<unsigned> getIndices() const;
+ /// Assert that this is a shufflevector and return the mask. See class
+ /// ShuffleVectorInst for a description of the mask representation.
+ ArrayRef<int> getShuffleMask() const;
+
+ /// Assert that this is a shufflevector and return the mask.
+ ///
+ /// TODO: This is a temporary hack until we update the bitcode format for
+ /// shufflevector.
+ Constant *getShuffleMaskForBitcode() const;
+
/// Return a string representation for an opcode.
const char *getOpcodeName() const;
Value *CreateShuffleVector(Value *V1, Value *V2, Value *Mask,
const Twine &Name = "") {
- if (auto *V1C = dyn_cast<Constant>(V1))
- if (auto *V2C = dyn_cast<Constant>(V2))
- if (auto *MC = dyn_cast<Constant>(Mask))
- return Insert(Folder.CreateShuffleVector(V1C, V2C, MC), Name);
- return Insert(new ShuffleVectorInst(V1, V2, Mask), Name);
+ SmallVector<int, 16> IntMask;
+ ShuffleVectorInst::getShuffleMask(cast<Constant>(Mask), IntMask);
+ return CreateShuffleVector(V1, V2, IntMask, Name);
}
- Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<uint32_t> IntMask,
+ Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<uint32_t> Mask,
const Twine &Name = "") {
- Value *Mask = ConstantDataVector::get(Context, IntMask);
- return CreateShuffleVector(V1, V2, Mask, Name);
+ SmallVector<int, 16> IntMask;
+ IntMask.assign(Mask.begin(), Mask.end());
+ return CreateShuffleVector(V1, V2, IntMask, Name);
+ }
+
+ /// See class ShuffleVectorInst for a description of the mask representation.
+ Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &Name = "") {
+ if (auto *V1C = dyn_cast<Constant>(V1))
+ if (auto *V2C = dyn_cast<Constant>(V2))
+ return Insert(Folder.CreateShuffleVector(V1C, V2C, Mask), Name);
+ return Insert(new ShuffleVectorInst(V1, V2, Mask), Name);
}
Value *CreateExtractValue(Value *Agg,
virtual Value *CreateInsertElement(Constant *Vec, Constant *NewElt,
Constant *Idx) const = 0;
virtual Value *CreateShuffleVector(Constant *V1, Constant *V2,
- Constant *Mask) const = 0;
+ ArrayRef<int> Mask) const = 0;
virtual Value *CreateExtractValue(Constant *Agg,
ArrayRef<unsigned> IdxList) const = 0;
virtual Value *CreateInsertValue(Constant *Agg, Constant *Val,
// ShuffleVectorInst Class
//===----------------------------------------------------------------------===//
+constexpr int UndefMaskElem = -1;
+
/// This instruction constructs a fixed permutation of two
/// input vectors.
///
+/// For each element of the result vector, the shuffle mask selects an element
+/// from one of the input vectors to copy to the result. Non-negative elements
+/// in the mask represent an index into the concatenated pair of input vectors.
+/// UndefMaskElem (-1) specifies that the result element is undefined.
+///
+/// For scalable vectors, all the elements of the mask must be 0 or -1. This
+/// requirement may be relaxed in the future.
class ShuffleVectorInst : public Instruction {
+ SmallVector<int, 4> ShuffleMask;
+ Constant *ShuffleMaskForBitcode;
+
protected:
// Note: Instruction needs to be a friend here to call cloneImpl.
friend class Instruction;
Instruction *InsertBefor = nullptr);
ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
const Twine &NameStr, BasicBlock *InsertAtEnd);
+ ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &NameStr = "",
+ Instruction *InsertBefor = nullptr);
+ ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &NameStr, BasicBlock *InsertAtEnd);
- // allocate space for exactly three operands
- void *operator new(size_t s) {
- return User::operator new(s, 3);
- }
+ void *operator new(size_t s) { return User::operator new(s, 2); }
- /// Swap the first 2 operands and adjust the mask to preserve the semantics
+ /// Swap the operands and adjust the mask to preserve the semantics
/// of the instruction.
void commute();
/// formed with the specified operands.
static bool isValidOperands(const Value *V1, const Value *V2,
const Value *Mask);
+ static bool isValidOperands(const Value *V1, const Value *V2,
+ ArrayRef<int> Mask);
/// Overload to return most specific vector type.
///
/// Transparently provide more efficient getOperand methods.
DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
- Constant *getMask() const {
- return cast<Constant>(getOperand(2));
- }
-
- /// Return the shuffle mask value for the specified element of the mask.
- /// Return -1 if the element is undef.
- static int getMaskValue(const Constant *Mask, unsigned Elt);
-
/// Return the shuffle mask value of this instruction for the given element
- /// index. Return -1 if the element is undef.
- int getMaskValue(unsigned Elt) const {
- return getMaskValue(getMask(), Elt);
- }
+ /// index. Return UndefMaskElem if the element is undef.
+ int getMaskValue(unsigned Elt) const { return ShuffleMask[Elt]; }
/// Convert the input shuffle mask operand to a vector of integers. Undefined
- /// elements of the mask are returned as -1.
+ /// elements of the mask are returned as UndefMaskElem.
static void getShuffleMask(const Constant *Mask,
SmallVectorImpl<int> &Result);
/// Return the mask for this instruction as a vector of integers. Undefined
- /// elements of the mask are returned as -1.
+ /// elements of the mask are returned as UndefMaskElem.
void getShuffleMask(SmallVectorImpl<int> &Result) const {
- return getShuffleMask(getMask(), Result);
+ Result.assign(ShuffleMask.begin(), ShuffleMask.end());
}
- SmallVector<int, 16> getShuffleMask() const {
- SmallVector<int, 16> Mask;
- getShuffleMask(Mask);
- return Mask;
- }
+ /// Return the mask for this instruction, for use in bitcode.
+ ///
+ /// TODO: This is temporary until we decide a new bitcode encoding for
+ /// shufflevector.
+ Constant *getShuffleMaskForBitcode() const { return ShuffleMaskForBitcode; }
+
+ static Constant *convertShuffleMaskForBitcode(ArrayRef<int> Mask,
+ Type *ResultTy);
+
+ void setShuffleMask(ArrayRef<int> Mask);
+
+ ArrayRef<int> getShuffleMask() const { return ShuffleMask; }
/// Return true if this shuffle returns a vector with a different number of
/// elements than its source vectors.
/// shufflevector <4 x n> A, <4 x n> B, <1,2,3,4,5>
bool changesLength() const {
unsigned NumSourceElts = Op<0>()->getType()->getVectorNumElements();
- unsigned NumMaskElts = getMask()->getType()->getVectorNumElements();
+ unsigned NumMaskElts = ShuffleMask.size();
return NumSourceElts != NumMaskElts;
}
/// Example: shufflevector <2 x n> A, <2 x n> B, <1,2,3>
bool increasesLength() const {
unsigned NumSourceElts = Op<0>()->getType()->getVectorNumElements();
- unsigned NumMaskElts = getMask()->getType()->getVectorNumElements();
+ unsigned NumMaskElts = ShuffleMask.size();
return NumSourceElts < NumMaskElts;
}
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3>
/// TODO: Optionally allow length-changing shuffles.
bool isSingleSource() const {
- return !changesLength() && isSingleSourceMask(getMask());
+ return !changesLength() && isSingleSourceMask(ShuffleMask);
}
/// Return true if this shuffle mask chooses elements from exactly one source
/// from its input vectors.
/// Example: shufflevector <4 x n> A, <4 x n> B, <4,undef,6,undef>
bool isIdentity() const {
- return !changesLength() && isIdentityMask(getShuffleMask());
+ return !changesLength() && isIdentityMask(ShuffleMask);
}
/// Return true if this shuffle lengthens exactly one source vector with
/// In that case, the shuffle is better classified as an identity shuffle.
/// TODO: Optionally allow length-changing shuffles.
bool isSelect() const {
- return !changesLength() && isSelectMask(getMask());
+ return !changesLength() && isSelectMask(ShuffleMask);
}
/// Return true if this shuffle mask swaps the order of elements from exactly
/// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef>
/// TODO: Optionally allow length-changing shuffles.
bool isReverse() const {
- return !changesLength() && isReverseMask(getMask());
+ return !changesLength() && isReverseMask(ShuffleMask);
}
/// Return true if this shuffle mask chooses all elements with the same value
/// TODO: Optionally allow length-changing shuffles.
/// TODO: Optionally allow splats from other elements.
bool isZeroEltSplat() const {
- return !changesLength() && isZeroEltSplatMask(getMask());
+ return !changesLength() && isZeroEltSplatMask(ShuffleMask);
}
/// Return true if this shuffle mask is a transpose mask.
/// exact specification.
/// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6>
bool isTranspose() const {
- return !changesLength() && isTransposeMask(getMask());
+ return !changesLength() && isTransposeMask(ShuffleMask);
}
/// Return true if this shuffle mask is an extract subvector mask.
/// Return true if this shuffle mask is an extract subvector mask.
bool isExtractSubvectorMask(int &Index) const {
int NumSrcElts = Op<0>()->getType()->getVectorNumElements();
- return isExtractSubvectorMask(getMask(), NumSrcElts, Index);
+ return isExtractSubvectorMask(ShuffleMask, NumSrcElts, Index);
}
/// Change values in a shuffle permute mask assuming the two vector operands
};
template <>
-struct OperandTraits<ShuffleVectorInst> :
- public FixedNumOperandTraits<ShuffleVectorInst, 3> {
-};
+struct OperandTraits<ShuffleVectorInst>
+ : public FixedNumOperandTraits<ShuffleVectorInst, 2> {};
DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorInst, Value)
}
Instruction *CreateShuffleVector(Constant *V1, Constant *V2,
- Constant *Mask) const override {
+ ArrayRef<int> Mask) const override {
return new ShuffleVectorInst(V1, V2, Mask);
}
return const_cast<Pattern &>(P).match(V);
}
+template <typename Pattern> bool match(ArrayRef<int> Mask, const Pattern &P) {
+ return const_cast<Pattern &>(P).match(Mask);
+}
+
template <typename SubPattern_t> struct OneUse_match {
SubPattern_t SubPattern;
return TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>(Val, Idx);
}
-/// Matches ShuffleVectorInst.
+/// Matches shuffle.
+template <typename T0, typename T1, typename T2> struct Shuffle_match {
+ T0 Op1;
+ T1 Op2;
+ T2 Mask;
+
+ Shuffle_match(const T0 &Op1, const T1 &Op2, const T2 &Mask)
+ : Op1(Op1), Op2(Op2), Mask(Mask) {}
+
+ template <typename OpTy> bool match(OpTy *V) {
+ if (auto *I = dyn_cast<ShuffleVectorInst>(V)) {
+ return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) &&
+ Mask.match(I->getShuffleMask());
+ }
+ return false;
+ }
+};
+
+struct m_Mask {
+ ArrayRef<int> &MaskRef;
+ m_Mask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
+ bool match(ArrayRef<int> Mask) {
+ MaskRef = Mask;
+ return true;
+ }
+};
+
+struct m_ZeroMask {
+ bool match(ArrayRef<int> Mask) {
+ return all_of(Mask, [](int Elem) { return Elem == 0 || Elem == -1; });
+ }
+};
+
+struct m_SpecificMask {
+ ArrayRef<int> &MaskRef;
+ m_SpecificMask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
+ bool match(ArrayRef<int> Mask) { return MaskRef == Mask; }
+};
+
+struct m_SplatOrUndefMask {
+ int &SplatIndex;
+ m_SplatOrUndefMask(int &SplatIndex) : SplatIndex(SplatIndex) {}
+ bool match(ArrayRef<int> Mask) {
+ auto First = find_if(Mask, [](int Elem) { return Elem != -1; });
+ if (First == Mask.end())
+ return false;
+ SplatIndex = *First;
+ return all_of(Mask,
+ [First](int Elem) { return Elem == *First || Elem == -1; });
+ }
+};
+
+/// Matches ShuffleVectorInst independently of mask value.
+template <typename V1_t, typename V2_t>
+inline TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>
+m_ShuffleVector(const V1_t &v1, const V2_t &v2) {
+ return TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>(v1, v2);
+}
+
template <typename V1_t, typename V2_t, typename Mask_t>
-inline ThreeOps_match<V1_t, V2_t, Mask_t, Instruction::ShuffleVector>
-m_ShuffleVector(const V1_t &v1, const V2_t &v2, const Mask_t &m) {
- return ThreeOps_match<V1_t, V2_t, Mask_t, Instruction::ShuffleVector>(v1, v2,
- m);
+inline Shuffle_match<V1_t, V2_t, Mask_t>
+m_ShuffleVector(const V1_t &v1, const V2_t &v2, const Mask_t &mask) {
+ return Shuffle_match<V1_t, V2_t, Mask_t>(v1, v2, mask);
}
/// Matches LoadInst.
case Instruction::InsertElement:
return ConstantExpr::getInsertElement(Ops[0], Ops[1], Ops[2]);
case Instruction::ShuffleVector:
- return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2]);
+ return ConstantExpr::getShuffleVector(
+ Ops[0], Ops[1], cast<ShuffleVectorInst>(InstOrCE)->getShuffleMask());
}
}
return RootVec;
}
-static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
- Type *RetTy, const SimplifyQuery &Q,
+static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1,
+ ArrayRef<int> Mask, Type *RetTy,
+ const SimplifyQuery &Q,
unsigned MaxRecurse) {
- if (isa<UndefValue>(Mask))
+ if (all_of(Mask, [](int Elem) { return Elem == UndefMaskElem; }))
return UndefValue::get(RetTy);
Type *InVecTy = Op0->getType();
- ElementCount MaskEltCount = Mask->getType()->getVectorElementCount();
+ unsigned MaskNumElts = Mask.size();
ElementCount InVecEltCount = InVecTy->getVectorElementCount();
- assert(MaskEltCount.Scalable == InVecEltCount.Scalable &&
- "vscale mismatch between input vector and mask");
-
- bool Scalable = MaskEltCount.Scalable;
+ bool Scalable = InVecEltCount.Scalable;
SmallVector<int, 32> Indices;
- if (!Scalable) {
- ShuffleVectorInst::getShuffleMask(Mask, Indices);
- assert(MaskEltCount.Min == Indices.size() &&
- "Size of Indices not same as number of mask elements?");
- }
+ Indices.assign(Mask.begin(), Mask.end());
+ // Canonicalization: If mask does not select elements from an input vector,
+ // replace that input vector with undef.
if (!Scalable) {
- // Canonicalization: If mask does not select elements from an input vector,
- // replace that input vector with undef.
bool MaskSelects0 = false, MaskSelects1 = false;
- for (unsigned i = 0; i != MaskEltCount.Min; ++i) {
+ unsigned InVecNumElts = InVecEltCount.Min;
+ for (unsigned i = 0; i != MaskNumElts; ++i) {
if (Indices[i] == -1)
continue;
- if ((unsigned)Indices[i] < InVecEltCount.Min)
+ if ((unsigned)Indices[i] < InVecNumElts)
MaskSelects0 = true;
else
MaskSelects1 = true;
assert(isa<UndefValue>(Op1) && "Expected undef operand 1 for splat");
// Shuffle mask undefs become undefined constant result elements.
- SmallVector<Constant *, 16> VecC(MaskEltCount.Min, C);
- for (unsigned i = 0; i != MaskEltCount.Min; ++i)
+ SmallVector<Constant *, 16> VecC(MaskNumElts, C);
+ for (unsigned i = 0; i != MaskNumElts; ++i)
if (Indices[i] == -1)
VecC[i] = UndefValue::get(C->getType());
return ConstantVector::get(VecC);
// value type is same as the input vectors' type.
if (auto *OpShuf = dyn_cast<ShuffleVectorInst>(Op0))
if (isa<UndefValue>(Op1) && RetTy == InVecTy &&
- OpShuf->getMask()->getSplatValue())
+ is_splat(OpShuf->getShuffleMask()))
return Op0;
// All remaining transformation depend on the value of the mask, which is
// shuffle. This handles simple identity shuffles as well as chains of
// shuffles that may widen/narrow and/or move elements across lanes and back.
Value *RootVec = nullptr;
- for (unsigned i = 0; i != MaskEltCount.Min; ++i) {
+ for (unsigned i = 0; i != MaskNumElts; ++i) {
// Note that recursion is limited for each vector element, so if any element
// exceeds the limit, this will fail to simplify.
RootVec =
}
/// Given operands for a ShuffleVectorInst, fold the result or return null.
-Value *llvm::SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask,
- Type *RetTy, const SimplifyQuery &Q) {
+Value *llvm::SimplifyShuffleVectorInst(Value *Op0, Value *Op1,
+ ArrayRef<int> Mask, Type *RetTy,
+ const SimplifyQuery &Q) {
return ::SimplifyShuffleVectorInst(Op0, Op1, Mask, RetTy, Q, RecursionLimit);
}
}
case Instruction::ShuffleVector: {
auto *SVI = cast<ShuffleVectorInst>(I);
- Result = SimplifyShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
- SVI->getMask(), SVI->getType(), Q);
+ Result =
+ SimplifyShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
+ SVI->getShuffleMask(), SVI->getType(), Q);
break;
}
case Instruction::PHI:
for (unsigned i = 0, e = (1 << Level), val = !IsLeft; i != e; ++i, val += 2)
Mask[i] = val;
- SmallVector<int, 16> ActualMask = SI->getShuffleMask();
+ ArrayRef<int> ActualMask = SI->getShuffleMask();
return Mask == ActualMask;
}
// Fill the rest of the mask with -1 for undef.
std::fill(&ShuffleMask[MaskStart], ShuffleMask.end(), -1);
- SmallVector<int, 16> Mask = Shuffle->getShuffleMask();
+ ArrayRef<int> Mask = Shuffle->getShuffleMask();
if (ShuffleMask != Mask)
return RK_None;
APInt &DemandedLHS, APInt &DemandedRHS) {
// The length of scalable vectors is unknown at compile time, thus we
// cannot check their values
- if (Shuf->getMask()->getType()->getVectorElementCount().Scalable)
+ if (Shuf->getType()->getVectorElementCount().Scalable)
return false;
int NumElts = Shuf->getOperand(0)->getType()->getVectorNumElements();
- int NumMaskElts = Shuf->getMask()->getType()->getVectorNumElements();
+ int NumMaskElts = Shuf->getType()->getVectorNumElements();
DemandedLHS = DemandedRHS = APInt::getNullValue(NumElts);
if (DemandedElts.isNullValue())
return true;
// Simple case of a shuffle with zeroinitializer.
- if (isa<ConstantAggregateZero>(Shuf->getMask())) {
+ if (all_of(Shuf->getShuffleMask(), [](int Elt) { return Elt == 0; })) {
DemandedLHS.setBit(0);
return true;
}
// shuf (inselt ?, Splat, 0), ?, <0, undef, 0, ...>
Value *Splat;
- if (match(V, m_ShuffleVector(m_InsertElement(m_Value(), m_Value(Splat),
- m_ZeroInt()),
- m_Value(), m_ZeroInt())))
+ if (match(V, m_ShuffleVector(
+ m_InsertElement(m_Value(), m_Value(Splat), m_ZeroInt()),
+ m_Value(), m_ZeroMask())))
return Splat;
return nullptr;
if (auto *Shuf = dyn_cast<ShuffleVectorInst>(V)) {
// FIXME: We can safely allow undefs here. If Index was specified, we will
// check that the mask elt is defined at the required index.
- if (!Shuf->getMask()->getSplatValue())
+ if (!is_splat(Shuf->getShuffleMask()))
return false;
// Match any index.
return Error(ID.Loc, "expected three operands to shufflevector");
if (!ShuffleVectorInst::isValidOperands(Elts[0], Elts[1], Elts[2]))
return Error(ID.Loc, "invalid operands to shufflevector");
- ID.ConstantVal =
- ConstantExpr::getShuffleVector(Elts[0], Elts[1],Elts[2]);
+ SmallVector<int, 16> Mask;
+ ShuffleVectorInst::getShuffleMask(cast<Constant>(Elts[2]), Mask);
+ ID.ConstantVal = ConstantExpr::getShuffleVector(Elts[0], Elts[1], Mask);
} else if (Opc == Instruction::ExtractElement) {
if (Elts.size() != 2)
return Error(ID.Loc, "expected two operands to extractelement");
Type *CurFullTy = Type::getInt32Ty(Context);
unsigned NextCstNo = ValueList.size();
+ struct DelayedShufTy {
+ VectorType *OpTy;
+ VectorType *RTy;
+ Type *CurFullTy;
+ uint64_t Op0Idx;
+ uint64_t Op1Idx;
+ uint64_t Op2Idx;
+ };
+ std::vector<DelayedShufTy> DelayedShuffles;
while (true) {
Expected<BitstreamEntry> MaybeEntry = Stream.advanceSkippingSubblocks();
if (!MaybeEntry)
// Once all the constants have been read, go through and resolve forward
// references.
+ //
+ // We have to treat shuffles specially because they don't have three
+ // operands anymore. We need to convert the shuffle mask into an array,
+ // and we can't convert a forward reference.
+ for (auto &DelayedShuffle : DelayedShuffles) {
+ VectorType *OpTy = DelayedShuffle.OpTy;
+ VectorType *RTy = DelayedShuffle.RTy;
+ uint64_t Op0Idx = DelayedShuffle.Op0Idx;
+ uint64_t Op1Idx = DelayedShuffle.Op1Idx;
+ uint64_t Op2Idx = DelayedShuffle.Op2Idx;
+ Constant *Op0 = ValueList.getConstantFwdRef(Op0Idx, OpTy);
+ Constant *Op1 = ValueList.getConstantFwdRef(Op1Idx, OpTy);
+ Type *ShufTy =
+ VectorType::get(Type::getInt32Ty(Context), RTy->getElementCount());
+ Constant *Op2 = ValueList.getConstantFwdRef(Op2Idx, ShufTy);
+ if (!ShuffleVectorInst::isValidOperands(Op0, Op1, Op2))
+ return error("Invalid shufflevector operands");
+ SmallVector<int, 16> Mask;
+ ShuffleVectorInst::getShuffleMask(Op2, Mask);
+ Value *V = ConstantExpr::getShuffleVector(Op0, Op1, Mask);
+ ValueList.assignValue(V, NextCstNo, DelayedShuffle.CurFullTy);
+ ++NextCstNo;
+ }
ValueList.resolveConstantForwardRefs();
return Error::success();
case BitstreamEntry::Record:
VectorType *OpTy = dyn_cast<VectorType>(CurTy);
if (Record.size() < 3 || !OpTy)
return error("Invalid record");
- Constant *Op0 = ValueList.getConstantFwdRef(Record[0], OpTy);
- Constant *Op1 = ValueList.getConstantFwdRef(Record[1], OpTy);
- Type *ShufTy = VectorType::get(Type::getInt32Ty(Context),
- OpTy->getElementCount());
- Constant *Op2 = ValueList.getConstantFwdRef(Record[2], ShufTy);
- V = ConstantExpr::getShuffleVector(Op0, Op1, Op2);
- break;
+ DelayedShuffles.push_back(
+ {OpTy, OpTy, CurFullTy, Record[0], Record[1], Record[2]});
+ continue;
}
case bitc::CST_CODE_CE_SHUFVEC_EX: { // [opty, opval, opval, opval]
VectorType *RTy = dyn_cast<VectorType>(CurTy);
dyn_cast_or_null<VectorType>(getTypeByID(Record[0]));
if (Record.size() < 4 || !RTy || !OpTy)
return error("Invalid record");
- Constant *Op0 = ValueList.getConstantFwdRef(Record[1], OpTy);
- Constant *Op1 = ValueList.getConstantFwdRef(Record[2], OpTy);
- Type *ShufTy = VectorType::get(Type::getInt32Ty(Context),
- RTy->getElementCount());
- Constant *Op2 = ValueList.getConstantFwdRef(Record[3], ShufTy);
- V = ConstantExpr::getShuffleVector(Op0, Op1, Op2);
- break;
+ DelayedShuffles.push_back(
+ {OpTy, RTy, CurFullTy, Record[1], Record[2], Record[3]});
+ continue;
}
case bitc::CST_CODE_CE_CMP: { // CE_CMP: [opty, opval, opval, pred]
if (Record.size() < 4)
}
Record.push_back(VE.getValueID(C->getOperand(0)));
Record.push_back(VE.getValueID(C->getOperand(1)));
- Record.push_back(VE.getValueID(C->getOperand(2)));
+ Record.push_back(VE.getValueID(CE->getShuffleMaskForBitcode()));
break;
case Instruction::ICmp:
case Instruction::FCmp:
Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
pushValueAndType(I.getOperand(0), InstID, Vals);
pushValue(I.getOperand(1), InstID, Vals);
- pushValue(I.getOperand(2), InstID, Vals);
+ pushValue(cast<ShuffleVectorInst>(I).getShuffleMaskForBitcode(), InstID,
+ Vals);
break;
case Instruction::ICmp:
case Instruction::FCmp: {
if (OM.lookup(V).first)
return;
- if (const Constant *C = dyn_cast<Constant>(V))
- if (C->getNumOperands() && !isa<GlobalValue>(C))
+ if (const Constant *C = dyn_cast<Constant>(V)) {
+ if (C->getNumOperands() && !isa<GlobalValue>(C)) {
for (const Value *Op : C->operands())
if (!isa<BasicBlock>(Op) && !isa<GlobalValue>(Op))
orderValue(Op, OM);
+ if (auto *CE = dyn_cast<ConstantExpr>(C))
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ orderValue(CE->getShuffleMaskForBitcode(), OM);
+ }
+ }
// Note: we cannot cache this lookup above, since inserting into the map
// changes the map's size, and thus affects the other IDs.
for (const Argument &A : F.args())
orderValue(&A, OM);
for (const BasicBlock &BB : F)
- for (const Instruction &I : BB)
+ for (const Instruction &I : BB) {
for (const Value *Op : I.operands())
if ((isa<Constant>(*Op) && !isa<GlobalValue>(*Op)) ||
isa<InlineAsm>(*Op))
orderValue(Op, OM);
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+ orderValue(SVI->getShuffleMaskForBitcode(), OM);
+ }
for (const BasicBlock &BB : F)
for (const Instruction &I : BB)
orderValue(&I, OM);
predictValueUseListOrderImpl(V, F, IDPair.first, OM, Stack);
// Recursive descent into constants.
- if (const Constant *C = dyn_cast<Constant>(V))
- if (C->getNumOperands()) // Visit GlobalValues.
+ if (const Constant *C = dyn_cast<Constant>(V)) {
+ if (C->getNumOperands()) { // Visit GlobalValues.
for (const Value *Op : C->operands())
if (isa<Constant>(Op)) // Visit GlobalValues.
predictValueUseListOrder(Op, F, OM, Stack);
+ if (auto *CE = dyn_cast<ConstantExpr>(C))
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ predictValueUseListOrder(CE->getShuffleMaskForBitcode(), F, OM,
+ Stack);
+ }
+ }
}
static UseListOrderStack predictUseListOrder(const Module &M) {
for (const Argument &A : F.args())
predictValueUseListOrder(&A, &F, OM, Stack);
for (const BasicBlock &BB : F)
- for (const Instruction &I : BB)
+ for (const Instruction &I : BB) {
for (const Value *Op : I.operands())
if (isa<Constant>(*Op) || isa<InlineAsm>(*Op)) // Visit GlobalValues.
predictValueUseListOrder(Op, &F, OM, Stack);
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+ predictValueUseListOrder(SVI->getShuffleMaskForBitcode(), &F, OM,
+ Stack);
+ }
for (const BasicBlock &BB : F)
for (const Instruction &I : BB)
predictValueUseListOrder(&I, &F, OM, Stack);
EnumerateMetadata(&F, MD->getMetadata());
}
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+ EnumerateType(SVI->getShuffleMaskForBitcode()->getType());
EnumerateType(I.getType());
if (const auto *Call = dyn_cast<CallBase>(&I))
EnumerateAttributes(Call->getAttributes());
I != E; ++I)
if (!isa<BasicBlock>(*I)) // Don't enumerate BB operand to BlockAddress.
EnumerateValue(*I);
+ if (auto *CE = dyn_cast<ConstantExpr>(C))
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ EnumerateValue(CE->getShuffleMaskForBitcode());
// Finally, add the value. Doing this could make the ValueID reference be
// dangling, don't reuse it.
EnumerateOperandType(Op);
}
+ if (auto *CE = dyn_cast<ConstantExpr>(C))
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ EnumerateOperandType(CE->getShuffleMaskForBitcode());
}
void ValueEnumerator::EnumerateAttributes(AttributeList PAL) {
// Add all function-level constants to the value table.
for (const BasicBlock &BB : F) {
- for (const Instruction &I : BB)
+ for (const Instruction &I : BB) {
for (const Use &OI : I.operands()) {
if ((isa<Constant>(OI) && !isa<GlobalValue>(OI)) || isa<InlineAsm>(OI))
EnumerateValue(OI);
}
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+ EnumerateValue(SVI->getShuffleMaskForBitcode());
+ }
BasicBlocks.push_back(&BB);
ValueMap[&BB] = BasicBlocks.size();
}
}
static bool isBroadcastShuffle(ShuffleVectorInst *SVI) {
- SmallVector<int, 16> Mask(SVI->getShuffleMask());
+ ArrayRef<int> Mask(SVI->getShuffleMask());
int SplatElem = -1;
for (unsigned i = 0; i < Mask.size(); ++i) {
if (SplatElem != -1 && Mask[i] != -1 && Mask[i] != SplatElem)
assert(InsertPt != UserBB->end());
InsertedShuffle =
new ShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1),
- SVI->getOperand(2), "", &*InsertPt);
+ SVI->getShuffleMask(), "", &*InsertPt);
InsertedShuffle->setDebugLoc(SVI->getDebugLoc());
}
bool IRTranslator::translateShuffleVector(const User &U,
MachineIRBuilder &MIRBuilder) {
- SmallVector<int, 8> Mask;
- ShuffleVectorInst::getShuffleMask(cast<Constant>(U.getOperand(2)), Mask);
+ ArrayRef<int> Mask;
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(&U))
+ Mask = SVI->getShuffleMask();
+ else
+ Mask = cast<ConstantExpr>(U).getShuffleMask();
ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
MIRBuilder
.buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {getOrCreateVReg(U)},
void SelectionDAGBuilder::visitShuffleVector(const User &I) {
SDValue Src1 = getValue(I.getOperand(0));
SDValue Src2 = getValue(I.getOperand(1));
- Constant *MaskV = cast<Constant>(I.getOperand(2));
+ ArrayRef<int> Mask;
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
+ Mask = SVI->getShuffleMask();
+ else
+ Mask = cast<ConstantExpr>(I).getShuffleMask();
SDLoc DL = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
EVT SrcVT = Src1.getValueType();
unsigned SrcNumElts = SrcVT.getVectorNumElements();
- if (MaskV->isNullValue() && VT.isScalableVector()) {
+ if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
+ VT.isScalableVector()) {
// Canonical splat form of first element of first input vector.
SDValue FirstElt =
DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,
// for targets that support a SPLAT_VECTOR for non-scalable vector types.
assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
- SmallVector<int, 8> Mask;
- ShuffleVectorInst::getShuffleMask(MaskV, Mask);
unsigned MaskNumElts = Mask.size();
if (SrcNumElts == MaskNumElts) {
GenericValue Src1 = getOperandValue(I.getOperand(0), SF);
GenericValue Src2 = getOperandValue(I.getOperand(1), SF);
- GenericValue Src3 = getOperandValue(I.getOperand(2), SF);
GenericValue Dest;
// There is no need to check types of src1 and src2, because the compiled
Type *TyContained = Ty->getElementType();
unsigned src1Size = (unsigned)Src1.AggregateVal.size();
unsigned src2Size = (unsigned)Src2.AggregateVal.size();
- unsigned src3Size = (unsigned)Src3.AggregateVal.size();
+ unsigned src3Size = I.getShuffleMask().size();
Dest.AggregateVal.resize(src3Size);
break;
case Type::IntegerTyID:
for( unsigned i=0; i<src3Size; i++) {
- unsigned j = Src3.AggregateVal[i].IntVal.getZExtValue();
+ unsigned j = std::max(0, I.getMaskValue(i));
if(j < src1Size)
Dest.AggregateVal[i].IntVal = Src1.AggregateVal[j].IntVal;
else if(j < src1Size + src2Size)
break;
case Type::FloatTyID:
for( unsigned i=0; i<src3Size; i++) {
- unsigned j = Src3.AggregateVal[i].IntVal.getZExtValue();
+ unsigned j = std::max(0, I.getMaskValue(i));
if(j < src1Size)
Dest.AggregateVal[i].FloatVal = Src1.AggregateVal[j].FloatVal;
else if(j < src1Size + src2Size)
break;
case Type::DoubleTyID:
for( unsigned i=0; i<src3Size; i++) {
- unsigned j = Src3.AggregateVal[i].IntVal.getZExtValue();
+ unsigned j = std::max(0, I.getMaskValue(i));
if(j < src1Size)
Dest.AggregateVal[i].DoubleVal = Src1.AggregateVal[j].DoubleVal;
else if(j < src1Size + src2Size)
isa<GlobalValue>(V) ? GlobalPrefix : LocalPrefix);
}
+static void PrintShuffleMask(raw_ostream &Out, Type *Ty, ArrayRef<int> Mask) {
+ Out << ", <";
+ if (Ty->getVectorIsScalable())
+ Out << "vscale x ";
+ Out << Mask.size() << " x i32> ";
+ bool FirstElt = true;
+ if (all_of(Mask, [](int Elt) { return Elt == 0; })) {
+ Out << "zeroinitializer";
+ } else if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) {
+ Out << "undef";
+ } else {
+ Out << "<";
+ for (int Elt : Mask) {
+ if (FirstElt)
+ FirstElt = false;
+ else
+ Out << ", ";
+ Out << "i32 ";
+ if (Elt == UndefMaskElem)
+ Out << "undef";
+ else
+ Out << Elt;
+ }
+ Out << ">";
+ }
+}
+
namespace {
class TypePrinting {
TypePrinter.print(CE->getType(), Out);
}
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ PrintShuffleMask(Out, CE->getType(), CE->getShuffleMask());
+
Out << ')';
return;
}
RMWI->getSyncScopeID());
} else if (const FenceInst *FI = dyn_cast<FenceInst>(&I)) {
writeAtomic(FI->getContext(), FI->getOrdering(), FI->getSyncScopeID());
+ } else if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(&I)) {
+ PrintShuffleMask(Out, SVI->getType(), SVI->getShuffleMask());
}
// Print Metadata info.
PointerType::getUnqual(VT));
Value *Load = Builder.CreateAlignedLoad(VT, Op, Align(1));
if (NumSrcElts == 2)
- Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
- { 0, 1, 0, 1 });
+ Rep = Builder.CreateShuffleVector(
+ Load, UndefValue::get(Load->getType()), ArrayRef<int>{0, 1, 0, 1});
else
- Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
- { 0, 1, 2, 3, 0, 1, 2, 3 });
+ Rep =
+ Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
+ ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
} else if (IsX86 && (Name.startswith("avx512.mask.shuf.i") ||
Name.startswith("avx512.mask.shuf.f"))) {
unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
return ConstantVector::get(Result);
}
-Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1,
- Constant *V2,
- Constant *Mask) {
- ElementCount MaskEltCount = Mask->getType()->getVectorElementCount();
+Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
+ ArrayRef<int> Mask) {
+ unsigned MaskNumElts = Mask.size();
+ ElementCount MaskEltCount = {MaskNumElts,
+ V1->getType()->getVectorIsScalable()};
Type *EltTy = V1->getType()->getVectorElementType();
// Undefined shuffle mask -> undefined value.
- if (isa<UndefValue>(Mask))
- return UndefValue::get(VectorType::get(EltTy, MaskEltCount));
-
- // Don't break the bitcode reader hack.
- if (isa<ConstantExpr>(Mask)) return nullptr;
+ if (all_of(Mask, [](int Elt) { return Elt == UndefMaskElem; })) {
+ return UndefValue::get(VectorType::get(EltTy, MaskNumElts));
+ }
// If the mask is all zeros this is a splat, no need to go through all
// elements.
- if (isa<ConstantAggregateZero>(Mask) && !MaskEltCount.Scalable) {
+ if (all_of(Mask, [](int Elt) { return Elt == 0; }) &&
+ !MaskEltCount.Scalable) {
Type *Ty = IntegerType::get(V1->getContext(), 32);
Constant *Elt =
ConstantExpr::getExtractElement(V1, ConstantInt::get(Ty, 0));
if (ValTy->isScalable())
return nullptr;
- unsigned MaskNumElts = MaskEltCount.Min;
unsigned SrcNumElts = V1->getType()->getVectorNumElements();
// Loop over the shuffle mask, evaluating each element.
SmallVector<Constant*, 32> Result;
for (unsigned i = 0; i != MaskNumElts; ++i) {
- int Elt = ShuffleVectorInst::getMaskValue(Mask, i);
+ int Elt = Mask[i];
if (Elt == -1) {
Result.push_back(UndefValue::get(EltTy));
continue;
Constant *ConstantFoldInsertElementInstruction(Constant *Val, Constant *Elt,
Constant *Idx);
Constant *ConstantFoldShuffleVectorInstruction(Constant *V1, Constant *V2,
- Constant *Mask);
+ ArrayRef<int> Mask);
Constant *ConstantFoldExtractValueInstruction(Constant *Agg,
ArrayRef<unsigned> Idxs);
Constant *ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val,
Constant *UndefV = UndefValue::get(VTy);
V = ConstantExpr::getInsertElement(UndefV, V, ConstantInt::get(I32Ty, 0));
// Build shuffle mask to perform the splat.
- Type *MaskTy = VectorType::get(I32Ty, EC);
- Constant *Zeros = ConstantAggregateZero::get(MaskTy);
+ SmallVector<int, 8> Zeros(EC.Min, 0);
// Splat.
return ConstantExpr::getShuffleVector(V, UndefV, Zeros);
}
return cast<CompareConstantExpr>(this)->predicate;
}
+ArrayRef<int> ConstantExpr::getShuffleMask() const {
+ return cast<ShuffleVectorConstantExpr>(this)->ShuffleMask;
+}
+
+Constant *ConstantExpr::getShuffleMaskForBitcode() const {
+ return cast<ShuffleVectorConstantExpr>(this)->ShuffleMaskForBitcode;
+}
+
Constant *
ConstantExpr::getWithOperandReplaced(unsigned OpNo, Constant *Op) const {
assert(Op->getType() == getOperand(OpNo)->getType() &&
case Instruction::ExtractValue:
return ConstantExpr::getExtractValue(Ops[0], getIndices(), OnlyIfReducedTy);
case Instruction::ShuffleVector:
- return ConstantExpr::getShuffleVector(Ops[0], Ops[1], Ops[2],
+ return ConstantExpr::getShuffleVector(Ops[0], Ops[1], getShuffleMask(),
OnlyIfReducedTy);
case Instruction::GetElementPtr: {
auto *GEPO = cast<GEPOperator>(this);
if (InRangeIndex && *InRangeIndex < 63)
SubClassOptionalData |= (*InRangeIndex + 1) << 1;
const ConstantExprKeyType Key(Instruction::GetElementPtr, ArgVec, 0,
- SubClassOptionalData, None, Ty);
+ SubClassOptionalData, None, None, Ty);
LLVMContextImpl *pImpl = C->getContext().pImpl;
return pImpl->ExprConstants.getOrCreate(ReqTy, Key);
}
Constant *ConstantExpr::getShuffleVector(Constant *V1, Constant *V2,
- Constant *Mask, Type *OnlyIfReducedTy) {
+ ArrayRef<int> Mask,
+ Type *OnlyIfReducedTy) {
assert(ShuffleVectorInst::isValidOperands(V1, V2, Mask) &&
"Invalid shuffle vector constant expr operands!");
if (Constant *FC = ConstantFoldShuffleVectorInstruction(V1, V2, Mask))
return FC; // Fold a few common cases.
- ElementCount NElts = Mask->getType()->getVectorElementCount();
+ unsigned NElts = Mask.size();
Type *EltTy = V1->getType()->getVectorElementType();
- Type *ShufTy = VectorType::get(EltTy, NElts);
+ bool TypeIsScalable = V1->getType()->getVectorIsScalable();
+ Type *ShufTy = VectorType::get(EltTy, NElts, TypeIsScalable);
if (OnlyIfReducedTy == ShufTy)
return nullptr;
// Look up the constant in the table first to ensure uniqueness
- Constant *ArgVec[] = { V1, V2, Mask };
- const ConstantExprKeyType Key(Instruction::ShuffleVector, ArgVec);
+ Constant *ArgVec[] = {V1, V2};
+ ConstantExprKeyType Key(Instruction::ShuffleVector, ArgVec, 0, 0, None, Mask);
LLVMContextImpl *pImpl = ShufTy->getContext().pImpl;
return pImpl->ExprConstants.getOrCreate(ShufTy, Key);
case Instruction::ExtractValue:
return ExtractValueInst::Create(Ops[0], getIndices());
case Instruction::ShuffleVector:
- return new ShuffleVectorInst(Ops[0], Ops[1], Ops[2]);
+ return new ShuffleVectorInst(Ops[0], Ops[1], getShuffleMask());
case Instruction::GetElementPtr: {
const auto *GO = cast<GEPOperator>(this);
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
#include "llvm/IR/OperandTraits.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
/// shufflevector constant exprs.
class ShuffleVectorConstantExpr : public ConstantExpr {
public:
- ShuffleVectorConstantExpr(Constant *C1, Constant *C2, Constant *C3)
- : ConstantExpr(VectorType::get(
- cast<VectorType>(C1->getType())->getElementType(),
- cast<VectorType>(C3->getType())->getElementCount()),
- Instruction::ShuffleVector,
- &Op<0>(), 3) {
+ ShuffleVectorConstantExpr(Constant *C1, Constant *C2, ArrayRef<int> Mask)
+ : ConstantExpr(
+ VectorType::get(cast<VectorType>(C1->getType())->getElementType(),
+ Mask.size(), C1->getType()->getVectorIsScalable()),
+ Instruction::ShuffleVector, &Op<0>(), 2) {
+ assert(ShuffleVectorInst::isValidOperands(C1, C2, Mask) &&
+ "Invalid shuffle vector instruction operands!");
Op<0>() = C1;
Op<1>() = C2;
- Op<2>() = C3;
+ ShuffleMask.assign(Mask.begin(), Mask.end());
+ ShuffleMaskForBitcode =
+ ShuffleVectorInst::convertShuffleMaskForBitcode(Mask, getType());
}
- // allocate space for exactly three operands
- void *operator new(size_t s) {
- return User::operator new(s, 3);
- }
+ SmallVector<int, 4> ShuffleMask;
+ Constant *ShuffleMaskForBitcode;
+
+ void *operator new(size_t s) { return User::operator new(s, 2); }
/// Transparently provide more efficient getOperand methods.
DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
template <>
struct OperandTraits<ShuffleVectorConstantExpr>
- : public FixedNumOperandTraits<ShuffleVectorConstantExpr, 3> {};
+ : public FixedNumOperandTraits<ShuffleVectorConstantExpr, 2> {};
DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ShuffleVectorConstantExpr, Value)
template <>
};
struct ConstantExprKeyType {
+private:
uint8_t Opcode;
uint8_t SubclassOptionalData;
uint16_t SubclassData;
ArrayRef<Constant *> Ops;
ArrayRef<unsigned> Indexes;
+ ArrayRef<int> ShuffleMask;
Type *ExplicitTy;
+ static ArrayRef<int> getShuffleMaskIfValid(const ConstantExpr *CE) {
+ if (CE->getOpcode() == Instruction::ShuffleVector)
+ return CE->getShuffleMask();
+ return None;
+ }
+
+ static ArrayRef<unsigned> getIndicesIfValid(const ConstantExpr *CE) {
+ if (CE->hasIndices())
+ return CE->getIndices();
+ return None;
+ }
+
+ static Type *getSourceElementTypeIfValid(const ConstantExpr *CE) {
+ if (auto *GEPCE = dyn_cast<GetElementPtrConstantExpr>(CE))
+ return GEPCE->getSourceElementType();
+ return nullptr;
+ }
+
+public:
ConstantExprKeyType(unsigned Opcode, ArrayRef<Constant *> Ops,
unsigned short SubclassData = 0,
unsigned short SubclassOptionalData = 0,
ArrayRef<unsigned> Indexes = None,
+ ArrayRef<int> ShuffleMask = None,
Type *ExplicitTy = nullptr)
: Opcode(Opcode), SubclassOptionalData(SubclassOptionalData),
SubclassData(SubclassData), Ops(Ops), Indexes(Indexes),
- ExplicitTy(ExplicitTy) {}
+ ShuffleMask(ShuffleMask), ExplicitTy(ExplicitTy) {}
ConstantExprKeyType(ArrayRef<Constant *> Operands, const ConstantExpr *CE)
: Opcode(CE->getOpcode()),
SubclassOptionalData(CE->getRawSubclassOptionalData()),
SubclassData(CE->isCompare() ? CE->getPredicate() : 0), Ops(Operands),
- Indexes(CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()),
- ExplicitTy(nullptr) {}
+ Indexes(getIndicesIfValid(CE)), ShuffleMask(getShuffleMaskIfValid(CE)),
+ ExplicitTy(getSourceElementTypeIfValid(CE)) {}
ConstantExprKeyType(const ConstantExpr *CE,
SmallVectorImpl<Constant *> &Storage)
: Opcode(CE->getOpcode()),
SubclassOptionalData(CE->getRawSubclassOptionalData()),
SubclassData(CE->isCompare() ? CE->getPredicate() : 0),
- Indexes(CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()),
- ExplicitTy(nullptr) {
+ Indexes(getIndicesIfValid(CE)), ShuffleMask(getShuffleMaskIfValid(CE)),
+ ExplicitTy(getSourceElementTypeIfValid(CE)) {
assert(Storage.empty() && "Expected empty storage");
for (unsigned I = 0, E = CE->getNumOperands(); I != E; ++I)
Storage.push_back(CE->getOperand(I));
bool operator==(const ConstantExprKeyType &X) const {
return Opcode == X.Opcode && SubclassData == X.SubclassData &&
SubclassOptionalData == X.SubclassOptionalData && Ops == X.Ops &&
- Indexes == X.Indexes;
+ Indexes == X.Indexes && ShuffleMask == X.ShuffleMask &&
+ ExplicitTy == X.ExplicitTy;
}
bool operator==(const ConstantExpr *CE) const {
for (unsigned I = 0, E = Ops.size(); I != E; ++I)
if (Ops[I] != CE->getOperand(I))
return false;
- if (Indexes != (CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()))
+ if (Indexes != getIndicesIfValid(CE))
+ return false;
+ if (ShuffleMask != getShuffleMaskIfValid(CE))
+ return false;
+ if (ExplicitTy != getSourceElementTypeIfValid(CE))
return false;
return true;
}
unsigned getHash() const {
- return hash_combine(Opcode, SubclassOptionalData, SubclassData,
- hash_combine_range(Ops.begin(), Ops.end()),
- hash_combine_range(Indexes.begin(), Indexes.end()));
+ return hash_combine(
+ Opcode, SubclassOptionalData, SubclassData,
+ hash_combine_range(Ops.begin(), Ops.end()),
+ hash_combine_range(Indexes.begin(), Indexes.end()),
+ hash_combine_range(ShuffleMask.begin(), ShuffleMask.end()), ExplicitTy);
}
using TypeClass = ConstantInfo<ConstantExpr>::TypeClass;
case Instruction::InsertElement:
return new InsertElementConstantExpr(Ops[0], Ops[1], Ops[2]);
case Instruction::ShuffleVector:
- return new ShuffleVectorConstantExpr(Ops[0], Ops[1], Ops[2]);
+ return new ShuffleVectorConstantExpr(Ops[0], Ops[1], ShuffleMask);
case Instruction::InsertValue:
return new InsertValueConstantExpr(Ops[0], Ops[1], Indexes, Ty);
case Instruction::ExtractValue:
return new ExtractValueConstantExpr(Ops[0], Indexes, Ty);
case Instruction::GetElementPtr:
- return GetElementPtrConstantExpr::Create(
- ExplicitTy ? ExplicitTy
- : cast<PointerType>(Ops[0]->getType()->getScalarType())
- ->getElementType(),
- Ops[0], Ops.slice(1), Ty, SubclassOptionalData);
+ return GetElementPtrConstantExpr::Create(ExplicitTy, Ops[0], Ops.slice(1),
+ Ty, SubclassOptionalData);
case Instruction::ICmp:
return new CompareConstantExpr(Ty, Instruction::ICmp, SubclassData,
Ops[0], Ops[1]);
LLVMValueRef LLVMConstShuffleVector(LLVMValueRef VectorAConstant,
LLVMValueRef VectorBConstant,
LLVMValueRef MaskConstant) {
+ SmallVector<int, 16> IntMask;
+ ShuffleVectorInst::getShuffleMask(unwrap<Constant>(MaskConstant), IntMask);
return wrap(ConstantExpr::getShuffleVector(unwrap<Constant>(VectorAConstant),
unwrap<Constant>(VectorBConstant),
- unwrap<Constant>(MaskConstant)));
+ IntMask));
}
LLVMValueRef LLVMConstExtractValue(LLVMValueRef AggConstant, unsigned *IdxList,
RMWI->isVolatile() == cast<AtomicRMWInst>(I2)->isVolatile() &&
RMWI->getOrdering() == cast<AtomicRMWInst>(I2)->getOrdering() &&
RMWI->getSyncScopeID() == cast<AtomicRMWInst>(I2)->getSyncScopeID();
+ if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I1))
+ return SVI->getShuffleMask() ==
+ cast<ShuffleVectorInst>(I2)->getShuffleMask();
return true;
}
ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
const Twine &Name,
Instruction *InsertBefore)
-: Instruction(VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
- cast<VectorType>(Mask->getType())->getElementCount()),
- ShuffleVector,
- OperandTraits<ShuffleVectorInst>::op_begin(this),
- OperandTraits<ShuffleVectorInst>::operands(this),
- InsertBefore) {
+ : Instruction(
+ VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
+ cast<VectorType>(Mask->getType())->getElementCount()),
+ ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this),
+ OperandTraits<ShuffleVectorInst>::operands(this), InsertBefore) {
assert(isValidOperands(V1, V2, Mask) &&
"Invalid shuffle vector instruction operands!");
+
Op<0>() = V1;
Op<1>() = V2;
- Op<2>() = Mask;
+ SmallVector<int, 16> MaskArr;
+ getShuffleMask(cast<Constant>(Mask), MaskArr);
+ setShuffleMask(MaskArr);
setName(Name);
}
ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, Value *Mask,
+ const Twine &Name, BasicBlock *InsertAtEnd)
+ : Instruction(
+ VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
+ cast<VectorType>(Mask->getType())->getElementCount()),
+ ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this),
+ OperandTraits<ShuffleVectorInst>::operands(this), InsertAtEnd) {
+ assert(isValidOperands(V1, V2, Mask) &&
+ "Invalid shuffle vector instruction operands!");
+
+ Op<0>() = V1;
+ Op<1>() = V2;
+ SmallVector<int, 16> MaskArr;
+ getShuffleMask(cast<Constant>(Mask), MaskArr);
+ setShuffleMask(MaskArr);
+ setName(Name);
+}
+
+ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
const Twine &Name,
- BasicBlock *InsertAtEnd)
-: Instruction(VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
- cast<VectorType>(Mask->getType())->getElementCount()),
- ShuffleVector,
- OperandTraits<ShuffleVectorInst>::op_begin(this),
- OperandTraits<ShuffleVectorInst>::operands(this),
- InsertAtEnd) {
+ Instruction *InsertBefore)
+ : Instruction(
+ VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
+ Mask.size(), V1->getType()->getVectorIsScalable()),
+ ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this),
+ OperandTraits<ShuffleVectorInst>::operands(this), InsertBefore) {
+ assert(isValidOperands(V1, V2, Mask) &&
+ "Invalid shuffle vector instruction operands!");
+ Op<0>() = V1;
+ Op<1>() = V2;
+ setShuffleMask(Mask);
+ setName(Name);
+}
+
+ShuffleVectorInst::ShuffleVectorInst(Value *V1, Value *V2, ArrayRef<int> Mask,
+ const Twine &Name, BasicBlock *InsertAtEnd)
+ : Instruction(
+ VectorType::get(cast<VectorType>(V1->getType())->getElementType(),
+ Mask.size(), V1->getType()->getVectorIsScalable()),
+ ShuffleVector, OperandTraits<ShuffleVectorInst>::op_begin(this),
+ OperandTraits<ShuffleVectorInst>::operands(this), InsertAtEnd) {
assert(isValidOperands(V1, V2, Mask) &&
"Invalid shuffle vector instruction operands!");
Op<0>() = V1;
Op<1>() = V2;
- Op<2>() = Mask;
+ setShuffleMask(Mask);
setName(Name);
}
void ShuffleVectorInst::commute() {
int NumOpElts = Op<0>()->getType()->getVectorNumElements();
- int NumMaskElts = getMask()->getType()->getVectorNumElements();
- SmallVector<Constant*, 16> NewMask(NumMaskElts);
- Type *Int32Ty = Type::getInt32Ty(getContext());
+ int NumMaskElts = ShuffleMask.size();
+ SmallVector<int, 16> NewMask(NumMaskElts);
for (int i = 0; i != NumMaskElts; ++i) {
int MaskElt = getMaskValue(i);
- if (MaskElt == -1) {
- NewMask[i] = UndefValue::get(Int32Ty);
+ if (MaskElt == UndefMaskElem) {
+ NewMask[i] = UndefMaskElem;
continue;
}
assert(MaskElt >= 0 && MaskElt < 2 * NumOpElts && "Out-of-range mask");
MaskElt = (MaskElt < NumOpElts) ? MaskElt + NumOpElts : MaskElt - NumOpElts;
- NewMask[i] = ConstantInt::get(Int32Ty, MaskElt);
+ NewMask[i] = MaskElt;
}
- Op<2>() = ConstantVector::get(NewMask);
+ setShuffleMask(NewMask);
Op<0>().swap(Op<1>());
}
bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
+ ArrayRef<int> Mask) {
+ // V1 and V2 must be vectors of the same type.
+ if (!V1->getType()->isVectorTy() || V1->getType() != V2->getType())
+ return false;
+
+ // Make sure the mask elements make sense.
+ int V1Size = cast<VectorType>(V1->getType())->getNumElements();
+ for (int Elem : Mask)
+ if (Elem != UndefMaskElem && Elem >= V1Size * 2)
+ return false;
+
+ if (V1->getType()->getVectorIsScalable())
+ if ((Mask[0] != 0 && Mask[0] != UndefMaskElem) || !is_splat(Mask))
+ return false;
+
+ return true;
+}
+
+bool ShuffleVectorInst::isValidOperands(const Value *V1, const Value *V2,
const Value *Mask) {
// V1 and V2 must be vectors of the same type.
if (!V1->getType()->isVectorTy() || V1->getType() != V2->getType())
// Mask must be vector of i32.
auto *MaskTy = dyn_cast<VectorType>(Mask->getType());
- if (!MaskTy || !MaskTy->getElementType()->isIntegerTy(32))
+ if (!MaskTy || !MaskTy->getElementType()->isIntegerTy(32) ||
+ MaskTy->isScalable() != V1->getType()->getVectorIsScalable())
return false;
// Check to see if Mask is valid.
return true;
}
- // The bitcode reader can create a place holder for a forward reference
- // used as the shuffle mask. When this occurs, the shuffle mask will
- // fall into this case and fail. To avoid this error, do this bit of
- // ugliness to allow such a mask pass.
- if (const auto *CE = dyn_cast<ConstantExpr>(Mask))
- if (CE->getOpcode() == Instruction::UserOp1)
- return true;
-
return false;
}
-int ShuffleVectorInst::getMaskValue(const Constant *Mask, unsigned i) {
- assert(i < Mask->getType()->getVectorNumElements() && "Index out of range");
- assert(!Mask->getType()->getVectorElementCount().Scalable &&
- "Length of scalable vectors unknown at compile time");
- if (auto *CDS = dyn_cast<ConstantDataSequential>(Mask))
- return CDS->getElementAsInteger(i);
- Constant *C = Mask->getAggregateElement(i);
- if (isa<UndefValue>(C))
- return -1;
- return cast<ConstantInt>(C)->getZExtValue();
-}
-
void ShuffleVectorInst::getShuffleMask(const Constant *Mask,
SmallVectorImpl<int> &Result) {
- assert(!Mask->getType()->getVectorElementCount().Scalable &&
- "Length of scalable vectors unknown at compile time");
- unsigned NumElts = Mask->getType()->getVectorNumElements();
+ unsigned NumElts = Mask->getType()->getVectorElementCount().Min;
if (isa<ConstantAggregateZero>(Mask)) {
Result.resize(NumElts, 0);
return;
}
}
+void ShuffleVectorInst::setShuffleMask(ArrayRef<int> Mask) {
+ ShuffleMask.assign(Mask.begin(), Mask.end());
+ ShuffleMaskForBitcode = convertShuffleMaskForBitcode(Mask, getType());
+}
+Constant *ShuffleVectorInst::convertShuffleMaskForBitcode(ArrayRef<int> Mask,
+ Type *ResultTy) {
+ Type *Int32Ty = Type::getInt32Ty(ResultTy->getContext());
+ if (ResultTy->getVectorIsScalable()) {
+ assert(is_splat(Mask) && "Unexpected shuffle");
+ Type *VecTy = VectorType::get(Int32Ty, Mask.size(), true);
+ if (Mask[0] == 0)
+ return Constant::getNullValue(VecTy);
+ return UndefValue::get(VecTy);
+ }
+ SmallVector<Constant *, 16> MaskConst;
+ for (int Elem : Mask) {
+ if (Elem == UndefMaskElem)
+ MaskConst.push_back(UndefValue::get(Int32Ty));
+ else
+ MaskConst.push_back(ConstantInt::get(Int32Ty, Elem));
+ }
+ return ConstantVector::get(MaskConst);
+}
+
static bool isSingleSourceMaskImpl(ArrayRef<int> Mask, int NumOpElts) {
assert(!Mask.empty() && "Shuffle mask must contain elements");
bool UsesLHS = false;
return false;
// The first part of the mask must choose elements from exactly 1 source op.
- SmallVector<int, 16> Mask = getShuffleMask();
+ ArrayRef<int> Mask = getShuffleMask();
if (!isIdentityMaskImpl(Mask, NumOpElts))
return false;
}
ShuffleVectorInst *ShuffleVectorInst::cloneImpl() const {
- return new ShuffleVectorInst(getOperand(0), getOperand(1), getOperand(2));
+ return new ShuffleVectorInst(getOperand(0), getOperand(1), getShuffleMask());
}
PHINode *PHINode::cloneImpl() const { return new PHINode(*this); }
void Verifier::visitShuffleVectorInst(ShuffleVectorInst &SV) {
Assert(ShuffleVectorInst::isValidOperands(SV.getOperand(0), SV.getOperand(1),
- SV.getOperand(2)),
+ SV.getShuffleMask()),
"Invalid shufflevector operands!", &SV);
visitInstruction(SV);
}
return FullVT->getNumElements() == 2 * HalfVT->getNumElements();
};
- Constant *M1, *M2;
+ ArrayRef<int> M1, M2;
Value *S1Op1, *S2Op1;
- if (!match(Op1, m_ShuffleVector(m_Value(S1Op1), m_Undef(), m_Constant(M1))) ||
- !match(Op2, m_ShuffleVector(m_Value(S2Op1), m_Undef(), m_Constant(M2))))
+ if (!match(Op1, m_ShuffleVector(m_Value(S1Op1), m_Undef(), m_Mask(M1))) ||
+ !match(Op2, m_ShuffleVector(m_Value(S2Op1), m_Undef(), m_Mask(M2))))
return false;
// Check that the operands are half as wide as the result and we extract
Arg.replaceAllUsesWith(NewVal);
} else if (IsV3) {
Value *Shuf = Builder.CreateShuffleVector(Load, UndefValue::get(V4Ty),
- {0, 1, 2},
+ ArrayRef<int>{0, 1, 2},
Arg.getName() + ".load");
Arg.replaceAllUsesWith(Shuf);
} else {
DL->getTypeSizeInBits(Val->getType())) {
assert(isVec3ToVec4Shuffle(EffectiveEltTy, Val->getType()));
Val = B.CreateShuffleVector(Val, UndefValue::get(Val->getType()),
- { 0, 1, 2 });
+ ArrayRef<int>{0, 1, 2});
}
Val = B.CreateBitCast(Val, EffectiveEltTy);
return false;
if (!match(I->getOperand(Op),
m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_ZeroInt()),
- m_Undef(), m_Zero()))) {
+ m_Undef(), m_ZeroMask()))) {
return false;
}
Instruction *Shuffle = cast<Instruction>(I->getOperand(Op));
Instruction *Insert = nullptr;
// The shuffle which broadcasts the index iv into a vector.
if (!match(BroadcastSplat,
- m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_Zero())))
+ m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_ZeroMask())))
return false;
// The insert element which initialises a vector with the index iv.
Instruction *Insert = nullptr;
if (!match(Shuffle,
- m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_Zero())))
+ m_ShuffleVector(m_Instruction(Insert), m_Undef(), m_ZeroMask())))
return false;
// Insert the limit into a vector.
NumElts = OpTy->getVectorNumElements();
if (NumElts == 2) {
// Extract down to 2 elements.
- Ops[0] = Builder.CreateShuffleVector(Ops[0], Ops[0], {0, 1});
+ Ops[0] = Builder.CreateShuffleVector(Ops[0], Ops[0], ArrayRef<int>{0, 1});
} else if (NumElts >= 8) {
SmallVector<uint32_t, 32> ConcatMask(NumElts);
unsigned SubElts = Ops[0]->getType()->getVectorNumElements();
InstCombiner::BuilderTy &Builder) {
auto *Shuf = dyn_cast<ShuffleVectorInst>(Trunc.getOperand(0));
if (Shuf && Shuf->hasOneUse() && isa<UndefValue>(Shuf->getOperand(1)) &&
- Shuf->getMask()->getSplatValue() &&
+ is_splat(Shuf->getShuffleMask()) &&
Shuf->getType() == Shuf->getOperand(0)->getType()) {
// trunc (shuf X, Undef, SplatMask) --> shuf (trunc X), Undef, SplatMask
Constant *NarrowUndef = UndefValue::get(Trunc.getType());
Value *NarrowOp = Builder.CreateTrunc(Shuf->getOperand(0), Trunc.getType());
- return new ShuffleVectorInst(NarrowOp, NarrowUndef, Shuf->getMask());
+ return new ShuffleVectorInst(NarrowOp, NarrowUndef, Shuf->getShuffleMask());
}
return nullptr;
Value *RHS = Builder.CreateBitCast(ShufOp1, DestTy);
// Return a new shuffle vector. Use the same element ID's, as we
// know the vector types match #elts.
- return new ShuffleVectorInst(LHS, RHS, Shuf->getOperand(2));
+ return new ShuffleVectorInst(LHS, RHS, Shuf->getShuffleMask());
}
}
return nullptr;
Value *Vec;
- Constant *Mask;
- if (match(BCSrcOp,
- m_ShuffleVector(m_Value(Vec), m_Undef(), m_Constant(Mask)))) {
+ ArrayRef<int> Mask;
+ if (match(BCSrcOp, m_ShuffleVector(m_Value(Vec), m_Undef(), m_Mask(Mask)))) {
// Check whether every element of Mask is the same constant
- if (auto *Elem = dyn_cast_or_null<ConstantInt>(Mask->getSplatValue())) {
+ if (is_splat(Mask)) {
auto *VecTy = cast<VectorType>(BCSrcOp->getType());
auto *EltTy = cast<IntegerType>(VecTy->getElementType());
if (C->isSplat(EltTy->getBitWidth())) {
// then:
// => %E = extractelement <N x iK> %vec, i32 Elem
// icmp <pred> iK %SplatVal, <pattern>
+ Value *Elem = Builder.getInt32(Mask[0]);
Value *Extract = Builder.CreateExtractElement(Vec, Elem);
Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth()));
return new ICmpInst(Pred, Extract, NewC);
bool IsFP = isa<FCmpInst>(Cmp);
Value *V1, *V2;
- Constant *M;
- if (!match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(M))))
+ ArrayRef<int> M;
+ if (!match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Mask(M))))
return nullptr;
// If both arguments of the cmp are shuffles that use the same mask and
// shuffle within a single vector, move the shuffle after the cmp:
// cmp (shuffle V1, M), (shuffle V2, M) --> shuffle (cmp V1, V2), M
Type *V1Ty = V1->getType();
- if (match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_Specific(M))) &&
+ if (match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_SpecificMask(M))) &&
V1Ty == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse())) {
Value *NewCmp = IsFP ? Builder.CreateFCmp(Pred, V1, V2)
: Builder.CreateICmp(Pred, V1, V2);
// Length-changing splats are ok, so adjust the constants as needed:
// cmp (shuffle V1, M), C --> shuffle (cmp V1, C'), M
Constant *ScalarC = C->getSplatValue(/* AllowUndefs */ true);
- Constant *ScalarM = M->getSplatValue(/* AllowUndefs */ true);
- if (ScalarC && ScalarM) {
+ int MaskSplatIndex;
+ if (ScalarC && match(M, m_SplatOrUndefMask(MaskSplatIndex))) {
// We allow undefs in matching, but this transform removes those for safety.
// Demanded elements analysis should be able to recover some/all of that.
C = ConstantVector::getSplat(V1Ty->getVectorElementCount(), ScalarC);
- M = ConstantVector::getSplat(M->getType()->getVectorElementCount(),
- ScalarM);
+ SmallVector<int, 8> NewM(M.size(), MaskSplatIndex);
Value *NewCmp = IsFP ? Builder.CreateFCmp(Pred, V1, C)
: Builder.CreateICmp(Pred, V1, C);
- return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()), M);
+ return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()),
+ NewM);
}
return nullptr;
Shuffle->getOperand(0)->getType()->getVectorNumElements();
// Handle trivial case of a splat. Only check the first element of LHS
// operand.
- if (isa<ConstantAggregateZero>(Shuffle->getMask()) &&
+ if (all_of(Shuffle->getShuffleMask(), [](int Elt) { return Elt == 0; }) &&
DemandedElts.isAllOnesValue()) {
if (!isa<UndefValue>(I->getOperand(1))) {
I->setOperand(1, UndefValue::get(I->getOperand(1)->getType()));
}
if (NewUndefElts) {
// Add additional discovered undefs.
- SmallVector<Constant*, 16> Elts;
+ SmallVector<int, 16> Elts;
for (unsigned i = 0; i < VWidth; ++i) {
if (UndefElts[i])
- Elts.push_back(UndefValue::get(Type::getInt32Ty(I->getContext())));
+ Elts.push_back(UndefMaskElem);
else
- Elts.push_back(ConstantInt::get(Type::getInt32Ty(I->getContext()),
- Shuffle->getMaskValue(i)));
+ Elts.push_back(Shuffle->getMaskValue(i));
}
- I->setOperand(2, ConstantVector::get(Elts));
+ Shuffle->setShuffleMask(Elts);
MadeChange = true;
}
break;
}
static bool isShuffleEquivalentToSelect(ShuffleVectorInst &Shuf) {
- int MaskSize = Shuf.getMask()->getType()->getVectorNumElements();
+ int MaskSize = Shuf.getShuffleMask().size();
int VecSize = Shuf.getOperand(0)->getType()->getVectorNumElements();
// A vector select does not change the size of the operands.
// inselt (shuf (inselt undef, X, 0), undef, <0,undef,0,undef>), X, 1
// --> shuf (inselt undef, X, 0), undef, <0,0,0,undef>
unsigned NumMaskElts = Shuf->getType()->getVectorNumElements();
- SmallVector<Constant *, 16> NewMaskVec(NumMaskElts);
- Type *I32Ty = IntegerType::getInt32Ty(Shuf->getContext());
- Constant *Zero = ConstantInt::getNullValue(I32Ty);
+ SmallVector<int, 16> NewMask(NumMaskElts);
for (unsigned i = 0; i != NumMaskElts; ++i)
- NewMaskVec[i] = i == IdxC ? Zero : Shuf->getMask()->getAggregateElement(i);
+ NewMask[i] = i == IdxC ? 0 : Shuf->getMaskValue(i);
- Constant *NewMask = ConstantVector::get(NewMaskVec);
return new ShuffleVectorInst(Op0, UndefValue::get(Op0->getType()), NewMask);
}
// For example:
// inselt (shuf X, IdMask), (extelt X, IdxC), IdxC --> shuf X, IdMask'
unsigned NumMaskElts = Shuf->getType()->getVectorNumElements();
- SmallVector<Constant *, 16> NewMaskVec(NumMaskElts);
- Type *I32Ty = IntegerType::getInt32Ty(Shuf->getContext());
- Constant *NewMaskEltC = ConstantInt::get(I32Ty, IdxC);
- Constant *OldMask = Shuf->getMask();
+ SmallVector<int, 16> NewMask(NumMaskElts);
+ ArrayRef<int> OldMask = Shuf->getShuffleMask();
for (unsigned i = 0; i != NumMaskElts; ++i) {
if (i != IdxC) {
// All mask elements besides the inserted element remain the same.
- NewMaskVec[i] = OldMask->getAggregateElement(i);
- } else if (OldMask->getAggregateElement(i) == NewMaskEltC) {
+ NewMask[i] = OldMask[i];
+ } else if (OldMask[i] == (int)IdxC) {
// If the mask element was already set, there's nothing to do
// (demanded elements analysis may unset it later).
return nullptr;
} else {
- assert(isa<UndefValue>(OldMask->getAggregateElement(i)) &&
+ assert(OldMask[i] == UndefMaskElem &&
"Unexpected shuffle mask element for identity shuffle");
- NewMaskVec[i] = NewMaskEltC;
+ NewMask[i] = IdxC;
}
}
- Constant *NewMask = ConstantVector::get(NewMaskVec);
return new ShuffleVectorInst(X, Shuf->getOperand(1), NewMask);
}
// mask vector with the insertelt index plus the length of the vector
// (because the constant vector operand of a shuffle is always the 2nd
// operand).
- Constant *Mask = Shuf->getMask();
- unsigned NumElts = Mask->getType()->getVectorNumElements();
+ ArrayRef<int> Mask = Shuf->getShuffleMask();
+ unsigned NumElts = Mask.size();
SmallVector<Constant *, 16> NewShufElts(NumElts);
- SmallVector<Constant *, 16> NewMaskElts(NumElts);
+ SmallVector<int, 16> NewMaskElts(NumElts);
for (unsigned I = 0; I != NumElts; ++I) {
if (I == InsEltIndex) {
NewShufElts[I] = InsEltScalar;
- Type *Int32Ty = Type::getInt32Ty(Shuf->getContext());
- NewMaskElts[I] = ConstantInt::get(Int32Ty, InsEltIndex + NumElts);
+ NewMaskElts[I] = InsEltIndex + NumElts;
} else {
// Copy over the existing values.
NewShufElts[I] = ShufConstVec->getAggregateElement(I);
- NewMaskElts[I] = Mask->getAggregateElement(I);
+ NewMaskElts[I] = Mask[I];
}
}
// Create new operands for a shuffle that includes the constant of the
// original insertelt. The old shuffle will be dead now.
return new ShuffleVectorInst(Shuf->getOperand(0),
- ConstantVector::get(NewShufElts),
- ConstantVector::get(NewMaskElts));
+ ConstantVector::get(NewShufElts), NewMaskElts);
} else if (auto *IEI = dyn_cast<InsertElementInst>(Inst)) {
// Transform sequences of insertelements ops with constant data/indexes into
// a single shuffle op.
if (isa<ConstantAggregateZero>(V))
return ConstantAggregateZero::get(VectorType::get(EltTy, Mask.size()));
- if (Constant *C = dyn_cast<Constant>(V)) {
- SmallVector<Constant *, 16> MaskValues;
- for (int i = 0, e = Mask.size(); i != e; ++i) {
- if (Mask[i] == -1)
- MaskValues.push_back(UndefValue::get(I32Ty));
- else
- MaskValues.push_back(ConstantInt::get(I32Ty, Mask[i]));
- }
+ if (Constant *C = dyn_cast<Constant>(V))
return ConstantExpr::getShuffleVector(C, UndefValue::get(C->getType()),
- ConstantVector::get(MaskValues));
- }
+ Mask);
Instruction *I = cast<Instruction>(V);
switch (I->getOpcode()) {
// Shuffles to: |EE|FF|GG|HH|
// +--+--+--+--+
static bool isShuffleExtractingFromLHS(ShuffleVectorInst &SVI,
- SmallVector<int, 16> &Mask) {
+ ArrayRef<int> Mask) {
unsigned LHSElems = SVI.getOperand(0)->getType()->getVectorNumElements();
unsigned MaskElems = Mask.size();
unsigned BegIdx = Mask.front();
// Example: shuf (mul X, {-1,-2,-3,-4}), X, {0,5,6,3} --> mul X, {-1,1,1,-4}
// Example: shuf X, (add X, {-1,-2,-3,-4}), {0,1,6,7} --> add X, {0,0,-3,-4}
// The existing binop constant vector remains in the same operand position.
- Constant *Mask = Shuf.getMask();
+ ArrayRef<int> Mask = Shuf.getShuffleMask();
Constant *NewC = Op0IsBinop ? ConstantExpr::getShuffleVector(C, IdC, Mask) :
ConstantExpr::getShuffleVector(IdC, C, Mask);
bool MightCreatePoisonOrUB =
- Mask->containsUndefElement() &&
+ is_contained(Mask, UndefMaskElem) &&
(Instruction::isIntDivRem(BOpcode) || Instruction::isShift(BOpcode));
if (MightCreatePoisonOrUB)
NewC = getSafeVectorConstantForBinop(BOpcode, NewC, true);
// An undef shuffle mask element may propagate as an undef constant element in
// the new binop. That would produce poison where the original code might not.
// If we already made a safe constant, then there's no danger.
- if (Mask->containsUndefElement() && !MightCreatePoisonOrUB)
+ if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
NewBO->dropPoisonGeneratingFlags();
return NewBO;
}
static Instruction *canonicalizeInsertSplat(ShuffleVectorInst &Shuf,
InstCombiner::BuilderTy &Builder) {
Value *Op0 = Shuf.getOperand(0), *Op1 = Shuf.getOperand(1);
- Constant *Mask = Shuf.getMask();
+ ArrayRef<int> Mask = Shuf.getShuffleMask();
Value *X;
uint64_t IndexC;
// Match a shuffle that is a splat to a non-zero element.
if (!match(Op0, m_OneUse(m_InsertElement(m_Undef(), m_Value(X),
m_ConstantInt(IndexC)))) ||
- !match(Op1, m_Undef()) || match(Mask, m_ZeroInt()) || IndexC == 0)
+ !match(Op1, m_Undef()) || match(Mask, m_ZeroMask()) || IndexC == 0)
return nullptr;
// Insert into element 0 of an undef vector.
// shuf (inselt undef, X, 2), undef, <2,2,undef>
// --> shuf (inselt undef, X, 0), undef, <0,0,undef>
unsigned NumMaskElts = Shuf.getType()->getVectorNumElements();
- SmallVector<Constant *, 16> NewMask(NumMaskElts, Zero);
+ SmallVector<int, 16> NewMask(NumMaskElts, 0);
for (unsigned i = 0; i != NumMaskElts; ++i)
- if (isa<UndefValue>(Mask->getAggregateElement(i)))
- NewMask[i] = Mask->getAggregateElement(i);
+ if (Mask[i] == UndefMaskElem)
+ NewMask[i] = Mask[i];
- return new ShuffleVectorInst(NewIns, UndefVec, ConstantVector::get(NewMask));
+ return new ShuffleVectorInst(NewIns, UndefVec, NewMask);
}
/// Try to fold shuffles that are the equivalent of a vector select.
BinaryOperator::BinaryOps BOpc = Opc0;
// Select the constant elements needed for the single binop.
- Constant *Mask = Shuf.getMask();
+ ArrayRef<int> Mask = Shuf.getShuffleMask();
Constant *NewC = ConstantExpr::getShuffleVector(C0, C1, Mask);
// We are moving a binop after a shuffle. When a shuffle has an undefined
// mask element, the result is undefined, but it is not poison or undefined
// behavior. That is not necessarily true for div/rem/shift.
bool MightCreatePoisonOrUB =
- Mask->containsUndefElement() &&
+ is_contained(Mask, UndefMaskElem) &&
(Instruction::isIntDivRem(BOpc) || Instruction::isShift(BOpc));
if (MightCreatePoisonOrUB)
NewC = getSafeVectorConstantForBinop(BOpc, NewC, ConstantsAreOp1);
NewBO->andIRFlags(B1);
if (DropNSW)
NewBO->setHasNoSignedWrap(false);
- if (Mask->containsUndefElement() && !MightCreatePoisonOrUB)
+ if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
NewBO->dropPoisonGeneratingFlags();
return NewBO;
}
// and have the same number of elements as this shuffle.
unsigned NarrowNumElts = Shuf.getType()->getVectorNumElements();
Value *NarrowCond;
- if (!match(Cond, m_OneUse(m_ShuffleVector(m_Value(NarrowCond), m_Undef(),
- m_Constant()))) ||
+ if (!match(Cond, m_OneUse(m_ShuffleVector(m_Value(NarrowCond), m_Undef()))) ||
NarrowCond->getType()->getVectorNumElements() != NarrowNumElts ||
!cast<ShuffleVectorInst>(Cond)->isIdentityWithPadding())
return nullptr;
// shuf (sel (shuf NarrowCond, undef, WideMask), X, Y), undef, NarrowMask) -->
// sel NarrowCond, (shuf X, undef, NarrowMask), (shuf Y, undef, NarrowMask)
Value *Undef = UndefValue::get(X->getType());
- Value *NarrowX = Builder.CreateShuffleVector(X, Undef, Shuf.getMask());
- Value *NarrowY = Builder.CreateShuffleVector(Y, Undef, Shuf.getMask());
+ Value *NarrowX = Builder.CreateShuffleVector(X, Undef, Shuf.getShuffleMask());
+ Value *NarrowY = Builder.CreateShuffleVector(Y, Undef, Shuf.getShuffleMask());
return SelectInst::Create(NarrowCond, NarrowX, NarrowY);
}
return nullptr;
Value *X, *Y;
- Constant *Mask;
- if (!match(Op0, m_ShuffleVector(m_Value(X), m_Value(Y), m_Constant(Mask))))
+ ArrayRef<int> Mask;
+ if (!match(Op0, m_ShuffleVector(m_Value(X), m_Value(Y), m_Mask(Mask))))
return nullptr;
// Be conservative with shuffle transforms. If we can't kill the 1st shuffle,
// shuf (shuf X, Y, <C0, C1, C2, undef, C4>), undef, <0, undef, 2, 3> -->
// shuf X, Y, <C0, undef, C2, undef>
unsigned NumElts = Shuf.getType()->getVectorNumElements();
- SmallVector<Constant *, 16> NewMask(NumElts);
- assert(NumElts < Mask->getType()->getVectorNumElements() &&
+ SmallVector<int, 16> NewMask(NumElts);
+ assert(NumElts < Mask.size() &&
"Identity with extract must have less elements than its inputs");
for (unsigned i = 0; i != NumElts; ++i) {
- Constant *ExtractMaskElt = Shuf.getMask()->getAggregateElement(i);
- Constant *MaskElt = Mask->getAggregateElement(i);
- NewMask[i] = isa<UndefValue>(ExtractMaskElt) ? ExtractMaskElt : MaskElt;
+ int ExtractMaskElt = Shuf.getMaskValue(i);
+ int MaskElt = Mask[i];
+ NewMask[i] = ExtractMaskElt == UndefMaskElem ? ExtractMaskElt : MaskElt;
}
- return new ShuffleVectorInst(X, Y, ConstantVector::get(NewMask));
+ return new ShuffleVectorInst(X, Y, NewMask);
}
/// Try to replace a shuffle with an insertelement or try to replace a shuffle
static Instruction *foldShuffleWithInsert(ShuffleVectorInst &Shuf,
InstCombiner &IC) {
Value *V0 = Shuf.getOperand(0), *V1 = Shuf.getOperand(1);
- SmallVector<int, 16> Mask = Shuf.getShuffleMask();
+ SmallVector<int, 16> Mask;
+ Shuf.getShuffleMask(Mask);
// The shuffle must not change vector sizes.
// TODO: This restriction could be removed if the insert has only one use
assert(WideElts > NarrowElts && "Unexpected types for identity with padding");
Type *I32Ty = IntegerType::getInt32Ty(Shuf.getContext());
- SmallVector<int, 16> Mask = Shuf.getShuffleMask();
+ ArrayRef<int> Mask = Shuf.getShuffleMask();
SmallVector<Constant *, 16> NewMask(Mask.size(), UndefValue::get(I32Ty));
for (int i = 0, e = Mask.size(); i != e; ++i) {
if (Mask[i] == -1)
Instruction *InstCombiner::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
Value *LHS = SVI.getOperand(0);
Value *RHS = SVI.getOperand(1);
- if (auto *V = SimplifyShuffleVectorInst(
- LHS, RHS, SVI.getMask(), SVI.getType(), SQ.getWithInstruction(&SVI)))
+ if (auto *V =
+ SimplifyShuffleVectorInst(LHS, RHS, SVI.getShuffleMask(),
+ SVI.getType(), SQ.getWithInstruction(&SVI)))
return replaceInstUsesWith(SVI, V);
// shuffle x, x, mask --> shuffle x, undef, mask'
unsigned VWidth = SVI.getType()->getVectorNumElements();
unsigned LHSWidth = LHS->getType()->getVectorNumElements();
- SmallVector<int, 16> Mask = SVI.getShuffleMask();
+ ArrayRef<int> Mask = SVI.getShuffleMask();
Type *Int32Ty = Type::getInt32Ty(SVI.getContext());
if (LHS == RHS) {
assert(!isa<UndefValue>(RHS) && "Shuffle with 2 undef ops not simplified?");
// Remap any references to RHS to use LHS.
- SmallVector<Constant*, 16> Elts;
+ SmallVector<int, 16> Elts;
for (unsigned i = 0; i != VWidth; ++i) {
// Propagate undef elements or force mask to LHS.
if (Mask[i] < 0)
- Elts.push_back(UndefValue::get(Int32Ty));
+ Elts.push_back(UndefMaskElem);
else
- Elts.push_back(ConstantInt::get(Int32Ty, Mask[i] % LHSWidth));
+ Elts.push_back(Mask[i] % LHSWidth);
}
- return new ShuffleVectorInst(LHS, UndefValue::get(RHS->getType()),
- ConstantVector::get(Elts));
+ return new ShuffleVectorInst(LHS, UndefValue::get(RHS->getType()), Elts);
}
// shuffle undef, x, mask --> shuffle x, undef, mask'
if (newLHS == LHS && newRHS == RHS)
return MadeChange ? &SVI : nullptr;
- SmallVector<int, 16> LHSMask;
- SmallVector<int, 16> RHSMask;
+ ArrayRef<int> LHSMask;
+ ArrayRef<int> RHSMask;
if (newLHS != LHS)
LHSMask = LHSShuffle->getShuffleMask();
if (RHSShuffle && newRHS != RHS)
// narrow binop on each pair of the source operands followed by concatenation
// of the results.
Value *L0, *L1, *R0, *R1;
- Constant *Mask;
- if (match(LHS, m_ShuffleVector(m_Value(L0), m_Value(L1), m_Constant(Mask))) &&
- match(RHS, m_ShuffleVector(m_Value(R0), m_Value(R1), m_Specific(Mask))) &&
+ ArrayRef<int> Mask;
+ if (match(LHS, m_ShuffleVector(m_Value(L0), m_Value(L1), m_Mask(Mask))) &&
+ match(RHS,
+ m_ShuffleVector(m_Value(R0), m_Value(R1), m_SpecificMask(Mask))) &&
LHS->hasOneUse() && RHS->hasOneUse() &&
cast<ShuffleVectorInst>(LHS)->isConcat() &&
cast<ShuffleVectorInst>(RHS)->isConcat()) {
if (!isSafeToSpeculativelyExecute(&Inst))
return nullptr;
- auto createBinOpShuffle = [&](Value *X, Value *Y, Constant *M) {
+ auto createBinOpShuffle = [&](Value *X, Value *Y, ArrayRef<int> M) {
Value *XY = Builder.CreateBinOp(Opcode, X, Y);
if (auto *BO = dyn_cast<BinaryOperator>(XY))
BO->copyIRFlags(&Inst);
// If both arguments of the binary operation are shuffles that use the same
// mask and shuffle within a single vector, move the shuffle after the binop.
Value *V1, *V2;
- if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(Mask))) &&
- match(RHS, m_ShuffleVector(m_Value(V2), m_Undef(), m_Specific(Mask))) &&
+ if (match(LHS, m_ShuffleVector(m_Value(V1), m_Undef(), m_Mask(Mask))) &&
+ match(RHS,
+ m_ShuffleVector(m_Value(V2), m_Undef(), m_SpecificMask(Mask))) &&
V1->getType() == V2->getType() &&
(LHS->hasOneUse() || RHS->hasOneUse() || LHS == RHS)) {
// Op(shuffle(V1, Mask), shuffle(V2, Mask)) -> shuffle(Op(V1, V2), Mask)
// If both arguments of a commutative binop are select-shuffles that use the
// same mask with commuted operands, the shuffles are unnecessary.
if (Inst.isCommutative() &&
- match(LHS, m_ShuffleVector(m_Value(V1), m_Value(V2), m_Constant(Mask))) &&
+ match(LHS, m_ShuffleVector(m_Value(V1), m_Value(V2), m_Mask(Mask))) &&
match(RHS, m_ShuffleVector(m_Specific(V2), m_Specific(V1),
- m_Specific(Mask)))) {
+ m_SpecificMask(Mask)))) {
auto *LShuf = cast<ShuffleVectorInst>(LHS);
auto *RShuf = cast<ShuffleVectorInst>(RHS);
// TODO: Allow shuffles that contain undefs in the mask?
// That is legal, but it reduces undef knowledge.
// TODO: Allow arbitrary shuffles by shuffling after binop?
// That might be legal, but we have to deal with poison.
- if (LShuf->isSelect() && !LShuf->getMask()->containsUndefElement() &&
- RShuf->isSelect() && !RShuf->getMask()->containsUndefElement()) {
+ if (LShuf->isSelect() &&
+ !is_contained(LShuf->getShuffleMask(), UndefMaskElem) &&
+ RShuf->isSelect() &&
+ !is_contained(RShuf->getShuffleMask(), UndefMaskElem)) {
// Example:
// LHS = shuffle V1, V2, <0, 5, 6, 3>
// RHS = shuffle V2, V1, <0, 5, 6, 3>
// other binops, so they can be folded. It may also enable demanded elements
// transforms.
Constant *C;
- if (match(&Inst, m_c_BinOp(
- m_OneUse(m_ShuffleVector(m_Value(V1), m_Undef(), m_Constant(Mask))),
- m_Constant(C))) &&
+ if (match(&Inst, m_c_BinOp(m_OneUse(m_ShuffleVector(m_Value(V1), m_Undef(),
+ m_Mask(Mask))),
+ m_Constant(C))) &&
V1->getType()->getVectorNumElements() <= NumElts) {
assert(Inst.getType()->getScalarType() == V1->getType()->getScalarType() &&
"Shuffle should not change scalar type");
// reorder is not possible. A 1-to-1 mapping is not required. Example:
// ShMask = <1,1,2,2> and C = <5,5,6,6> --> NewC = <undef,5,6,undef>
bool ConstOp1 = isa<Constant>(RHS);
- SmallVector<int, 16> ShMask;
- ShuffleVectorInst::getShuffleMask(Mask, ShMask);
+ ArrayRef<int> ShMask = Mask;
unsigned SrcVecNumElts = V1->getType()->getVectorNumElements();
UndefValue *UndefScalar = UndefValue::get(C->getType()->getScalarType());
SmallVector<Constant *, 16> NewVecC(SrcVecNumElts, UndefScalar);
std::swap(LHS, RHS);
Value *X;
- Constant *MaskC;
- const APInt *SplatIndex;
+ ArrayRef<int> MaskC;
+ int SplatIndex;
BinaryOperator *BO;
if (!match(LHS, m_OneUse(m_ShuffleVector(m_Value(X), m_Undef(),
- m_Constant(MaskC)))) ||
- !match(MaskC, m_APIntAllowUndef(SplatIndex)) ||
+ m_Mask(MaskC)))) ||
+ !match(MaskC, m_SplatOrUndefMask(SplatIndex)) ||
X->getType() != Inst.getType() || !match(RHS, m_OneUse(m_BinOp(BO))) ||
BO->getOpcode() != Opcode)
return nullptr;
// moving 'Y' before the splat shuffle, we are implicitly assuming
// that it is not undef/poison at the splat index.
Value *Y, *OtherOp;
- if (isSplatValue(BO->getOperand(0), SplatIndex->getZExtValue())) {
+ if (isSplatValue(BO->getOperand(0), SplatIndex)) {
Y = BO->getOperand(0);
OtherOp = BO->getOperand(1);
- } else if (isSplatValue(BO->getOperand(1), SplatIndex->getZExtValue())) {
+ } else if (isSplatValue(BO->getOperand(1), SplatIndex)) {
Y = BO->getOperand(1);
OtherOp = BO->getOperand(0);
} else {
// bo (splat X), (bo Y, OtherOp) --> bo (splat (bo X, Y)), OtherOp
Value *NewBO = Builder.CreateBinOp(Opcode, X, Y);
UndefValue *Undef = UndefValue::get(Inst.getType());
- Constant *NewMask = ConstantInt::get(MaskC->getType(), *SplatIndex);
+ SmallVector<int, 8> NewMask(MaskC.size(), SplatIndex);
Value *NewSplat = Builder.CreateShuffleVector(NewBO, Undef, NewMask);
Instruction *R = BinaryOperator::Create(Opcode, NewSplat, OtherOp);
}
void visitShuffleVectorInst(ShuffleVectorInst &I) {
- insertShadowCheck(I.getOperand(2), &I);
IRBuilder<> IRB(&I);
setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1),
- I.getOperand(2), "_msprop"));
+ I.getShuffleMask(), "_msprop"));
setOriginForNaryOp(I);
}
e.commutative = true;
}
- if (CmpInst *C = dyn_cast<CmpInst>(I)) {
+ if (auto *C = dyn_cast<CmpInst>(I)) {
// Sort the operand value numbers so x<y and y>x get the same value number.
CmpInst::Predicate Predicate = C->getPredicate();
if (e.varargs[0] > e.varargs[1]) {
}
e.opcode = (C->getOpcode() << 8) | Predicate;
e.commutative = true;
- } else if (InsertValueInst *E = dyn_cast<InsertValueInst>(I)) {
- for (InsertValueInst::idx_iterator II = E->idx_begin(), IE = E->idx_end();
- II != IE; ++II)
- e.varargs.push_back(*II);
+ } else if (auto *E = dyn_cast<InsertValueInst>(I)) {
+ e.varargs.append(E->idx_begin(), E->idx_end());
+ } else if (auto *SVI = dyn_cast<ShuffleVectorInst>(I)) {
+ ArrayRef<int> ShuffleMask = SVI->getShuffleMask();
+ e.varargs.append(ShuffleMask.begin(), ShuffleMask.end());
}
return e;
// instead of value numbers. Those index numbers should not be
// translated.
if ((i > 1 && Exp.opcode == Instruction::InsertValue) ||
- (i > 0 && Exp.opcode == Instruction::ExtractValue))
+ (i > 0 && Exp.opcode == Instruction::ExtractValue) ||
+ (i > 1 && Exp.opcode == Instruction::ShuffleVector))
continue;
Exp.varargs[i] = phiTranslate(Pred, PhiBlock, Exp.varargs[i], Gvn);
}
class InstructionUseExpr : public GVNExpression::BasicExpression {
unsigned MemoryUseOrder = -1;
bool Volatile = false;
+ std::vector<int> ShuffleMask;
public:
InstructionUseExpr(Instruction *I, ArrayRecycler<Value *> &R,
void setMemoryUseOrder(unsigned MUO) { MemoryUseOrder = MUO; }
void setVolatile(bool V) { Volatile = V; }
+ void setShuffleMask(ArrayRef<int> Mask) {
+ ShuffleMask.assign(Mask.begin(), Mask.end());
+ }
hash_code getHashValue() const override {
return hash_combine(GVNExpression::BasicExpression::getHashValue(),
- MemoryUseOrder, Volatile);
+ MemoryUseOrder, Volatile, ArrayRef<int>(ShuffleMask));
}
template <typename Function> hash_code getHashValue(Function MapFn) {
- hash_code H =
- hash_combine(getOpcode(), getType(), MemoryUseOrder, Volatile);
+ hash_code H = hash_combine(getOpcode(), getType(), MemoryUseOrder, Volatile,
+ ArrayRef<int>(ShuffleMask));
for (auto *V : operands())
H = hash_combine(H, MapFn(V));
return H;
CmpInst::Predicate Predicate = C->getPredicate();
E->setOpcode((C->getOpcode() << 8) | Predicate);
}
+ if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I))
+ E->setShuffleMask(SVI->getShuffleMask());
return E;
}
case Instruction::Select:
case Instruction::ExtractElement:
case Instruction::InsertElement:
- case Instruction::ShuffleVector:
case Instruction::GetElementPtr:
E = createExpression(I);
break;
+ case Instruction::ShuffleVector:
+ // FIXME: Add support for shufflevector to createExpression.
+ return nullptr;
default:
return nullptr;
}
auto *SV = cast<ShuffleVectorInst>(I);
UndefValue *VecUndef = UndefValue::get(SV->getOperand(0)->getType());
std::string Name = suffixed_name_or(I, ".base", "base_sv");
- return new ShuffleVectorInst(VecUndef, VecUndef, SV->getOperand(2),
+ return new ShuffleVectorInst(VecUndef, VecUndef, SV->getShuffleMask(),
Name, SV);
}
};
auto *O1 = B.CreateZExtOrTrunc(
SI->getOperand(1), VectorType::get(ScalarTruncatedTy, Elements1));
- NewI = B.CreateShuffleVector(O0, O1, SI->getMask());
+ NewI = B.CreateShuffleVector(O0, O1, SI->getShuffleMask());
} else if (isa<LoadInst>(I) || isa<PHINode>(I)) {
// Don't do anything with the operands, just extend the result.
continue;
EXPECT_TRUE(match(EX3, m_ExtractElement(m_Constant(), m_ConstantInt())));
// Test matching shufflevector
- EXPECT_TRUE(match(SI1, m_ShuffleVector(m_Value(), m_Undef(), m_Zero())));
- EXPECT_TRUE(match(SI2, m_ShuffleVector(m_Value(A), m_Value(B), m_Value(C))));
+ ArrayRef<int> Mask;
+ EXPECT_TRUE(match(SI1, m_ShuffleVector(m_Value(), m_Undef(), m_ZeroMask())));
+ EXPECT_TRUE(
+ match(SI2, m_ShuffleVector(m_Value(A), m_Value(B), m_Mask(Mask))));
EXPECT_TRUE(A == VI3);
EXPECT_TRUE(B == VI4);
- EXPECT_TRUE(C == IdxVec);
A = B = C = nullptr; // reset
// Test matching the vector splat pattern
EXPECT_TRUE(match(
SI1,
m_ShuffleVector(m_InsertElement(m_Undef(), m_SpecificInt(1), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_FALSE(match(
SI3, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_FALSE(match(
SI4, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_TRUE(match(
SP1,
m_ShuffleVector(m_InsertElement(m_Undef(), m_SpecificInt(2), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_TRUE(match(
SP2, m_ShuffleVector(m_InsertElement(m_Undef(), m_Value(A), m_Zero()),
- m_Undef(), m_Zero())));
+ m_Undef(), m_ZeroMask())));
EXPECT_TRUE(A == Val);
}
projects / platform / upstream / llvm.git / commitdiff