return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, &MaskVec[0]);
}
+static SDValue peekThroughBitcasts(SDValue V) {
+ while (V.getNode() && V.getOpcode() == ISD::BITCAST)
+ V = V.getOperand(0);
+ return V;
+}
+
static bool getTargetShuffleMaskIndices(SDValue MaskNode,
unsigned MaskEltSizeInBits,
SmallVectorImpl<uint64_t> &RawMask) {
- while (MaskNode.getOpcode() == ISD::BITCAST)
- MaskNode = MaskNode.getOperand(0);
+ MaskNode = peekThroughBitcasts(MaskNode);
MVT VT = MaskNode.getSimpleValueType();
assert(VT.isVector() && "Can't produce a non-vector with a build_vector!");
}
static const Constant *getTargetShuffleMaskConstant(SDValue MaskNode) {
- while (MaskNode.getOpcode() == ISD::BITCAST)
- MaskNode = MaskNode.getOperand(0);
+ MaskNode = peekThroughBitcasts(MaskNode);
auto *MaskLoad = dyn_cast<LoadSDNode>(MaskNode);
if (!MaskLoad)
SDValue V1 = Ops[0];
SDValue V2 = IsUnary ? V1 : Ops[1];
- while (V1.getOpcode() == ISD::BITCAST)
- V1 = V1->getOperand(0);
- while (V2.getOpcode() == ISD::BITCAST)
- V2 = V2->getOperand(0);
+ V1 = peekThroughBitcasts(V1);
+ V2 = peekThroughBitcasts(V2);
for (int i = 0, Size = Mask.size(); i < Size; ++i) {
int M = Mask[i];
SmallBitVector ZeroMask(NumElems, false);
SmallBitVector UndefMask(NumElems, false);
- auto PeekThroughBitcast = [](SDValue V) {
- while (V.getNode() && V.getOpcode() == ISD::BITCAST)
- V = V.getOperand(0);
- return V;
- };
-
// For each element in the initializer, see if we've found a load, zero or an
// undef.
for (unsigned i = 0; i < NumElems; ++i) {
- SDValue Elt = PeekThroughBitcast(Elts[i]);
+ SDValue Elt = peekThroughBitcasts(Elts[i]);
if (!Elt.getNode())
return SDValue();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
int FirstLoadedElt = LoadMask.find_first();
- SDValue EltBase = PeekThroughBitcast(Elts[FirstLoadedElt]);
+ SDValue EltBase = peekThroughBitcasts(Elts[FirstLoadedElt]);
LoadSDNode *LDBase = cast<LoadSDNode>(EltBase);
EVT LDBaseVT = EltBase.getValueType();
bool IsConsecutiveLoadWithZeros = true;
for (int i = FirstLoadedElt + 1; i <= LastLoadedElt; ++i) {
if (LoadMask[i]) {
- SDValue Elt = PeekThroughBitcast(Elts[i]);
+ SDValue Elt = peekThroughBitcasts(Elts[i]);
LoadSDNode *LD = cast<LoadSDNode>(Elt);
if (!DAG.isConsecutiveLoad(LD, LDBase,
Elt.getValueType().getStoreSizeInBits() / 8,
static SmallBitVector computeZeroableShuffleElements(ArrayRef<int> Mask,
SDValue V1, SDValue V2) {
SmallBitVector Zeroable(Mask.size(), false);
-
- while (V1.getOpcode() == ISD::BITCAST)
- V1 = V1->getOperand(0);
- while (V2.getOpcode() == ISD::BITCAST)
- V2 = V2->getOperand(0);
+ V1 = peekThroughBitcasts(V1);
+ V2 = peekThroughBitcasts(V2);
bool V1IsZero = ISD::isBuildVectorAllZeros(V1.getNode());
bool V2IsZero = ISD::isBuildVectorAllZeros(V2.getNode());
SelectionDAG &DAG) {
MVT VT = V.getSimpleValueType();
MVT EltVT = VT.getVectorElementType();
- while (V.getOpcode() == ISD::BITCAST)
- V = V.getOperand(0);
+ V = peekThroughBitcasts(V);
+
// If the bitcasts shift the element size, we can't extract an equivalent
// element from it.
MVT NewVT = V.getSimpleValueType();
/// This is particularly important because the set of instructions varies
/// significantly based on whether the operand is a load or not.
static bool isShuffleFoldableLoad(SDValue V) {
- while (V.getOpcode() == ISD::BITCAST)
- V = V.getOperand(0);
-
+ V = peekThroughBitcasts(V);
return ISD::isNON_EXTLoad(V.getNode());
}
MVT BroadcastVT = VT;
// Peek through any bitcast (only useful for loads).
- SDValue BC = V;
- while (BC.getOpcode() == ISD::BITCAST)
- BC = BC.getOperand(0);
+ SDValue BC = peekThroughBitcasts(V);
// Also check the simpler case, where we can directly reuse the scalar.
if (V.getOpcode() == ISD::BUILD_VECTOR ||
// particularly important as it lets us merge shuffles that this routine itself
// creates.
auto GetPackNode = [](SDValue V) {
- while (V.getOpcode() == ISD::BITCAST)
- V = V.getOperand(0);
-
+ V = peekThroughBitcasts(V);
return V.getOpcode() == X86ISD::PACKUS ? V : SDValue();
};
if (SDValue V1Pack = GetPackNode(V1))
// Rather than splitting build-vectors, just build two narrower build
// vectors. This helps shuffling with splats and zeros.
auto SplitVector = [&](SDValue V) {
- while (V.getOpcode() == ISD::BITCAST)
- V = V->getOperand(0);
+ V = peekThroughBitcasts(V);
MVT OrigVT = V.getSimpleValueType();
int OrigNumElements = OrigVT.getVectorNumElements();
OpVT.is256BitVector() && SubVecVT.is128BitVector()) {
auto *Idx2 = dyn_cast<ConstantSDNode>(Vec.getOperand(2));
if (Idx2 && Idx2->getZExtValue() == 0) {
- SDValue SubVec2 = Vec.getOperand(1);
- // If needed, look through a bitcast to get to the load.
- if (SubVec2.getNode() && SubVec2.getOpcode() == ISD::BITCAST)
- SubVec2 = SubVec2.getOperand(0);
-
+ // If needed, look through bitcasts to get to the load.
+ SDValue SubVec2 = peekThroughBitcasts(Vec.getOperand(1));
if (auto *FirstLd = dyn_cast<LoadSDNode>(SubVec2)) {
bool Fast;
unsigned Alignment = FirstLd->getAlignment();
// Find the operand that enters the chain. Note that multiple uses are OK
// here, we're not going to remove the operand we find.
- while (Input.getOpcode() == ISD::BITCAST)
- Input = Input.getOperand(0);
+ Input = peekThroughBitcasts(Input);
MVT VT = Input.getSimpleValueType();
MVT RootVT = Root.getSimpleValueType();
SDValue N00 = N0->getOperand(0);
SDValue N01 = N0->getOperand(1);
- // Look through a bitcast.
- if (N01->getOpcode() == ISD::BITCAST)
- N01 = N01->getOperand(0);
+ N01 = peekThroughBitcasts(N01);
// Either match a direct AllOnes for 128 and 256-bit vectors, or an
// insert_subvector building a 256-bit AllOnes vector.
// The other side of the AND should be a splat of 2^C, where C
// is the number of bits in the source type.
- if (N1.getOpcode() == ISD::BITCAST)
- N1 = N1.getOperand(0);
+ N1 = peekThroughBitcasts(N1);
if (N1.getOpcode() != ISD::BUILD_VECTOR)
return SDValue();
BuildVectorSDNode *Vector = cast<BuildVectorSDNode>(N1);
return SDValue();
// Validate that X, Y, and Mask are bitcasts, and see through them.
- if (Mask.getOpcode() == ISD::BITCAST)
- Mask = Mask.getOperand(0);
- if (X.getOpcode() == ISD::BITCAST)
- X = X.getOperand(0);
- if (Y.getOpcode() == ISD::BITCAST)
- Y = Y.getOperand(0);
+ Mask = peekThroughBitcasts(Mask);
+ X = peekThroughBitcasts(X);
+ Y = peekThroughBitcasts(Y);
EVT MaskVT = Mask.getValueType();
}
static SDValue combineVZextMovl(SDNode *N, SelectionDAG &DAG) {
- SDValue Op = N->getOperand(0);
- if (Op.getOpcode() == ISD::BITCAST)
- Op = Op.getOperand(0);
+ SDValue Op = peekThroughBitcasts(N->getOperand(0));
EVT VT = N->getValueType(0), OpVT = Op.getValueType();
if (Op.getOpcode() == X86ISD::VZEXT_LOAD &&
VT.getVectorElementType().getSizeInBits() ==
unsigned InputBits = OpEltVT.getSizeInBits() * VT.getVectorNumElements();
// (vzext (bitcast (vzext (x)) -> (vzext x)
- SDValue V = Op;
- while (V.getOpcode() == ISD::BITCAST)
- V = V.getOperand(0);
-
+ SDValue V = peekThroughBitcasts(Op);
if (V != Op && V.getOpcode() == X86ISD::VZEXT) {
MVT InnerVT = V.getSimpleValueType();
MVT InnerEltVT = InnerVT.getVectorElementType();