From 4041f2217b14d395e22d7e646d6d80af1d22ec2f Mon Sep 17 00:00:00 2001 From: Chandler Carruth Date: Fri, 20 Feb 2015 04:25:04 +0000 Subject: [PATCH] [x86] Remove the old vector shuffle lowering code and its flag. The new shuffle lowering has been the default for some time. I've enabled the new legality testing by default with no really blocking regressions. I've fuzz tested this very heavily (many millions of fuzz test cases have passed at this point). And this cleans up a ton of code. =] Thanks again to the many folks that helped with this transition. There was a lot of work by others that went into the new shuffle lowering to make it really excellent. In case you aren't using a diff algorithm that can handle this: X86ISelLowering.cpp: 22 insertions(+), 2940 deletions(-) llvm-svn: 229964 --- llvm/lib/Target/X86/X86ISelLowering.cpp | 13396 ++++++++----------- .../test/CodeGen/X86/avx2-pmovx-256-old-shuffle.ll | 27 - 2 files changed, 5239 insertions(+), 8184 deletions(-) delete mode 100644 llvm/test/CodeGen/X86/avx2-pmovx-256-old-shuffle.ll diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 30cd7ae..90718e9 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -67,11 +67,6 @@ static cl::opt ExperimentalVectorWideningLegalization( "rather than promotion."), cl::Hidden); -static cl::opt ExperimentalVectorShuffleLowering( - "x86-experimental-vector-shuffle-lowering", cl::init(true), - cl::desc("Enable an experimental vector shuffle lowering code path."), - cl::Hidden); - static cl::opt ReciprocalEstimateRefinementSteps( "x86-recip-refinement-steps", cl::init(1), cl::desc("Specify the number of Newton-Raphson iterations applied to the " @@ -3613,17 +3608,6 @@ static bool isTargetShuffle(unsigned Opcode) { } static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT, - SDValue V1, SelectionDAG &DAG) { - switch(Opc) { - default: llvm_unreachable("Unknown x86 shuffle node"); - case X86ISD::MOVSHDUP: - case X86ISD::MOVSLDUP: - case X86ISD::MOVDDUP: - return DAG.getNode(Opc, dl, VT, V1); - } -} - -static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT, SDValue V1, unsigned TargetMask, SelectionDAG &DAG) { switch(Opc) { @@ -3638,20 +3622,6 @@ static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT, } static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT, - SDValue V1, SDValue V2, unsigned TargetMask, - SelectionDAG &DAG) { - switch(Opc) { - default: llvm_unreachable("Unknown x86 shuffle node"); - case X86ISD::PALIGNR: - case X86ISD::VALIGN: - case X86ISD::SHUFP: - case X86ISD::VPERM2X128: - return DAG.getNode(Opc, dl, VT, V1, V2, - DAG.getConstant(TargetMask, MVT::i8)); - } -} - -static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG) { switch(Opc) { default: llvm_unreachable("Unknown x86 shuffle node"); @@ -3937,176 +3907,6 @@ static bool isSequentialOrUndefInRange(ArrayRef Mask, return true; } -/// isPSHUFDMask - Return true if the node specifies a shuffle of elements that -/// is suitable for input to PSHUFD. That is, it doesn't reference the other -/// operand - by default will match for first operand. -static bool isPSHUFDMask(ArrayRef Mask, MVT VT, - bool TestSecondOperand = false) { - if (VT != MVT::v4f32 && VT != MVT::v4i32 && - VT != MVT::v2f64 && VT != MVT::v2i64) - return false; - - unsigned NumElems = VT.getVectorNumElements(); - unsigned Lo = TestSecondOperand ? NumElems : 0; - unsigned Hi = Lo + NumElems; - - for (unsigned i = 0; i < NumElems; ++i) - if (!isUndefOrInRange(Mask[i], (int)Lo, (int)Hi)) - return false; - - return true; -} - -/// isPSHUFHWMask - Return true if the node specifies a shuffle of elements that -/// is suitable for input to PSHUFHW. -static bool isPSHUFHWMask(ArrayRef Mask, MVT VT, bool HasInt256) { - if (VT != MVT::v8i16 && (!HasInt256 || VT != MVT::v16i16)) - return false; - - // Lower quadword copied in order or undef. - if (!isSequentialOrUndefInRange(Mask, 0, 4, 0)) - return false; - - // Upper quadword shuffled. - for (unsigned i = 4; i != 8; ++i) - if (!isUndefOrInRange(Mask[i], 4, 8)) - return false; - - if (VT == MVT::v16i16) { - // Lower quadword copied in order or undef. - if (!isSequentialOrUndefInRange(Mask, 8, 4, 8)) - return false; - - // Upper quadword shuffled. - for (unsigned i = 12; i != 16; ++i) - if (!isUndefOrInRange(Mask[i], 12, 16)) - return false; - } - - return true; -} - -/// isPSHUFLWMask - Return true if the node specifies a shuffle of elements that -/// is suitable for input to PSHUFLW. -static bool isPSHUFLWMask(ArrayRef Mask, MVT VT, bool HasInt256) { - if (VT != MVT::v8i16 && (!HasInt256 || VT != MVT::v16i16)) - return false; - - // Upper quadword copied in order. - if (!isSequentialOrUndefInRange(Mask, 4, 4, 4)) - return false; - - // Lower quadword shuffled. - for (unsigned i = 0; i != 4; ++i) - if (!isUndefOrInRange(Mask[i], 0, 4)) - return false; - - if (VT == MVT::v16i16) { - // Upper quadword copied in order. - if (!isSequentialOrUndefInRange(Mask, 12, 4, 12)) - return false; - - // Lower quadword shuffled. - for (unsigned i = 8; i != 12; ++i) - if (!isUndefOrInRange(Mask[i], 8, 12)) - return false; - } - - return true; -} - -/// \brief Return true if the mask specifies a shuffle of elements that is -/// suitable for input to intralane (palignr) or interlane (valign) vector -/// right-shift. -static bool isAlignrMask(ArrayRef Mask, MVT VT, bool InterLane) { - unsigned NumElts = VT.getVectorNumElements(); - unsigned NumLanes = InterLane ? 1: VT.getSizeInBits()/128; - unsigned NumLaneElts = NumElts/NumLanes; - - // Do not handle 64-bit element shuffles with palignr. - if (NumLaneElts == 2) - return false; - - for (unsigned l = 0; l != NumElts; l+=NumLaneElts) { - unsigned i; - for (i = 0; i != NumLaneElts; ++i) { - if (Mask[i+l] >= 0) - break; - } - - // Lane is all undef, go to next lane - if (i == NumLaneElts) - continue; - - int Start = Mask[i+l]; - - // Make sure its in this lane in one of the sources - if (!isUndefOrInRange(Start, l, l+NumLaneElts) && - !isUndefOrInRange(Start, l+NumElts, l+NumElts+NumLaneElts)) - return false; - - // If not lane 0, then we must match lane 0 - if (l != 0 && Mask[i] >= 0 && !isUndefOrEqual(Start, Mask[i]+l)) - return false; - - // Correct second source to be contiguous with first source - if (Start >= (int)NumElts) - Start -= NumElts - NumLaneElts; - - // Make sure we're shifting in the right direction. - if (Start <= (int)(i+l)) - return false; - - Start -= i; - - // Check the rest of the elements to see if they are consecutive. - for (++i; i != NumLaneElts; ++i) { - int Idx = Mask[i+l]; - - // Make sure its in this lane - if (!isUndefOrInRange(Idx, l, l+NumLaneElts) && - !isUndefOrInRange(Idx, l+NumElts, l+NumElts+NumLaneElts)) - return false; - - // If not lane 0, then we must match lane 0 - if (l != 0 && Mask[i] >= 0 && !isUndefOrEqual(Idx, Mask[i]+l)) - return false; - - if (Idx >= (int)NumElts) - Idx -= NumElts - NumLaneElts; - - if (!isUndefOrEqual(Idx, Start+i)) - return false; - - } - } - - return true; -} - -/// \brief Return true if the node specifies a shuffle of elements that is -/// suitable for input to PALIGNR. -static bool isPALIGNRMask(ArrayRef Mask, MVT VT, - const X86Subtarget *Subtarget) { - if ((VT.is128BitVector() && !Subtarget->hasSSSE3()) || - (VT.is256BitVector() && !Subtarget->hasInt256()) || - VT.is512BitVector()) - // FIXME: Add AVX512BW. - return false; - - return isAlignrMask(Mask, VT, false); -} - -/// \brief Return true if the node specifies a shuffle of elements that is -/// suitable for input to VALIGN. -static bool isVALIGNMask(ArrayRef Mask, MVT VT, - const X86Subtarget *Subtarget) { - // FIXME: Add AVX512VL. - if (!VT.is512BitVector() || !Subtarget->hasAVX512()) - return false; - return isAlignrMask(Mask, VT, true); -} - /// CommuteVectorShuffleMask - Change values in a shuffle permute mask assuming /// the two vector operands have swapped position. static void CommuteVectorShuffleMask(SmallVectorImpl &Mask, @@ -4122,8540 +3922,6016 @@ static void CommuteVectorShuffleMask(SmallVectorImpl &Mask, } } -/// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand -/// specifies a shuffle of elements that is suitable for input to 128/256-bit -/// SHUFPS and SHUFPD. If Commuted is true, then it checks for sources to be -/// reverse of what x86 shuffles want. -static bool isSHUFPMask(ArrayRef Mask, MVT VT, bool Commuted = false) { - - unsigned NumElems = VT.getVectorNumElements(); - unsigned NumLanes = VT.getSizeInBits()/128; - unsigned NumLaneElems = NumElems/NumLanes; - - if (NumLaneElems != 2 && NumLaneElems != 4) +/// isVEXTRACTIndex - Return true if the specified +/// EXTRACT_SUBVECTOR operand specifies a vector extract that is +/// suitable for instruction that extract 128 or 256 bit vectors +static bool isVEXTRACTIndex(SDNode *N, unsigned vecWidth) { + assert((vecWidth == 128 || vecWidth == 256) && "Unexpected vector width"); + if (!isa(N->getOperand(1).getNode())) return false; - unsigned EltSize = VT.getVectorElementType().getSizeInBits(); - bool symmetricMaskRequired = - (VT.getSizeInBits() >= 256) && (EltSize == 32); + // The index should be aligned on a vecWidth-bit boundary. + uint64_t Index = + cast(N->getOperand(1).getNode())->getZExtValue(); - // VSHUFPSY divides the resulting vector into 4 chunks. - // The sources are also splitted into 4 chunks, and each destination - // chunk must come from a different source chunk. - // - // SRC1 => X7 X6 X5 X4 X3 X2 X1 X0 - // SRC2 => Y7 Y6 Y5 Y4 Y3 Y2 Y1 Y9 - // - // DST => Y7..Y4, Y7..Y4, X7..X4, X7..X4, - // Y3..Y0, Y3..Y0, X3..X0, X3..X0 - // - // VSHUFPDY divides the resulting vector into 4 chunks. - // The sources are also splitted into 4 chunks, and each destination - // chunk must come from a different source chunk. - // - // SRC1 => X3 X2 X1 X0 - // SRC2 => Y3 Y2 Y1 Y0 - // - // DST => Y3..Y2, X3..X2, Y1..Y0, X1..X0 - // - SmallVector MaskVal(NumLaneElems, -1); - unsigned HalfLaneElems = NumLaneElems/2; - for (unsigned l = 0; l != NumElems; l += NumLaneElems) { - for (unsigned i = 0; i != NumLaneElems; ++i) { - int Idx = Mask[i+l]; - unsigned RngStart = l + ((Commuted == (igetSimpleValueType(0); + unsigned ElSize = VT.getVectorElementType().getSizeInBits(); + bool Result = (Index * ElSize) % vecWidth == 0; - return true; + return Result; } -/// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand -/// specifies a shuffle of elements that is suitable for input to MOVHLPS. -static bool isMOVHLPSMask(ArrayRef Mask, MVT VT) { - if (!VT.is128BitVector()) +/// isVINSERTIndex - Return true if the specified INSERT_SUBVECTOR +/// operand specifies a subvector insert that is suitable for input to +/// insertion of 128 or 256-bit subvectors +static bool isVINSERTIndex(SDNode *N, unsigned vecWidth) { + assert((vecWidth == 128 || vecWidth == 256) && "Unexpected vector width"); + if (!isa(N->getOperand(2).getNode())) return false; + // The index should be aligned on a vecWidth-bit boundary. + uint64_t Index = + cast(N->getOperand(2).getNode())->getZExtValue(); - unsigned NumElems = VT.getVectorNumElements(); - - if (NumElems != 4) - return false; + MVT VT = N->getSimpleValueType(0); + unsigned ElSize = VT.getVectorElementType().getSizeInBits(); + bool Result = (Index * ElSize) % vecWidth == 0; - // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3 - return isUndefOrEqual(Mask[0], 6) && - isUndefOrEqual(Mask[1], 7) && - isUndefOrEqual(Mask[2], 2) && - isUndefOrEqual(Mask[3], 3); + return Result; } -/// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form -/// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef, -/// <2, 3, 2, 3> -static bool isMOVHLPS_v_undef_Mask(ArrayRef Mask, MVT VT) { - if (!VT.is128BitVector()) - return false; - - unsigned NumElems = VT.getVectorNumElements(); - - if (NumElems != 4) - return false; +bool X86::isVINSERT128Index(SDNode *N) { + return isVINSERTIndex(N, 128); +} - return isUndefOrEqual(Mask[0], 2) && - isUndefOrEqual(Mask[1], 3) && - isUndefOrEqual(Mask[2], 2) && - isUndefOrEqual(Mask[3], 3); +bool X86::isVINSERT256Index(SDNode *N) { + return isVINSERTIndex(N, 256); } -/// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand -/// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}. -static bool isMOVLPMask(ArrayRef Mask, MVT VT) { - if (!VT.is128BitVector()) - return false; +bool X86::isVEXTRACT128Index(SDNode *N) { + return isVEXTRACTIndex(N, 128); +} - unsigned NumElems = VT.getVectorNumElements(); +bool X86::isVEXTRACT256Index(SDNode *N) { + return isVEXTRACTIndex(N, 256); +} - if (NumElems != 2 && NumElems != 4) - return false; +static unsigned getExtractVEXTRACTImmediate(SDNode *N, unsigned vecWidth) { + assert((vecWidth == 128 || vecWidth == 256) && "Unsupported vector width"); + if (!isa(N->getOperand(1).getNode())) + llvm_unreachable("Illegal extract subvector for VEXTRACT"); - for (unsigned i = 0, e = NumElems/2; i != e; ++i) - if (!isUndefOrEqual(Mask[i], i + NumElems)) - return false; + uint64_t Index = + cast(N->getOperand(1).getNode())->getZExtValue(); - for (unsigned i = NumElems/2, e = NumElems; i != e; ++i) - if (!isUndefOrEqual(Mask[i], i)) - return false; + MVT VecVT = N->getOperand(0).getSimpleValueType(); + MVT ElVT = VecVT.getVectorElementType(); - return true; + unsigned NumElemsPerChunk = vecWidth / ElVT.getSizeInBits(); + return Index / NumElemsPerChunk; } -/// isMOVLHPSMask - Return true if the specified VECTOR_SHUFFLE operand -/// specifies a shuffle of elements that is suitable for input to MOVLHPS. -static bool isMOVLHPSMask(ArrayRef Mask, MVT VT) { - if (!VT.is128BitVector()) - return false; +static unsigned getInsertVINSERTImmediate(SDNode *N, unsigned vecWidth) { + assert((vecWidth == 128 || vecWidth == 256) && "Unsupported vector width"); + if (!isa(N->getOperand(2).getNode())) + llvm_unreachable("Illegal insert subvector for VINSERT"); - unsigned NumElems = VT.getVectorNumElements(); + uint64_t Index = + cast(N->getOperand(2).getNode())->getZExtValue(); - if (NumElems != 2 && NumElems != 4) - return false; + MVT VecVT = N->getSimpleValueType(0); + MVT ElVT = VecVT.getVectorElementType(); - for (unsigned i = 0, e = NumElems/2; i != e; ++i) - if (!isUndefOrEqual(Mask[i], i)) - return false; + unsigned NumElemsPerChunk = vecWidth / ElVT.getSizeInBits(); + return Index / NumElemsPerChunk; +} - for (unsigned i = 0, e = NumElems/2; i != e; ++i) - if (!isUndefOrEqual(Mask[i + e], i + NumElems)) - return false; +/// getExtractVEXTRACT128Immediate - Return the appropriate immediate +/// to extract the specified EXTRACT_SUBVECTOR index with VEXTRACTF128 +/// and VINSERTI128 instructions. +unsigned X86::getExtractVEXTRACT128Immediate(SDNode *N) { + return getExtractVEXTRACTImmediate(N, 128); +} - return true; +/// getExtractVEXTRACT256Immediate - Return the appropriate immediate +/// to extract the specified EXTRACT_SUBVECTOR index with VEXTRACTF64x4 +/// and VINSERTI64x4 instructions. +unsigned X86::getExtractVEXTRACT256Immediate(SDNode *N) { + return getExtractVEXTRACTImmediate(N, 256); } -/// isINSERTPSMask - Return true if the specified VECTOR_SHUFFLE operand -/// specifies a shuffle of elements that is suitable for input to INSERTPS. -/// i. e: If all but one element come from the same vector. -static bool isINSERTPSMask(ArrayRef Mask, MVT VT) { - // TODO: Deal with AVX's VINSERTPS - if (!VT.is128BitVector() || (VT != MVT::v4f32 && VT != MVT::v4i32)) - return false; +/// getInsertVINSERT128Immediate - Return the appropriate immediate +/// to insert at the specified INSERT_SUBVECTOR index with VINSERTF128 +/// and VINSERTI128 instructions. +unsigned X86::getInsertVINSERT128Immediate(SDNode *N) { + return getInsertVINSERTImmediate(N, 128); +} - unsigned CorrectPosV1 = 0; - unsigned CorrectPosV2 = 0; - for (int i = 0, e = (int)VT.getVectorNumElements(); i != e; ++i) { - if (Mask[i] == -1) { - ++CorrectPosV1; - ++CorrectPosV2; - continue; - } +/// getInsertVINSERT256Immediate - Return the appropriate immediate +/// to insert at the specified INSERT_SUBVECTOR index with VINSERTF46x4 +/// and VINSERTI64x4 instructions. +unsigned X86::getInsertVINSERT256Immediate(SDNode *N) { + return getInsertVINSERTImmediate(N, 256); +} - if (Mask[i] == i) - ++CorrectPosV1; - else if (Mask[i] == i + 4) - ++CorrectPosV2; - } +/// isZero - Returns true if Elt is a constant integer zero +static bool isZero(SDValue V) { + ConstantSDNode *C = dyn_cast(V); + return C && C->isNullValue(); +} - if (CorrectPosV1 == 3 || CorrectPosV2 == 3) - // We have 3 elements (undefs count as elements from any vector) from one - // vector, and one from another. +/// isZeroNode - Returns true if Elt is a constant zero or a floating point +/// constant +0.0. +bool X86::isZeroNode(SDValue Elt) { + if (isZero(Elt)) return true; - + if (ConstantFPSDNode *CFP = dyn_cast(Elt)) + return CFP->getValueAPF().isPosZero(); return false; } -// -// Some special combinations that can be optimized. -// -static -SDValue Compact8x32ShuffleNode(ShuffleVectorSDNode *SVOp, - SelectionDAG &DAG) { - MVT VT = SVOp->getSimpleValueType(0); - SDLoc dl(SVOp); - - if (VT != MVT::v8i32 && VT != MVT::v8f32) - return SDValue(); - - ArrayRef Mask = SVOp->getMask(); - - // These are the special masks that may be optimized. - static const int MaskToOptimizeEven[] = {0, 8, 2, 10, 4, 12, 6, 14}; - static const int MaskToOptimizeOdd[] = {1, 9, 3, 11, 5, 13, 7, 15}; - bool MatchEvenMask = true; - bool MatchOddMask = true; - for (int i=0; i<8; ++i) { - if (!isUndefOrEqual(Mask[i], MaskToOptimizeEven[i])) - MatchEvenMask = false; - if (!isUndefOrEqual(Mask[i], MaskToOptimizeOdd[i])) - MatchOddMask = false; - } - - if (!MatchEvenMask && !MatchOddMask) - return SDValue(); - - SDValue UndefNode = DAG.getNode(ISD::UNDEF, dl, VT); - - SDValue Op0 = SVOp->getOperand(0); - SDValue Op1 = SVOp->getOperand(1); +/// getZeroVector - Returns a vector of specified type with all zero elements. +/// +static SDValue getZeroVector(EVT VT, const X86Subtarget *Subtarget, + SelectionDAG &DAG, SDLoc dl) { + assert(VT.isVector() && "Expected a vector type"); - if (MatchEvenMask) { - // Shift the second operand right to 32 bits. - static const int ShiftRightMask[] = {-1, 0, -1, 2, -1, 4, -1, 6 }; - Op1 = DAG.getVectorShuffle(VT, dl, Op1, UndefNode, ShiftRightMask); - } else { - // Shift the first operand left to 32 bits. - static const int ShiftLeftMask[] = {1, -1, 3, -1, 5, -1, 7, -1 }; - Op0 = DAG.getVectorShuffle(VT, dl, Op0, UndefNode, ShiftLeftMask); - } - static const int BlendMask[] = {0, 9, 2, 11, 4, 13, 6, 15}; - return DAG.getVectorShuffle(VT, dl, Op0, Op1, BlendMask); + // Always build SSE zero vectors as <4 x i32> bitcasted + // to their dest type. This ensures they get CSE'd. + SDValue Vec; + if (VT.is128BitVector()) { // SSE + if (Subtarget->hasSSE2()) { // SSE2 + SDValue Cst = DAG.getConstant(0, MVT::i32); + Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); + } else { // SSE1 + SDValue Cst = DAG.getConstantFP(+0.0, MVT::f32); + Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f32, Cst, Cst, Cst, Cst); + } + } else if (VT.is256BitVector()) { // AVX + if (Subtarget->hasInt256()) { // AVX2 + SDValue Cst = DAG.getConstant(0, MVT::i32); + SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst }; + Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops); + } else { + // 256-bit logic and arithmetic instructions in AVX are all + // floating-point, no support for integer ops. Emit fp zeroed vectors. + SDValue Cst = DAG.getConstantFP(+0.0, MVT::f32); + SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst }; + Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8f32, Ops); + } + } else if (VT.is512BitVector()) { // AVX-512 + SDValue Cst = DAG.getConstant(0, MVT::i32); + SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst, + Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst }; + Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i32, Ops); + } else if (VT.getScalarType() == MVT::i1) { + assert(VT.getVectorNumElements() <= 16 && "Unexpected vector type"); + SDValue Cst = DAG.getConstant(0, MVT::i1); + SmallVector Ops(VT.getVectorNumElements(), Cst); + return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); + } else + llvm_unreachable("Unexpected vector type"); + + return DAG.getNode(ISD::BITCAST, dl, VT, Vec); } -/// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand -/// specifies a shuffle of elements that is suitable for input to UNPCKL. -static bool isUNPCKLMask(ArrayRef Mask, MVT VT, - bool HasInt256, bool V2IsSplat = false) { +/// getOnesVector - Returns a vector of specified type with all bits set. +/// Always build ones vectors as <4 x i32> or <8 x i32>. For 256-bit types with +/// no AVX2 supprt, use two <4 x i32> inserted in a <8 x i32> appropriately. +/// Then bitcast to their original type, ensuring they get CSE'd. +static SDValue getOnesVector(MVT VT, bool HasInt256, SelectionDAG &DAG, + SDLoc dl) { + assert(VT.isVector() && "Expected a vector type"); - assert(VT.getSizeInBits() >= 128 && - "Unsupported vector type for unpckl"); + SDValue Cst = DAG.getConstant(~0U, MVT::i32); + SDValue Vec; + if (VT.is256BitVector()) { + if (HasInt256) { // AVX2 + SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst }; + Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops); + } else { // AVX + Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); + Vec = Concat128BitVectors(Vec, Vec, MVT::v8i32, 8, DAG, dl); + } + } else if (VT.is128BitVector()) { + Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); + } else + llvm_unreachable("Unexpected vector type"); - unsigned NumElts = VT.getVectorNumElements(); - if (VT.is256BitVector() && NumElts != 4 && NumElts != 8 && - (!HasInt256 || (NumElts != 16 && NumElts != 32))) - return false; + return DAG.getNode(ISD::BITCAST, dl, VT, Vec); +} - assert((!VT.is512BitVector() || VT.getScalarType().getSizeInBits() >= 32) && - "Unsupported vector type for unpckh"); +/// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd +/// operation of specified width. +static SDValue getMOVL(SelectionDAG &DAG, SDLoc dl, EVT VT, SDValue V1, + SDValue V2) { + unsigned NumElems = VT.getVectorNumElements(); + SmallVector Mask; + Mask.push_back(NumElems); + for (unsigned i = 1; i != NumElems; ++i) + Mask.push_back(i); + return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); +} - // AVX defines UNPCK* to operate independently on 128-bit lanes. - unsigned NumLanes = VT.getSizeInBits()/128; - unsigned NumLaneElts = NumElts/NumLanes; +/// getUnpackl - Returns a vector_shuffle node for an unpackl operation. +static SDValue getUnpackl(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1, + SDValue V2) { + unsigned NumElems = VT.getVectorNumElements(); + SmallVector Mask; + for (unsigned i = 0, e = NumElems/2; i != e; ++i) { + Mask.push_back(i); + Mask.push_back(i + NumElems); + } + return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); +} - for (unsigned l = 0; l != NumElts; l += NumLaneElts) { - for (unsigned i = 0, j = l; i != NumLaneElts; i += 2, ++j) { - int BitI = Mask[l+i]; - int BitI1 = Mask[l+i+1]; - if (!isUndefOrEqual(BitI, j)) - return false; - if (V2IsSplat) { - if (!isUndefOrEqual(BitI1, NumElts)) - return false; - } else { - if (!isUndefOrEqual(BitI1, j + NumElts)) - return false; - } - } +/// getUnpackh - Returns a vector_shuffle node for an unpackh operation. +static SDValue getUnpackh(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1, + SDValue V2) { + unsigned NumElems = VT.getVectorNumElements(); + SmallVector Mask; + for (unsigned i = 0, Half = NumElems/2; i != Half; ++i) { + Mask.push_back(i + Half); + Mask.push_back(i + NumElems + Half); } + return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); +} - return true; +/// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified +/// vector of zero or undef vector. This produces a shuffle where the low +/// element of V2 is swizzled into the zero/undef vector, landing at element +/// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3). +static SDValue getShuffleVectorZeroOrUndef(SDValue V2, unsigned Idx, + bool IsZero, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + MVT VT = V2.getSimpleValueType(); + SDValue V1 = IsZero + ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT); + unsigned NumElems = VT.getVectorNumElements(); + SmallVector MaskVec; + for (unsigned i = 0; i != NumElems; ++i) + // If this is the insertion idx, put the low elt of V2 here. + MaskVec.push_back(i == Idx ? NumElems : i); + return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, &MaskVec[0]); } -/// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand -/// specifies a shuffle of elements that is suitable for input to UNPCKH. -static bool isUNPCKHMask(ArrayRef Mask, MVT VT, - bool HasInt256, bool V2IsSplat = false) { - assert(VT.getSizeInBits() >= 128 && - "Unsupported vector type for unpckh"); +/// getTargetShuffleMask - Calculates the shuffle mask corresponding to the +/// target specific opcode. Returns true if the Mask could be calculated. Sets +/// IsUnary to true if only uses one source. Note that this will set IsUnary for +/// shuffles which use a single input multiple times, and in those cases it will +/// adjust the mask to only have indices within that single input. +static bool getTargetShuffleMask(SDNode *N, MVT VT, + SmallVectorImpl &Mask, bool &IsUnary) { + unsigned NumElems = VT.getVectorNumElements(); + SDValue ImmN; - unsigned NumElts = VT.getVectorNumElements(); - if (VT.is256BitVector() && NumElts != 4 && NumElts != 8 && - (!HasInt256 || (NumElts != 16 && NumElts != 32))) - return false; + IsUnary = false; + bool IsFakeUnary = false; + switch(N->getOpcode()) { + case X86ISD::BLENDI: + ImmN = N->getOperand(N->getNumOperands()-1); + DecodeBLENDMask(VT, cast(ImmN)->getZExtValue(), Mask); + break; + case X86ISD::SHUFP: + ImmN = N->getOperand(N->getNumOperands()-1); + DecodeSHUFPMask(VT, cast(ImmN)->getZExtValue(), Mask); + IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1); + break; + case X86ISD::UNPCKH: + DecodeUNPCKHMask(VT, Mask); + IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1); + break; + case X86ISD::UNPCKL: + DecodeUNPCKLMask(VT, Mask); + IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1); + break; + case X86ISD::MOVHLPS: + DecodeMOVHLPSMask(NumElems, Mask); + IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1); + break; + case X86ISD::MOVLHPS: + DecodeMOVLHPSMask(NumElems, Mask); + IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1); + break; + case X86ISD::PALIGNR: + ImmN = N->getOperand(N->getNumOperands()-1); + DecodePALIGNRMask(VT, cast(ImmN)->getZExtValue(), Mask); + break; + case X86ISD::PSHUFD: + case X86ISD::VPERMILPI: + ImmN = N->getOperand(N->getNumOperands()-1); + DecodePSHUFMask(VT, cast(ImmN)->getZExtValue(), Mask); + IsUnary = true; + break; + case X86ISD::PSHUFHW: + ImmN = N->getOperand(N->getNumOperands()-1); + DecodePSHUFHWMask(VT, cast(ImmN)->getZExtValue(), Mask); + IsUnary = true; + break; + case X86ISD::PSHUFLW: + ImmN = N->getOperand(N->getNumOperands()-1); + DecodePSHUFLWMask(VT, cast(ImmN)->getZExtValue(), Mask); + IsUnary = true; + break; + case X86ISD::PSHUFB: { + IsUnary = true; + SDValue MaskNode = N->getOperand(1); + while (MaskNode->getOpcode() == ISD::BITCAST) + MaskNode = MaskNode->getOperand(0); - assert((!VT.is512BitVector() || VT.getScalarType().getSizeInBits() >= 32) && - "Unsupported vector type for unpckh"); + if (MaskNode->getOpcode() == ISD::BUILD_VECTOR) { + // If we have a build-vector, then things are easy. + EVT VT = MaskNode.getValueType(); + assert(VT.isVector() && + "Can't produce a non-vector with a build_vector!"); + if (!VT.isInteger()) + return false; - // AVX defines UNPCK* to operate independently on 128-bit lanes. - unsigned NumLanes = VT.getSizeInBits()/128; - unsigned NumLaneElts = NumElts/NumLanes; + int NumBytesPerElement = VT.getVectorElementType().getSizeInBits() / 8; - for (unsigned l = 0; l != NumElts; l += NumLaneElts) { - for (unsigned i = 0, j = l+NumLaneElts/2; i != NumLaneElts; i += 2, ++j) { - int BitI = Mask[l+i]; - int BitI1 = Mask[l+i+1]; - if (!isUndefOrEqual(BitI, j)) - return false; - if (V2IsSplat) { - if (isUndefOrEqual(BitI1, NumElts)) - return false; - } else { - if (!isUndefOrEqual(BitI1, j+NumElts)) + SmallVector RawMask; + for (int i = 0, e = MaskNode->getNumOperands(); i < e; ++i) { + SDValue Op = MaskNode->getOperand(i); + if (Op->getOpcode() == ISD::UNDEF) { + RawMask.push_back((uint64_t)SM_SentinelUndef); + continue; + } + auto *CN = dyn_cast(Op.getNode()); + if (!CN) return false; + APInt MaskElement = CN->getAPIntValue(); + + // We now have to decode the element which could be any integer size and + // extract each byte of it. + for (int j = 0; j < NumBytesPerElement; ++j) { + // Note that this is x86 and so always little endian: the low byte is + // the first byte of the mask. + RawMask.push_back(MaskElement.getLoBits(8).getZExtValue()); + MaskElement = MaskElement.lshr(8); + } } + DecodePSHUFBMask(RawMask, Mask); + break; } - } - return true; -} - -/// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form -/// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef, -/// <0, 0, 1, 1> -static bool isUNPCKL_v_undef_Mask(ArrayRef Mask, MVT VT, bool HasInt256) { - unsigned NumElts = VT.getVectorNumElements(); - bool Is256BitVec = VT.is256BitVector(); - - if (VT.is512BitVector()) - return false; - assert((VT.is128BitVector() || VT.is256BitVector()) && - "Unsupported vector type for unpckh"); - - if (Is256BitVec && NumElts != 4 && NumElts != 8 && - (!HasInt256 || (NumElts != 16 && NumElts != 32))) - return false; - // For 256-bit i64/f64, use MOVDDUPY instead, so reject the matching pattern - // FIXME: Need a better way to get rid of this, there's no latency difference - // between UNPCKLPD and MOVDDUP, the later should always be checked first and - // the former later. We should also remove the "_undef" special mask. - if (NumElts == 4 && Is256BitVec) - return false; + auto *MaskLoad = dyn_cast(MaskNode); + if (!MaskLoad) + return false; - // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate - // independently on 128-bit lanes. - unsigned NumLanes = VT.getSizeInBits()/128; - unsigned NumLaneElts = NumElts/NumLanes; + SDValue Ptr = MaskLoad->getBasePtr(); + if (Ptr->getOpcode() == X86ISD::Wrapper) + Ptr = Ptr->getOperand(0); - for (unsigned l = 0; l != NumElts; l += NumLaneElts) { - for (unsigned i = 0, j = l; i != NumLaneElts; i += 2, ++j) { - int BitI = Mask[l+i]; - int BitI1 = Mask[l+i+1]; + auto *MaskCP = dyn_cast(Ptr); + if (!MaskCP || MaskCP->isMachineConstantPoolEntry()) + return false; - if (!isUndefOrEqual(BitI, j)) - return false; - if (!isUndefOrEqual(BitI1, j)) + if (auto *C = dyn_cast(MaskCP->getConstVal())) { + DecodePSHUFBMask(C, Mask); + if (Mask.empty()) return false; + break; } - } - - return true; -} -/// isUNPCKH_v_undef_Mask - Special case of isUNPCKHMask for canonical form -/// of vector_shuffle v, v, <2, 6, 3, 7>, i.e. vector_shuffle v, undef, -/// <2, 2, 3, 3> -static bool isUNPCKH_v_undef_Mask(ArrayRef Mask, MVT VT, bool HasInt256) { - unsigned NumElts = VT.getVectorNumElements(); - - if (VT.is512BitVector()) return false; - - assert((VT.is128BitVector() || VT.is256BitVector()) && - "Unsupported vector type for unpckh"); - - if (VT.is256BitVector() && NumElts != 4 && NumElts != 8 && - (!HasInt256 || (NumElts != 16 && NumElts != 32))) + } + case X86ISD::VPERMI: + ImmN = N->getOperand(N->getNumOperands()-1); + DecodeVPERMMask(cast(ImmN)->getZExtValue(), Mask); + IsUnary = true; + break; + case X86ISD::MOVSS: + case X86ISD::MOVSD: + DecodeScalarMoveMask(VT, /* IsLoad */ false, Mask); + break; + case X86ISD::VPERM2X128: + ImmN = N->getOperand(N->getNumOperands()-1); + DecodeVPERM2X128Mask(VT, cast(ImmN)->getZExtValue(), Mask); + if (Mask.empty()) return false; + break; + case X86ISD::MOVSLDUP: + DecodeMOVSLDUPMask(VT, Mask); + IsUnary = true; + break; + case X86ISD::MOVSHDUP: + DecodeMOVSHDUPMask(VT, Mask); + IsUnary = true; + break; + case X86ISD::MOVDDUP: + DecodeMOVDDUPMask(VT, Mask); + IsUnary = true; + break; + case X86ISD::MOVLHPD: + case X86ISD::MOVLPD: + case X86ISD::MOVLPS: + // Not yet implemented return false; - - // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate - // independently on 128-bit lanes. - unsigned NumLanes = VT.getSizeInBits()/128; - unsigned NumLaneElts = NumElts/NumLanes; - - for (unsigned l = 0; l != NumElts; l += NumLaneElts) { - for (unsigned i = 0, j = l+NumLaneElts/2; i != NumLaneElts; i += 2, ++j) { - int BitI = Mask[l+i]; - int BitI1 = Mask[l+i+1]; - if (!isUndefOrEqual(BitI, j)) - return false; - if (!isUndefOrEqual(BitI1, j)) - return false; - } + default: llvm_unreachable("unknown target shuffle node"); } - return true; -} -// Match for INSERTI64x4 INSERTF64x4 instructions (src0[0], src1[0]) or -// (src1[0], src0[1]), manipulation with 256-bit sub-vectors -static bool isINSERT64x4Mask(ArrayRef Mask, MVT VT, unsigned int *Imm) { - if (!VT.is512BitVector()) - return false; + // If we have a fake unary shuffle, the shuffle mask is spread across two + // inputs that are actually the same node. Re-map the mask to always point + // into the first input. + if (IsFakeUnary) + for (int &M : Mask) + if (M >= (int)Mask.size()) + M -= Mask.size(); - unsigned NumElts = VT.getVectorNumElements(); - unsigned HalfSize = NumElts/2; - if (isSequentialOrUndefInRange(Mask, 0, HalfSize, 0)) { - if (isSequentialOrUndefInRange(Mask, HalfSize, HalfSize, NumElts)) { - *Imm = 1; - return true; - } - } - if (isSequentialOrUndefInRange(Mask, 0, HalfSize, NumElts)) { - if (isSequentialOrUndefInRange(Mask, HalfSize, HalfSize, HalfSize)) { - *Imm = 0; - return true; - } - } - return false; + return true; } -/// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand -/// specifies a shuffle of elements that is suitable for input to MOVSS, -/// MOVSD, and MOVD, i.e. setting the lowest element. -static bool isMOVLMask(ArrayRef Mask, EVT VT) { - if (VT.getVectorElementType().getSizeInBits() < 32) - return false; - if (!VT.is128BitVector()) - return false; +/// getShuffleScalarElt - Returns the scalar element that will make up the ith +/// element of the result of the vector shuffle. +static SDValue getShuffleScalarElt(SDNode *N, unsigned Index, SelectionDAG &DAG, + unsigned Depth) { + if (Depth == 6) + return SDValue(); // Limit search depth. - unsigned NumElts = VT.getVectorNumElements(); + SDValue V = SDValue(N, 0); + EVT VT = V.getValueType(); + unsigned Opcode = V.getOpcode(); - if (!isUndefOrEqual(Mask[0], NumElts)) - return false; + // Recurse into ISD::VECTOR_SHUFFLE node to find scalars. + if (const ShuffleVectorSDNode *SV = dyn_cast(N)) { + int Elt = SV->getMaskElt(Index); - for (unsigned i = 1; i != NumElts; ++i) - if (!isUndefOrEqual(Mask[i], i)) - return false; + if (Elt < 0) + return DAG.getUNDEF(VT.getVectorElementType()); - return true; -} + unsigned NumElems = VT.getVectorNumElements(); + SDValue NewV = (Elt < (int)NumElems) ? SV->getOperand(0) + : SV->getOperand(1); + return getShuffleScalarElt(NewV.getNode(), Elt % NumElems, DAG, Depth+1); + } -/// isVPERM2X128Mask - Match 256-bit shuffles where the elements are considered -/// as permutations between 128-bit chunks or halves. As an example: this -/// shuffle bellow: -/// vector_shuffle <4, 5, 6, 7, 12, 13, 14, 15> -/// The first half comes from the second half of V1 and the second half from the -/// the second half of V2. -static bool isVPERM2X128Mask(ArrayRef Mask, MVT VT, bool HasFp256) { - if (!HasFp256 || !VT.is256BitVector()) - return false; + // Recurse into target specific vector shuffles to find scalars. + if (isTargetShuffle(Opcode)) { + MVT ShufVT = V.getSimpleValueType(); + unsigned NumElems = ShufVT.getVectorNumElements(); + SmallVector ShuffleMask; + bool IsUnary; - // The shuffle result is divided into half A and half B. In total the two - // sources have 4 halves, namely: C, D, E, F. The final values of A and - // B must come from C, D, E or F. - unsigned HalfSize = VT.getVectorNumElements()/2; - bool MatchA = false, MatchB = false; + if (!getTargetShuffleMask(N, ShufVT, ShuffleMask, IsUnary)) + return SDValue(); - // Check if A comes from one of C, D, E, F. - for (unsigned Half = 0; Half != 4; ++Half) { - if (isSequentialOrUndefInRange(Mask, 0, HalfSize, Half*HalfSize)) { - MatchA = true; - break; - } - } + int Elt = ShuffleMask[Index]; + if (Elt < 0) + return DAG.getUNDEF(ShufVT.getVectorElementType()); - // Check if B comes from one of C, D, E, F. - for (unsigned Half = 0; Half != 4; ++Half) { - if (isSequentialOrUndefInRange(Mask, HalfSize, HalfSize, Half*HalfSize)) { - MatchB = true; - break; - } + SDValue NewV = (Elt < (int)NumElems) ? N->getOperand(0) + : N->getOperand(1); + return getShuffleScalarElt(NewV.getNode(), Elt % NumElems, DAG, + Depth+1); } - return MatchA && MatchB; -} + // Actual nodes that may contain scalar elements + if (Opcode == ISD::BITCAST) { + V = V.getOperand(0); + EVT SrcVT = V.getValueType(); + unsigned NumElems = VT.getVectorNumElements(); -/// getShuffleVPERM2X128Immediate - Return the appropriate immediate to shuffle -/// the specified VECTOR_MASK mask with VPERM2F128/VPERM2I128 instructions. -static unsigned getShuffleVPERM2X128Immediate(ShuffleVectorSDNode *SVOp) { - MVT VT = SVOp->getSimpleValueType(0); + if (!SrcVT.isVector() || SrcVT.getVectorNumElements() != NumElems) + return SDValue(); + } - unsigned HalfSize = VT.getVectorNumElements()/2; + if (V.getOpcode() == ISD::SCALAR_TO_VECTOR) + return (Index == 0) ? V.getOperand(0) + : DAG.getUNDEF(VT.getVectorElementType()); - unsigned FstHalf = 0, SndHalf = 0; - for (unsigned i = 0; i < HalfSize; ++i) { - if (SVOp->getMaskElt(i) > 0) { - FstHalf = SVOp->getMaskElt(i)/HalfSize; - break; - } - } - for (unsigned i = HalfSize; i < HalfSize*2; ++i) { - if (SVOp->getMaskElt(i) > 0) { - SndHalf = SVOp->getMaskElt(i)/HalfSize; - break; - } - } + if (V.getOpcode() == ISD::BUILD_VECTOR) + return V.getOperand(Index); - return (FstHalf | (SndHalf << 4)); + return SDValue(); } -// Symmetric in-lane mask. Each lane has 4 elements (for imm8) -static bool isPermImmMask(ArrayRef Mask, MVT VT, unsigned& Imm8) { - unsigned EltSize = VT.getVectorElementType().getSizeInBits(); - if (EltSize < 32) - return false; +/// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8. +/// +static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros, + unsigned NumNonZero, unsigned NumZero, + SelectionDAG &DAG, + const X86Subtarget* Subtarget, + const TargetLowering &TLI) { + if (NumNonZero > 8) + return SDValue(); - unsigned NumElts = VT.getVectorNumElements(); - Imm8 = 0; - if (VT.is128BitVector() || (VT.is256BitVector() && EltSize == 64)) { - for (unsigned i = 0; i != NumElts; ++i) { - if (Mask[i] < 0) - continue; - Imm8 |= Mask[i] << (i*2); + SDLoc dl(Op); + SDValue V; + bool First = true; + for (unsigned i = 0; i < 16; ++i) { + bool ThisIsNonZero = (NonZeros & (1 << i)) != 0; + if (ThisIsNonZero && First) { + if (NumZero) + V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl); + else + V = DAG.getUNDEF(MVT::v8i16); + First = false; } - return true; - } - - unsigned LaneSize = 4; - SmallVector MaskVal(LaneSize, -1); - for (unsigned l = 0; l != NumElts; l += LaneSize) { - for (unsigned i = 0; i != LaneSize; ++i) { - if (!isUndefOrInRange(Mask[i+l], l, l+LaneSize)) - return false; - if (Mask[i+l] < 0) - continue; - if (MaskVal[i] < 0) { - MaskVal[i] = Mask[i+l] - l; - Imm8 |= MaskVal[i] << (i*2); - continue; + if ((i & 1) != 0) { + SDValue ThisElt, LastElt; + bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0; + if (LastIsNonZero) { + LastElt = DAG.getNode(ISD::ZERO_EXTEND, dl, + MVT::i16, Op.getOperand(i-1)); } - if (Mask[i+l] != (signed)(MaskVal[i]+l)) - return false; + if (ThisIsNonZero) { + ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i)); + ThisElt = DAG.getNode(ISD::SHL, dl, MVT::i16, + ThisElt, DAG.getConstant(8, MVT::i8)); + if (LastIsNonZero) + ThisElt = DAG.getNode(ISD::OR, dl, MVT::i16, ThisElt, LastElt); + } else + ThisElt = LastElt; + + if (ThisElt.getNode()) + V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, ThisElt, + DAG.getIntPtrConstant(i/2)); } } - return true; -} -/// isVPERMILPMask - Return true if the specified VECTOR_SHUFFLE operand -/// specifies a shuffle of elements that is suitable for input to VPERMILPD*. -/// Note that VPERMIL mask matching is different depending whether theunderlying -/// type is 32 or 64. In the VPERMILPS the high half of the mask should point -/// to the same elements of the low, but to the higher half of the source. -/// In VPERMILPD the two lanes could be shuffled independently of each other -/// with the same restriction that lanes can't be crossed. Also handles PSHUFDY. -static bool isVPERMILPMask(ArrayRef Mask, MVT VT) { - unsigned EltSize = VT.getVectorElementType().getSizeInBits(); - if (VT.getSizeInBits() < 256 || EltSize < 32) - return false; - bool symmetricMaskRequired = (EltSize == 32); - unsigned NumElts = VT.getVectorNumElements(); + return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V); +} - unsigned NumLanes = VT.getSizeInBits()/128; - unsigned LaneSize = NumElts/NumLanes; - // 2 or 4 elements in one lane +/// LowerBuildVectorv8i16 - Custom lower build_vector of v8i16. +/// +static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros, + unsigned NumNonZero, unsigned NumZero, + SelectionDAG &DAG, + const X86Subtarget* Subtarget, + const TargetLowering &TLI) { + if (NumNonZero > 4) + return SDValue(); - SmallVector ExpectedMaskVal(LaneSize, -1); - for (unsigned l = 0; l != NumElts; l += LaneSize) { - for (unsigned i = 0; i != LaneSize; ++i) { - if (!isUndefOrInRange(Mask[i+l], l, l+LaneSize)) - return false; - if (symmetricMaskRequired) { - if (ExpectedMaskVal[i] < 0 && Mask[i+l] >= 0) { - ExpectedMaskVal[i] = Mask[i+l] - l; - continue; - } - if (!isUndefOrEqual(Mask[i+l], ExpectedMaskVal[i]+l)) - return false; + SDLoc dl(Op); + SDValue V; + bool First = true; + for (unsigned i = 0; i < 8; ++i) { + bool isNonZero = (NonZeros & (1 << i)) != 0; + if (isNonZero) { + if (First) { + if (NumZero) + V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl); + else + V = DAG.getUNDEF(MVT::v8i16); + First = false; } + V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, + MVT::v8i16, V, Op.getOperand(i), + DAG.getIntPtrConstant(i)); } } - return true; + + return V; } -/// isCommutedMOVLMask - Returns true if the shuffle mask is except the reverse -/// of what x86 movss want. X86 movs requires the lowest element to be lowest -/// element of vector 2 and the other elements to come from vector 1 in order. -static bool isCommutedMOVLMask(ArrayRef Mask, MVT VT, - bool V2IsSplat = false, bool V2IsUndef = false) { - if (!VT.is128BitVector()) - return false; +/// LowerBuildVectorv4x32 - Custom lower build_vector of v4i32 or v4f32. +static SDValue LowerBuildVectorv4x32(SDValue Op, SelectionDAG &DAG, + const X86Subtarget *Subtarget, + const TargetLowering &TLI) { + // Find all zeroable elements. + std::bitset<4> Zeroable; + for (int i=0; i < 4; ++i) { + SDValue Elt = Op->getOperand(i); + Zeroable[i] = (Elt.getOpcode() == ISD::UNDEF || X86::isZeroNode(Elt)); + } + assert(Zeroable.size() - Zeroable.count() > 1 && + "We expect at least two non-zero elements!"); - unsigned NumOps = VT.getVectorNumElements(); - if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16) - return false; + // We only know how to deal with build_vector nodes where elements are either + // zeroable or extract_vector_elt with constant index. + SDValue FirstNonZero; + unsigned FirstNonZeroIdx; + for (unsigned i=0; i < 4; ++i) { + if (Zeroable[i]) + continue; + SDValue Elt = Op->getOperand(i); + if (Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT || + !isa(Elt.getOperand(1))) + return SDValue(); + // Make sure that this node is extracting from a 128-bit vector. + MVT VT = Elt.getOperand(0).getSimpleValueType(); + if (!VT.is128BitVector()) + return SDValue(); + if (!FirstNonZero.getNode()) { + FirstNonZero = Elt; + FirstNonZeroIdx = i; + } + } - if (!isUndefOrEqual(Mask[0], 0)) - return false; + assert(FirstNonZero.getNode() && "Unexpected build vector of all zeros!"); + SDValue V1 = FirstNonZero.getOperand(0); + MVT VT = V1.getSimpleValueType(); - for (unsigned i = 1; i != NumOps; ++i) - if (!(isUndefOrEqual(Mask[i], i+NumOps) || - (V2IsUndef && isUndefOrInRange(Mask[i], NumOps, NumOps*2)) || - (V2IsSplat && isUndefOrEqual(Mask[i], NumOps)))) - return false; - - return true; -} - -/// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand -/// specifies a shuffle of elements that is suitable for input to MOVSHDUP. -/// Masks to match: <1, 1, 3, 3> or <1, 1, 3, 3, 5, 5, 7, 7> -static bool isMOVSHDUPMask(ArrayRef Mask, MVT VT, - const X86Subtarget *Subtarget) { - if (!Subtarget->hasSSE3()) - return false; - - unsigned NumElems = VT.getVectorNumElements(); - - if ((VT.is128BitVector() && NumElems != 4) || - (VT.is256BitVector() && NumElems != 8) || - (VT.is512BitVector() && NumElems != 16)) - return false; - - // "i+1" is the value the indexed mask element must have - for (unsigned i = 0; i != NumElems; i += 2) - if (!isUndefOrEqual(Mask[i], i+1) || - !isUndefOrEqual(Mask[i+1], i+1)) - return false; - - return true; -} - -/// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand -/// specifies a shuffle of elements that is suitable for input to MOVSLDUP. -/// Masks to match: <0, 0, 2, 2> or <0, 0, 2, 2, 4, 4, 6, 6> -static bool isMOVSLDUPMask(ArrayRef Mask, MVT VT, - const X86Subtarget *Subtarget) { - if (!Subtarget->hasSSE3()) - return false; - - unsigned NumElems = VT.getVectorNumElements(); - - if ((VT.is128BitVector() && NumElems != 4) || - (VT.is256BitVector() && NumElems != 8) || - (VT.is512BitVector() && NumElems != 16)) - return false; - - // "i" is the value the indexed mask element must have - for (unsigned i = 0; i != NumElems; i += 2) - if (!isUndefOrEqual(Mask[i], i) || - !isUndefOrEqual(Mask[i+1], i)) - return false; + // See if this build_vector can be lowered as a blend with zero. + SDValue Elt; + unsigned EltMaskIdx, EltIdx; + int Mask[4]; + for (EltIdx = 0; EltIdx < 4; ++EltIdx) { + if (Zeroable[EltIdx]) { + // The zero vector will be on the right hand side. + Mask[EltIdx] = EltIdx+4; + continue; + } - return true; -} + Elt = Op->getOperand(EltIdx); + // By construction, Elt is a EXTRACT_VECTOR_ELT with constant index. + EltMaskIdx = cast(Elt.getOperand(1))->getZExtValue(); + if (Elt.getOperand(0) != V1 || EltMaskIdx != EltIdx) + break; + Mask[EltIdx] = EltIdx; + } -/// isMOVDDUPYMask - Return true if the specified VECTOR_SHUFFLE operand -/// specifies a shuffle of elements that is suitable for input to 256-bit -/// version of MOVDDUP. -static bool isMOVDDUPYMask(ArrayRef Mask, MVT VT, bool HasFp256) { - if (!HasFp256 || !VT.is256BitVector()) - return false; + if (EltIdx == 4) { + // Let the shuffle legalizer deal with blend operations. + SDValue VZero = getZeroVector(VT, Subtarget, DAG, SDLoc(Op)); + if (V1.getSimpleValueType() != VT) + V1 = DAG.getNode(ISD::BITCAST, SDLoc(V1), VT, V1); + return DAG.getVectorShuffle(VT, SDLoc(V1), V1, VZero, &Mask[0]); + } - unsigned NumElts = VT.getVectorNumElements(); - if (NumElts != 4) - return false; + // See if we can lower this build_vector to a INSERTPS. + if (!Subtarget->hasSSE41()) + return SDValue(); - for (unsigned i = 0; i != NumElts/2; ++i) - if (!isUndefOrEqual(Mask[i], 0)) - return false; - for (unsigned i = NumElts/2; i != NumElts; ++i) - if (!isUndefOrEqual(Mask[i], NumElts/2)) - return false; - return true; -} + SDValue V2 = Elt.getOperand(0); + if (Elt == FirstNonZero && EltIdx == FirstNonZeroIdx) + V1 = SDValue(); -/// isMOVDDUPMask - Return true if the specified VECTOR_SHUFFLE operand -/// specifies a shuffle of elements that is suitable for input to 128-bit -/// version of MOVDDUP. -static bool isMOVDDUPMask(ArrayRef Mask, MVT VT) { - if (!VT.is128BitVector()) - return false; + bool CanFold = true; + for (unsigned i = EltIdx + 1; i < 4 && CanFold; ++i) { + if (Zeroable[i]) + continue; - unsigned e = VT.getVectorNumElements() / 2; - for (unsigned i = 0; i != e; ++i) - if (!isUndefOrEqual(Mask[i], i)) - return false; - for (unsigned i = 0; i != e; ++i) - if (!isUndefOrEqual(Mask[e+i], i)) - return false; - return true; -} + SDValue Current = Op->getOperand(i); + SDValue SrcVector = Current->getOperand(0); + if (!V1.getNode()) + V1 = SrcVector; + CanFold = SrcVector == V1 && + cast(Current.getOperand(1))->getZExtValue() == i; + } -/// isVEXTRACTIndex - Return true if the specified -/// EXTRACT_SUBVECTOR operand specifies a vector extract that is -/// suitable for instruction that extract 128 or 256 bit vectors -static bool isVEXTRACTIndex(SDNode *N, unsigned vecWidth) { - assert((vecWidth == 128 || vecWidth == 256) && "Unexpected vector width"); - if (!isa(N->getOperand(1).getNode())) - return false; + if (!CanFold) + return SDValue(); - // The index should be aligned on a vecWidth-bit boundary. - uint64_t Index = - cast(N->getOperand(1).getNode())->getZExtValue(); + assert(V1.getNode() && "Expected at least two non-zero elements!"); + if (V1.getSimpleValueType() != MVT::v4f32) + V1 = DAG.getNode(ISD::BITCAST, SDLoc(V1), MVT::v4f32, V1); + if (V2.getSimpleValueType() != MVT::v4f32) + V2 = DAG.getNode(ISD::BITCAST, SDLoc(V2), MVT::v4f32, V2); - MVT VT = N->getSimpleValueType(0); - unsigned ElSize = VT.getVectorElementType().getSizeInBits(); - bool Result = (Index * ElSize) % vecWidth == 0; + // Ok, we can emit an INSERTPS instruction. + unsigned ZMask = Zeroable.to_ulong(); - return Result; + unsigned InsertPSMask = EltMaskIdx << 6 | EltIdx << 4 | ZMask; + assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!"); + SDValue Result = DAG.getNode(X86ISD::INSERTPS, SDLoc(Op), MVT::v4f32, V1, V2, + DAG.getIntPtrConstant(InsertPSMask)); + return DAG.getNode(ISD::BITCAST, SDLoc(Op), VT, Result); } -/// isVINSERTIndex - Return true if the specified INSERT_SUBVECTOR -/// operand specifies a subvector insert that is suitable for input to -/// insertion of 128 or 256-bit subvectors -static bool isVINSERTIndex(SDNode *N, unsigned vecWidth) { - assert((vecWidth == 128 || vecWidth == 256) && "Unexpected vector width"); - if (!isa(N->getOperand(2).getNode())) - return false; - // The index should be aligned on a vecWidth-bit boundary. - uint64_t Index = - cast(N->getOperand(2).getNode())->getZExtValue(); - - MVT VT = N->getSimpleValueType(0); - unsigned ElSize = VT.getVectorElementType().getSizeInBits(); - bool Result = (Index * ElSize) % vecWidth == 0; - - return Result; +/// Return a vector logical shift node. +static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, + unsigned NumBits, SelectionDAG &DAG, + const TargetLowering &TLI, SDLoc dl) { + assert(VT.is128BitVector() && "Unknown type for VShift"); + MVT ShVT = MVT::v2i64; + unsigned Opc = isLeft ? X86ISD::VSHLDQ : X86ISD::VSRLDQ; + SrcOp = DAG.getNode(ISD::BITCAST, dl, ShVT, SrcOp); + MVT ScalarShiftTy = TLI.getScalarShiftAmountTy(SrcOp.getValueType()); + assert(NumBits % 8 == 0 && "Only support byte sized shifts"); + SDValue ShiftVal = DAG.getConstant(NumBits/8, ScalarShiftTy); + return DAG.getNode(ISD::BITCAST, dl, VT, + DAG.getNode(Opc, dl, ShVT, SrcOp, ShiftVal)); } -bool X86::isVINSERT128Index(SDNode *N) { - return isVINSERTIndex(N, 128); -} +static SDValue +LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, SDLoc dl, SelectionDAG &DAG) { -bool X86::isVINSERT256Index(SDNode *N) { - return isVINSERTIndex(N, 256); -} + // Check if the scalar load can be widened into a vector load. And if + // the address is "base + cst" see if the cst can be "absorbed" into + // the shuffle mask. + if (LoadSDNode *LD = dyn_cast(SrcOp)) { + SDValue Ptr = LD->getBasePtr(); + if (!ISD::isNormalLoad(LD) || LD->isVolatile()) + return SDValue(); + EVT PVT = LD->getValueType(0); + if (PVT != MVT::i32 && PVT != MVT::f32) + return SDValue(); -bool X86::isVEXTRACT128Index(SDNode *N) { - return isVEXTRACTIndex(N, 128); -} + int FI = -1; + int64_t Offset = 0; + if (FrameIndexSDNode *FINode = dyn_cast(Ptr)) { + FI = FINode->getIndex(); + Offset = 0; + } else if (DAG.isBaseWithConstantOffset(Ptr) && + isa(Ptr.getOperand(0))) { + FI = cast(Ptr.getOperand(0))->getIndex(); + Offset = Ptr.getConstantOperandVal(1); + Ptr = Ptr.getOperand(0); + } else { + return SDValue(); + } -bool X86::isVEXTRACT256Index(SDNode *N) { - return isVEXTRACTIndex(N, 256); -} + // FIXME: 256-bit vector instructions don't require a strict alignment, + // improve this code to support it better. + unsigned RequiredAlign = VT.getSizeInBits()/8; + SDValue Chain = LD->getChain(); + // Make sure the stack object alignment is at least 16 or 32. + MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); + if (DAG.InferPtrAlignment(Ptr) < RequiredAlign) { + if (MFI->isFixedObjectIndex(FI)) { + // Can't change the alignment. FIXME: It's possible to compute + // the exact stack offset and reference FI + adjust offset instead. + // If someone *really* cares about this. That's the way to implement it. + return SDValue(); + } else { + MFI->setObjectAlignment(FI, RequiredAlign); + } + } -/// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle -/// the specified VECTOR_SHUFFLE mask with PSHUF* and SHUFP* instructions. -/// Handles 128-bit and 256-bit. -static unsigned getShuffleSHUFImmediate(ShuffleVectorSDNode *N) { - MVT VT = N->getSimpleValueType(0); + // (Offset % 16 or 32) must be multiple of 4. Then address is then + // Ptr + (Offset & ~15). + if (Offset < 0) + return SDValue(); + if ((Offset % RequiredAlign) & 3) + return SDValue(); + int64_t StartOffset = Offset & ~(RequiredAlign-1); + if (StartOffset) + Ptr = DAG.getNode(ISD::ADD, SDLoc(Ptr), Ptr.getValueType(), + Ptr,DAG.getConstant(StartOffset, Ptr.getValueType())); - assert((VT.getSizeInBits() >= 128) && - "Unsupported vector type for PSHUF/SHUFP"); + int EltNo = (Offset - StartOffset) >> 2; + unsigned NumElems = VT.getVectorNumElements(); - // Handle 128 and 256-bit vector lengths. AVX defines PSHUF/SHUFP to operate - // independently on 128-bit lanes. - unsigned NumElts = VT.getVectorNumElements(); - unsigned NumLanes = VT.getSizeInBits()/128; - unsigned NumLaneElts = NumElts/NumLanes; + EVT NVT = EVT::getVectorVT(*DAG.getContext(), PVT, NumElems); + SDValue V1 = DAG.getLoad(NVT, dl, Chain, Ptr, + LD->getPointerInfo().getWithOffset(StartOffset), + false, false, false, 0); - assert((NumLaneElts == 2 || NumLaneElts == 4 || NumLaneElts == 8) && - "Only supports 2, 4 or 8 elements per lane"); + SmallVector Mask(NumElems, EltNo); - unsigned Shift = (NumLaneElts >= 4) ? 1 : 0; - unsigned Mask = 0; - for (unsigned i = 0; i != NumElts; ++i) { - int Elt = N->getMaskElt(i); - if (Elt < 0) continue; - Elt &= NumLaneElts - 1; - unsigned ShAmt = (i << Shift) % 8; - Mask |= Elt << ShAmt; + return DAG.getVectorShuffle(NVT, dl, V1, DAG.getUNDEF(NVT), &Mask[0]); } - return Mask; + return SDValue(); } -/// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle -/// the specified VECTOR_SHUFFLE mask with the PSHUFHW instruction. -static unsigned getShufflePSHUFHWImmediate(ShuffleVectorSDNode *N) { - MVT VT = N->getSimpleValueType(0); - - assert((VT == MVT::v8i16 || VT == MVT::v16i16) && - "Unsupported vector type for PSHUFHW"); +/// Given the initializing elements 'Elts' of a vector of type 'VT', see if the +/// elements can be replaced by a single large load which has the same value as +/// a build_vector or insert_subvector whose loaded operands are 'Elts'. +/// +/// Example: -> zextload a +/// +/// FIXME: we'd also like to handle the case where the last elements are zero +/// rather than undef via VZEXT_LOAD, but we do not detect that case today. +/// There's even a handy isZeroNode for that purpose. +static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef Elts, + SDLoc &DL, SelectionDAG &DAG, + bool isAfterLegalize) { + unsigned NumElems = Elts.size(); - unsigned NumElts = VT.getVectorNumElements(); + LoadSDNode *LDBase = nullptr; + unsigned LastLoadedElt = -1U; - unsigned Mask = 0; - for (unsigned l = 0; l != NumElts; l += 8) { - // 8 nodes per lane, but we only care about the last 4. - for (unsigned i = 0; i < 4; ++i) { - int Elt = N->getMaskElt(l+i+4); - if (Elt < 0) continue; - Elt &= 0x3; // only 2-bits. - Mask |= Elt << (i * 2); + // For each element in the initializer, see if we've found a load or an undef. + // If we don't find an initial load element, or later load elements are + // non-consecutive, bail out. + for (unsigned i = 0; i < NumElems; ++i) { + SDValue Elt = Elts[i]; + // Look through a bitcast. + if (Elt.getNode() && Elt.getOpcode() == ISD::BITCAST) + Elt = Elt.getOperand(0); + if (!Elt.getNode() || + (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.getNode()))) + return SDValue(); + if (!LDBase) { + if (Elt.getNode()->getOpcode() == ISD::UNDEF) + return SDValue(); + LDBase = cast(Elt.getNode()); + LastLoadedElt = i; + continue; } - } - - return Mask; -} - -/// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle -/// the specified VECTOR_SHUFFLE mask with the PSHUFLW instruction. -static unsigned getShufflePSHUFLWImmediate(ShuffleVectorSDNode *N) { - MVT VT = N->getSimpleValueType(0); - - assert((VT == MVT::v8i16 || VT == MVT::v16i16) && - "Unsupported vector type for PSHUFHW"); + if (Elt.getOpcode() == ISD::UNDEF) + continue; - unsigned NumElts = VT.getVectorNumElements(); - - unsigned Mask = 0; - for (unsigned l = 0; l != NumElts; l += 8) { - // 8 nodes per lane, but we only care about the first 4. - for (unsigned i = 0; i < 4; ++i) { - int Elt = N->getMaskElt(l+i); - if (Elt < 0) continue; - Elt &= 0x3; // only 2-bits - Mask |= Elt << (i * 2); - } + LoadSDNode *LD = cast(Elt); + EVT LdVT = Elt.getValueType(); + // Each loaded element must be the correct fractional portion of the + // requested vector load. + if (LdVT.getSizeInBits() != VT.getSizeInBits() / NumElems) + return SDValue(); + if (!DAG.isConsecutiveLoad(LD, LDBase, LdVT.getSizeInBits() / 8, i)) + return SDValue(); + LastLoadedElt = i; } - return Mask; -} + // If we have found an entire vector of loads and undefs, then return a large + // load of the entire vector width starting at the base pointer. If we found + // consecutive loads for the low half, generate a vzext_load node. + if (LastLoadedElt == NumElems - 1) { + assert(LDBase && "Did not find base load for merging consecutive loads"); + EVT EltVT = LDBase->getValueType(0); + // Ensure that the input vector size for the merged loads matches the + // cumulative size of the input elements. + if (VT.getSizeInBits() != EltVT.getSizeInBits() * NumElems) + return SDValue(); -/// \brief Return the appropriate immediate to shuffle the specified -/// VECTOR_SHUFFLE mask with the PALIGNR (if InterLane is false) or with -/// VALIGN (if Interlane is true) instructions. -static unsigned getShuffleAlignrImmediate(ShuffleVectorSDNode *SVOp, - bool InterLane) { - MVT VT = SVOp->getSimpleValueType(0); - unsigned EltSize = InterLane ? 1 : - VT.getVectorElementType().getSizeInBits() >> 3; + if (isAfterLegalize && + !DAG.getTargetLoweringInfo().isOperationLegal(ISD::LOAD, VT)) + return SDValue(); - unsigned NumElts = VT.getVectorNumElements(); - unsigned NumLanes = VT.is512BitVector() ? 1 : VT.getSizeInBits()/128; - unsigned NumLaneElts = NumElts/NumLanes; - - int Val = 0; - unsigned i; - for (i = 0; i != NumElts; ++i) { - Val = SVOp->getMaskElt(i); - if (Val >= 0) - break; - } - if (Val >= (int)NumElts) - Val -= NumElts - NumLaneElts; + SDValue NewLd = SDValue(); - assert(Val - i > 0 && "PALIGNR imm should be positive"); - return (Val - i) * EltSize; -} + NewLd = DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(), + LDBase->getPointerInfo(), LDBase->isVolatile(), + LDBase->isNonTemporal(), LDBase->isInvariant(), + LDBase->getAlignment()); -/// \brief Return the appropriate immediate to shuffle the specified -/// VECTOR_SHUFFLE mask with the PALIGNR instruction. -static unsigned getShufflePALIGNRImmediate(ShuffleVectorSDNode *SVOp) { - return getShuffleAlignrImmediate(SVOp, false); -} + if (LDBase->hasAnyUseOfValue(1)) { + SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, + SDValue(LDBase, 1), + SDValue(NewLd.getNode(), 1)); + DAG.ReplaceAllUsesOfValueWith(SDValue(LDBase, 1), NewChain); + DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(LDBase, 1), + SDValue(NewLd.getNode(), 1)); + } -/// \brief Return the appropriate immediate to shuffle the specified -/// VECTOR_SHUFFLE mask with the VALIGN instruction. -static unsigned getShuffleVALIGNImmediate(ShuffleVectorSDNode *SVOp) { - return getShuffleAlignrImmediate(SVOp, true); -} + return NewLd; + } + //TODO: The code below fires only for for loading the low v2i32 / v2f32 + //of a v4i32 / v4f32. It's probably worth generalizing. + EVT EltVT = VT.getVectorElementType(); + if (NumElems == 4 && LastLoadedElt == 1 && (EltVT.getSizeInBits() == 32) && + DAG.getTargetLoweringInfo().isTypeLegal(MVT::v2i64)) { + SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other); + SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() }; + SDValue ResNode = + DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, MVT::i64, + LDBase->getPointerInfo(), + LDBase->getAlignment(), + false/*isVolatile*/, true/*ReadMem*/, + false/*WriteMem*/); -static unsigned getExtractVEXTRACTImmediate(SDNode *N, unsigned vecWidth) { - assert((vecWidth == 128 || vecWidth == 256) && "Unsupported vector width"); - if (!isa(N->getOperand(1).getNode())) - llvm_unreachable("Illegal extract subvector for VEXTRACT"); + // Make sure the newly-created LOAD is in the same position as LDBase in + // terms of dependency. We create a TokenFactor for LDBase and ResNode, and + // update uses of LDBase's output chain to use the TokenFactor. + if (LDBase->hasAnyUseOfValue(1)) { + SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, + SDValue(LDBase, 1), SDValue(ResNode.getNode(), 1)); + DAG.ReplaceAllUsesOfValueWith(SDValue(LDBase, 1), NewChain); + DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(LDBase, 1), + SDValue(ResNode.getNode(), 1)); + } - uint64_t Index = - cast(N->getOperand(1).getNode())->getZExtValue(); + return DAG.getNode(ISD::BITCAST, DL, VT, ResNode); + } + return SDValue(); +} - MVT VecVT = N->getOperand(0).getSimpleValueType(); - MVT ElVT = VecVT.getVectorElementType(); +/// LowerVectorBroadcast - Attempt to use the vbroadcast instruction +/// to generate a splat value for the following cases: +/// 1. A splat BUILD_VECTOR which uses a single scalar load, or a constant. +/// 2. A splat shuffle which uses a scalar_to_vector node which comes from +/// a scalar load, or a constant. +/// The VBROADCAST node is returned when a pattern is found, +/// or SDValue() otherwise. +static SDValue LowerVectorBroadcast(SDValue Op, const X86Subtarget* Subtarget, + SelectionDAG &DAG) { + // VBROADCAST requires AVX. + // TODO: Splats could be generated for non-AVX CPUs using SSE + // instructions, but there's less potential gain for only 128-bit vectors. + if (!Subtarget->hasAVX()) + return SDValue(); - unsigned NumElemsPerChunk = vecWidth / ElVT.getSizeInBits(); - return Index / NumElemsPerChunk; -} + MVT VT = Op.getSimpleValueType(); + SDLoc dl(Op); -static unsigned getInsertVINSERTImmediate(SDNode *N, unsigned vecWidth) { - assert((vecWidth == 128 || vecWidth == 256) && "Unsupported vector width"); - if (!isa(N->getOperand(2).getNode())) - llvm_unreachable("Illegal insert subvector for VINSERT"); + assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) && + "Unsupported vector type for broadcast."); - uint64_t Index = - cast(N->getOperand(2).getNode())->getZExtValue(); + SDValue Ld; + bool ConstSplatVal; - MVT VecVT = N->getSimpleValueType(0); - MVT ElVT = VecVT.getVectorElementType(); + switch (Op.getOpcode()) { + default: + // Unknown pattern found. + return SDValue(); - unsigned NumElemsPerChunk = vecWidth / ElVT.getSizeInBits(); - return Index / NumElemsPerChunk; -} + case ISD::BUILD_VECTOR: { + auto *BVOp = cast(Op.getNode()); + BitVector UndefElements; + SDValue Splat = BVOp->getSplatValue(&UndefElements); -/// getExtractVEXTRACT128Immediate - Return the appropriate immediate -/// to extract the specified EXTRACT_SUBVECTOR index with VEXTRACTF128 -/// and VINSERTI128 instructions. -unsigned X86::getExtractVEXTRACT128Immediate(SDNode *N) { - return getExtractVEXTRACTImmediate(N, 128); -} + // We need a splat of a single value to use broadcast, and it doesn't + // make any sense if the value is only in one element of the vector. + if (!Splat || (VT.getVectorNumElements() - UndefElements.count()) <= 1) + return SDValue(); -/// getExtractVEXTRACT256Immediate - Return the appropriate immediate -/// to extract the specified EXTRACT_SUBVECTOR index with VEXTRACTF64x4 -/// and VINSERTI64x4 instructions. -unsigned X86::getExtractVEXTRACT256Immediate(SDNode *N) { - return getExtractVEXTRACTImmediate(N, 256); -} + Ld = Splat; + ConstSplatVal = (Ld.getOpcode() == ISD::Constant || + Ld.getOpcode() == ISD::ConstantFP); -/// getInsertVINSERT128Immediate - Return the appropriate immediate -/// to insert at the specified INSERT_SUBVECTOR index with VINSERTF128 -/// and VINSERTI128 instructions. -unsigned X86::getInsertVINSERT128Immediate(SDNode *N) { - return getInsertVINSERTImmediate(N, 128); -} + // Make sure that all of the users of a non-constant load are from the + // BUILD_VECTOR node. + if (!ConstSplatVal && !BVOp->isOnlyUserOf(Ld.getNode())) + return SDValue(); + break; + } -/// getInsertVINSERT256Immediate - Return the appropriate immediate -/// to insert at the specified INSERT_SUBVECTOR index with VINSERTF46x4 -/// and VINSERTI64x4 instructions. -unsigned X86::getInsertVINSERT256Immediate(SDNode *N) { - return getInsertVINSERTImmediate(N, 256); -} + case ISD::VECTOR_SHUFFLE: { + ShuffleVectorSDNode *SVOp = cast(Op); -/// isZero - Returns true if Elt is a constant integer zero -static bool isZero(SDValue V) { - ConstantSDNode *C = dyn_cast(V); - return C && C->isNullValue(); -} + // Shuffles must have a splat mask where the first element is + // broadcasted. + if ((!SVOp->isSplat()) || SVOp->getMaskElt(0) != 0) + return SDValue(); -/// isZeroNode - Returns true if Elt is a constant zero or a floating point -/// constant +0.0. -bool X86::isZeroNode(SDValue Elt) { - if (isZero(Elt)) - return true; - if (ConstantFPSDNode *CFP = dyn_cast(Elt)) - return CFP->getValueAPF().isPosZero(); - return false; -} + SDValue Sc = Op.getOperand(0); + if (Sc.getOpcode() != ISD::SCALAR_TO_VECTOR && + Sc.getOpcode() != ISD::BUILD_VECTOR) { -/// ShouldXformToMOVHLPS - Return true if the node should be transformed to -/// match movhlps. The lower half elements should come from upper half of -/// V1 (and in order), and the upper half elements should come from the upper -/// half of V2 (and in order). -static bool ShouldXformToMOVHLPS(ArrayRef Mask, MVT VT) { - if (!VT.is128BitVector()) - return false; - if (VT.getVectorNumElements() != 4) - return false; - for (unsigned i = 0, e = 2; i != e; ++i) - if (!isUndefOrEqual(Mask[i], i+2)) - return false; - for (unsigned i = 2; i != 4; ++i) - if (!isUndefOrEqual(Mask[i], i+4)) - return false; - return true; -} + if (!Subtarget->hasInt256()) + return SDValue(); -/// isScalarLoadToVector - Returns true if the node is a scalar load that -/// is promoted to a vector. It also returns the LoadSDNode by reference if -/// required. -static bool isScalarLoadToVector(SDNode *N, LoadSDNode **LD = nullptr) { - if (N->getOpcode() != ISD::SCALAR_TO_VECTOR) - return false; - N = N->getOperand(0).getNode(); - if (!ISD::isNON_EXTLoad(N)) - return false; - if (LD) - *LD = cast(N); - return true; -} + // Use the register form of the broadcast instruction available on AVX2. + if (VT.getSizeInBits() >= 256) + Sc = Extract128BitVector(Sc, 0, DAG, dl); + return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Sc); + } -// Test whether the given value is a vector value which will be legalized -// into a load. -static bool WillBeConstantPoolLoad(SDNode *N) { - if (N->getOpcode() != ISD::BUILD_VECTOR) - return false; + Ld = Sc.getOperand(0); + ConstSplatVal = (Ld.getOpcode() == ISD::Constant || + Ld.getOpcode() == ISD::ConstantFP); + + // The scalar_to_vector node and the suspected + // load node must have exactly one user. + // Constants may have multiple users. - // Check for any non-constant elements. - for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) - switch (N->getOperand(i).getNode()->getOpcode()) { - case ISD::UNDEF: - case ISD::ConstantFP: - case ISD::Constant: + // AVX-512 has register version of the broadcast + bool hasRegVer = Subtarget->hasAVX512() && VT.is512BitVector() && + Ld.getValueType().getSizeInBits() >= 32; + if (!ConstSplatVal && ((!Sc.hasOneUse() || !Ld.hasOneUse()) && + !hasRegVer)) + return SDValue(); break; - default: - return false; } + } - // Vectors of all-zeros and all-ones are materialized with special - // instructions rather than being loaded. - return !ISD::isBuildVectorAllZeros(N) && - !ISD::isBuildVectorAllOnes(N); -} + unsigned ScalarSize = Ld.getValueType().getSizeInBits(); + bool IsGE256 = (VT.getSizeInBits() >= 256); -/// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to -/// match movlp{s|d}. The lower half elements should come from lower half of -/// V1 (and in order), and the upper half elements should come from the upper -/// half of V2 (and in order). And since V1 will become the source of the -/// MOVLP, it must be either a vector load or a scalar load to vector. -static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2, - ArrayRef Mask, MVT VT) { - if (!VT.is128BitVector()) - return false; + // When optimizing for size, generate up to 5 extra bytes for a broadcast + // instruction to save 8 or more bytes of constant pool data. + // TODO: If multiple splats are generated to load the same constant, + // it may be detrimental to overall size. There needs to be a way to detect + // that condition to know if this is truly a size win. + const Function *F = DAG.getMachineFunction().getFunction(); + bool OptForSize = F->hasFnAttribute(Attribute::OptimizeForSize); - if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1)) - return false; - // Is V2 is a vector load, don't do this transformation. We will try to use - // load folding shufps op. - if (ISD::isNON_EXTLoad(V2) || WillBeConstantPoolLoad(V2)) - return false; + // Handle broadcasting a single constant scalar from the constant pool + // into a vector. + // On Sandybridge (no AVX2), it is still better to load a constant vector + // from the constant pool and not to broadcast it from a scalar. + // But override that restriction when optimizing for size. + // TODO: Check if splatting is recommended for other AVX-capable CPUs. + if (ConstSplatVal && (Subtarget->hasAVX2() || OptForSize)) { + EVT CVT = Ld.getValueType(); + assert(!CVT.isVector() && "Must not broadcast a vector type"); - unsigned NumElems = VT.getVectorNumElements(); + // Splat f32, i32, v4f64, v4i64 in all cases with AVX2. + // For size optimization, also splat v2f64 and v2i64, and for size opt + // with AVX2, also splat i8 and i16. + // With pattern matching, the VBROADCAST node may become a VMOVDDUP. + if (ScalarSize == 32 || (IsGE256 && ScalarSize == 64) || + (OptForSize && (ScalarSize == 64 || Subtarget->hasAVX2()))) { + const Constant *C = nullptr; + if (ConstantSDNode *CI = dyn_cast(Ld)) + C = CI->getConstantIntValue(); + else if (ConstantFPSDNode *CF = dyn_cast(Ld)) + C = CF->getConstantFPValue(); - if (NumElems != 2 && NumElems != 4) - return false; - for (unsigned i = 0, e = NumElems/2; i != e; ++i) - if (!isUndefOrEqual(Mask[i], i)) - return false; - for (unsigned i = NumElems/2, e = NumElems; i != e; ++i) - if (!isUndefOrEqual(Mask[i], i+NumElems)) - return false; - return true; -} + assert(C && "Invalid constant type"); -/// isZeroShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved -/// to an zero vector. -/// FIXME: move to dag combiner / method on ShuffleVectorSDNode -static bool isZeroShuffle(ShuffleVectorSDNode *N) { - SDValue V1 = N->getOperand(0); - SDValue V2 = N->getOperand(1); - unsigned NumElems = N->getValueType(0).getVectorNumElements(); - for (unsigned i = 0; i != NumElems; ++i) { - int Idx = N->getMaskElt(i); - if (Idx >= (int)NumElems) { - unsigned Opc = V2.getOpcode(); - if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.getNode())) - continue; - if (Opc != ISD::BUILD_VECTOR || - !X86::isZeroNode(V2.getOperand(Idx-NumElems))) - return false; - } else if (Idx >= 0) { - unsigned Opc = V1.getOpcode(); - if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V1.getNode())) - continue; - if (Opc != ISD::BUILD_VECTOR || - !X86::isZeroNode(V1.getOperand(Idx))) - return false; + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + SDValue CP = DAG.getConstantPool(C, TLI.getPointerTy()); + unsigned Alignment = cast(CP)->getAlignment(); + Ld = DAG.getLoad(CVT, dl, DAG.getEntryNode(), CP, + MachinePointerInfo::getConstantPool(), + false, false, false, Alignment); + + return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); } } - return true; -} -/// getZeroVector - Returns a vector of specified type with all zero elements. -/// -static SDValue getZeroVector(EVT VT, const X86Subtarget *Subtarget, - SelectionDAG &DAG, SDLoc dl) { - assert(VT.isVector() && "Expected a vector type"); + bool IsLoad = ISD::isNormalLoad(Ld.getNode()); - // Always build SSE zero vectors as <4 x i32> bitcasted - // to their dest type. This ensures they get CSE'd. - SDValue Vec; - if (VT.is128BitVector()) { // SSE - if (Subtarget->hasSSE2()) { // SSE2 - SDValue Cst = DAG.getConstant(0, MVT::i32); - Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); - } else { // SSE1 - SDValue Cst = DAG.getConstantFP(+0.0, MVT::f32); - Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f32, Cst, Cst, Cst, Cst); - } - } else if (VT.is256BitVector()) { // AVX - if (Subtarget->hasInt256()) { // AVX2 - SDValue Cst = DAG.getConstant(0, MVT::i32); - SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst }; - Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops); - } else { - // 256-bit logic and arithmetic instructions in AVX are all - // floating-point, no support for integer ops. Emit fp zeroed vectors. - SDValue Cst = DAG.getConstantFP(+0.0, MVT::f32); - SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst }; - Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8f32, Ops); - } - } else if (VT.is512BitVector()) { // AVX-512 - SDValue Cst = DAG.getConstant(0, MVT::i32); - SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst, - Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst }; - Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i32, Ops); - } else if (VT.getScalarType() == MVT::i1) { - assert(VT.getVectorNumElements() <= 16 && "Unexpected vector type"); - SDValue Cst = DAG.getConstant(0, MVT::i1); - SmallVector Ops(VT.getVectorNumElements(), Cst); - return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); - } else - llvm_unreachable("Unexpected vector type"); + // Handle AVX2 in-register broadcasts. + if (!IsLoad && Subtarget->hasInt256() && + (ScalarSize == 32 || (IsGE256 && ScalarSize == 64))) + return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); - return DAG.getNode(ISD::BITCAST, dl, VT, Vec); -} + // The scalar source must be a normal load. + if (!IsLoad) + return SDValue(); -/// getOnesVector - Returns a vector of specified type with all bits set. -/// Always build ones vectors as <4 x i32> or <8 x i32>. For 256-bit types with -/// no AVX2 supprt, use two <4 x i32> inserted in a <8 x i32> appropriately. -/// Then bitcast to their original type, ensuring they get CSE'd. -static SDValue getOnesVector(MVT VT, bool HasInt256, SelectionDAG &DAG, - SDLoc dl) { - assert(VT.isVector() && "Expected a vector type"); + if (ScalarSize == 32 || (IsGE256 && ScalarSize == 64) || + (Subtarget->hasVLX() && ScalarSize == 64)) + return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); - SDValue Cst = DAG.getConstant(~0U, MVT::i32); - SDValue Vec; - if (VT.is256BitVector()) { - if (HasInt256) { // AVX2 - SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst }; - Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops); - } else { // AVX - Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); - Vec = Concat128BitVectors(Vec, Vec, MVT::v8i32, 8, DAG, dl); - } - } else if (VT.is128BitVector()) { - Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); - } else - llvm_unreachable("Unexpected vector type"); + // The integer check is needed for the 64-bit into 128-bit so it doesn't match + // double since there is no vbroadcastsd xmm + if (Subtarget->hasInt256() && Ld.getValueType().isInteger()) { + if (ScalarSize == 8 || ScalarSize == 16 || ScalarSize == 64) + return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); + } - return DAG.getNode(ISD::BITCAST, dl, VT, Vec); + // Unsupported broadcast. + return SDValue(); } -/// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements -/// that point to V2 points to its first element. -static void NormalizeMask(SmallVectorImpl &Mask, unsigned NumElems) { - for (unsigned i = 0; i != NumElems; ++i) { - if (Mask[i] > (int)NumElems) { - Mask[i] = NumElems; - } - } -} +/// \brief For an EXTRACT_VECTOR_ELT with a constant index return the real +/// underlying vector and index. +/// +/// Modifies \p ExtractedFromVec to the real vector and returns the real +/// index. +static int getUnderlyingExtractedFromVec(SDValue &ExtractedFromVec, + SDValue ExtIdx) { + int Idx = cast(ExtIdx)->getZExtValue(); + if (!isa(ExtractedFromVec)) + return Idx; -/// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd -/// operation of specified width. -static SDValue getMOVL(SelectionDAG &DAG, SDLoc dl, EVT VT, SDValue V1, - SDValue V2) { - unsigned NumElems = VT.getVectorNumElements(); - SmallVector Mask; - Mask.push_back(NumElems); - for (unsigned i = 1; i != NumElems; ++i) - Mask.push_back(i); - return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); -} + // For 256-bit vectors, LowerEXTRACT_VECTOR_ELT_SSE4 may have already + // lowered this: + // (extract_vector_elt (v8f32 %vreg1), Constant<6>) + // to: + // (extract_vector_elt (vector_shuffle<2,u,u,u> + // (extract_subvector (v8f32 %vreg0), Constant<4>), + // undef) + // Constant<0>) + // In this case the vector is the extract_subvector expression and the index + // is 2, as specified by the shuffle. + ShuffleVectorSDNode *SVOp = cast(ExtractedFromVec); + SDValue ShuffleVec = SVOp->getOperand(0); + MVT ShuffleVecVT = ShuffleVec.getSimpleValueType(); + assert(ShuffleVecVT.getVectorElementType() == + ExtractedFromVec.getSimpleValueType().getVectorElementType()); -/// getUnpackl - Returns a vector_shuffle node for an unpackl operation. -static SDValue getUnpackl(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1, - SDValue V2) { - unsigned NumElems = VT.getVectorNumElements(); - SmallVector Mask; - for (unsigned i = 0, e = NumElems/2; i != e; ++i) { - Mask.push_back(i); - Mask.push_back(i + NumElems); + int ShuffleIdx = SVOp->getMaskElt(Idx); + if (isUndefOrInRange(ShuffleIdx, 0, ShuffleVecVT.getVectorNumElements())) { + ExtractedFromVec = ShuffleVec; + return ShuffleIdx; } - return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); + return Idx; } -/// getUnpackh - Returns a vector_shuffle node for an unpackh operation. -static SDValue getUnpackh(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1, - SDValue V2) { - unsigned NumElems = VT.getVectorNumElements(); - SmallVector Mask; - for (unsigned i = 0, Half = NumElems/2; i != Half; ++i) { - Mask.push_back(i + Half); - Mask.push_back(i + NumElems + Half); - } - return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]); -} +static SDValue buildFromShuffleMostly(SDValue Op, SelectionDAG &DAG) { + MVT VT = Op.getSimpleValueType(); -// PromoteSplati8i16 - All i16 and i8 vector types can't be used directly by -// a generic shuffle instruction because the target has no such instructions. -// Generate shuffles which repeat i16 and i8 several times until they can be -// represented by v4f32 and then be manipulated by target suported shuffles. -static SDValue PromoteSplati8i16(SDValue V, SelectionDAG &DAG, int &EltNo) { - MVT VT = V.getSimpleValueType(); - int NumElems = VT.getVectorNumElements(); - SDLoc dl(V); + // Skip if insert_vec_elt is not supported. + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + if (!TLI.isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT)) + return SDValue(); - while (NumElems > 4) { - if (EltNo < NumElems/2) { - V = getUnpackl(DAG, dl, VT, V, V); - } else { - V = getUnpackh(DAG, dl, VT, V, V); - EltNo -= NumElems/2; - } - NumElems >>= 1; - } - return V; -} + SDLoc DL(Op); + unsigned NumElems = Op.getNumOperands(); -/// getLegalSplat - Generate a legal splat with supported x86 shuffles -static SDValue getLegalSplat(SelectionDAG &DAG, SDValue V, int EltNo) { - MVT VT = V.getSimpleValueType(); - SDLoc dl(V); - - if (VT.is128BitVector()) { - V = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V); - int SplatMask[4] = { EltNo, EltNo, EltNo, EltNo }; - V = DAG.getVectorShuffle(MVT::v4f32, dl, V, DAG.getUNDEF(MVT::v4f32), - &SplatMask[0]); - } else if (VT.is256BitVector()) { - // To use VPERMILPS to splat scalars, the second half of indicies must - // refer to the higher part, which is a duplication of the lower one, - // because VPERMILPS can only handle in-lane permutations. - int SplatMask[8] = { EltNo, EltNo, EltNo, EltNo, - EltNo+4, EltNo+4, EltNo+4, EltNo+4 }; - - V = DAG.getNode(ISD::BITCAST, dl, MVT::v8f32, V); - V = DAG.getVectorShuffle(MVT::v8f32, dl, V, DAG.getUNDEF(MVT::v8f32), - &SplatMask[0]); - } else - llvm_unreachable("Vector size not supported"); + SDValue VecIn1; + SDValue VecIn2; + SmallVector InsertIndices; + SmallVector Mask(NumElems, -1); - return DAG.getNode(ISD::BITCAST, dl, VT, V); -} + for (unsigned i = 0; i != NumElems; ++i) { + unsigned Opc = Op.getOperand(i).getOpcode(); -/// PromoteSplat - Splat is promoted to target supported vector shuffles. -static SDValue PromoteSplat(ShuffleVectorSDNode *SV, SelectionDAG &DAG) { - MVT SrcVT = SV->getSimpleValueType(0); - SDValue V1 = SV->getOperand(0); - SDLoc dl(SV); + if (Opc == ISD::UNDEF) + continue; - int EltNo = SV->getSplatIndex(); - int NumElems = SrcVT.getVectorNumElements(); - bool Is256BitVec = SrcVT.is256BitVector(); + if (Opc != ISD::EXTRACT_VECTOR_ELT) { + // Quit if more than 1 elements need inserting. + if (InsertIndices.size() > 1) + return SDValue(); - assert(((SrcVT.is128BitVector() && NumElems > 4) || Is256BitVec) && - "Unknown how to promote splat for type"); + InsertIndices.push_back(i); + continue; + } - // Extract the 128-bit part containing the splat element and update - // the splat element index when it refers to the higher register. - if (Is256BitVec) { - V1 = Extract128BitVector(V1, EltNo, DAG, dl); - if (EltNo >= NumElems/2) - EltNo -= NumElems/2; - } + SDValue ExtractedFromVec = Op.getOperand(i).getOperand(0); + SDValue ExtIdx = Op.getOperand(i).getOperand(1); + // Quit if non-constant index. + if (!isa(ExtIdx)) + return SDValue(); + int Idx = getUnderlyingExtractedFromVec(ExtractedFromVec, ExtIdx); + + // Quit if extracted from vector of different type. + if (ExtractedFromVec.getValueType() != VT) + return SDValue(); - // All i16 and i8 vector types can't be used directly by a generic shuffle - // instruction because the target has no such instruction. Generate shuffles - // which repeat i16 and i8 several times until they fit in i32, and then can - // be manipulated by target suported shuffles. - MVT EltVT = SrcVT.getVectorElementType(); - if (EltVT == MVT::i8 || EltVT == MVT::i16) - V1 = PromoteSplati8i16(V1, DAG, EltNo); + if (!VecIn1.getNode()) + VecIn1 = ExtractedFromVec; + else if (VecIn1 != ExtractedFromVec) { + if (!VecIn2.getNode()) + VecIn2 = ExtractedFromVec; + else if (VecIn2 != ExtractedFromVec) + // Quit if more than 2 vectors to shuffle + return SDValue(); + } - // Recreate the 256-bit vector and place the same 128-bit vector - // into the low and high part. This is necessary because we want - // to use VPERM* to shuffle the vectors - if (Is256BitVec) { - V1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT, V1, V1); + if (ExtractedFromVec == VecIn1) + Mask[i] = Idx; + else if (ExtractedFromVec == VecIn2) + Mask[i] = Idx + NumElems; } - return getLegalSplat(DAG, V1, EltNo); -} + if (!VecIn1.getNode()) + return SDValue(); -/// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified -/// vector of zero or undef vector. This produces a shuffle where the low -/// element of V2 is swizzled into the zero/undef vector, landing at element -/// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3). -static SDValue getShuffleVectorZeroOrUndef(SDValue V2, unsigned Idx, - bool IsZero, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - MVT VT = V2.getSimpleValueType(); - SDValue V1 = IsZero - ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT); - unsigned NumElems = VT.getVectorNumElements(); - SmallVector MaskVec; - for (unsigned i = 0; i != NumElems; ++i) - // If this is the insertion idx, put the low elt of V2 here. - MaskVec.push_back(i == Idx ? NumElems : i); - return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, &MaskVec[0]); + VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT); + SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, &Mask[0]); + for (unsigned i = 0, e = InsertIndices.size(); i != e; ++i) { + unsigned Idx = InsertIndices[i]; + NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx), + DAG.getIntPtrConstant(Idx)); + } + + return NV; } -/// getTargetShuffleMask - Calculates the shuffle mask corresponding to the -/// target specific opcode. Returns true if the Mask could be calculated. Sets -/// IsUnary to true if only uses one source. Note that this will set IsUnary for -/// shuffles which use a single input multiple times, and in those cases it will -/// adjust the mask to only have indices within that single input. -static bool getTargetShuffleMask(SDNode *N, MVT VT, - SmallVectorImpl &Mask, bool &IsUnary) { - unsigned NumElems = VT.getVectorNumElements(); - SDValue ImmN; - - IsUnary = false; - bool IsFakeUnary = false; - switch(N->getOpcode()) { - case X86ISD::BLENDI: - ImmN = N->getOperand(N->getNumOperands()-1); - DecodeBLENDMask(VT, cast(ImmN)->getZExtValue(), Mask); - break; - case X86ISD::SHUFP: - ImmN = N->getOperand(N->getNumOperands()-1); - DecodeSHUFPMask(VT, cast(ImmN)->getZExtValue(), Mask); - IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1); - break; - case X86ISD::UNPCKH: - DecodeUNPCKHMask(VT, Mask); - IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1); - break; - case X86ISD::UNPCKL: - DecodeUNPCKLMask(VT, Mask); - IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1); - break; - case X86ISD::MOVHLPS: - DecodeMOVHLPSMask(NumElems, Mask); - IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1); - break; - case X86ISD::MOVLHPS: - DecodeMOVLHPSMask(NumElems, Mask); - IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1); - break; - case X86ISD::PALIGNR: - ImmN = N->getOperand(N->getNumOperands()-1); - DecodePALIGNRMask(VT, cast(ImmN)->getZExtValue(), Mask); - break; - case X86ISD::PSHUFD: - case X86ISD::VPERMILPI: - ImmN = N->getOperand(N->getNumOperands()-1); - DecodePSHUFMask(VT, cast(ImmN)->getZExtValue(), Mask); - IsUnary = true; - break; - case X86ISD::PSHUFHW: - ImmN = N->getOperand(N->getNumOperands()-1); - DecodePSHUFHWMask(VT, cast(ImmN)->getZExtValue(), Mask); - IsUnary = true; - break; - case X86ISD::PSHUFLW: - ImmN = N->getOperand(N->getNumOperands()-1); - DecodePSHUFLWMask(VT, cast(ImmN)->getZExtValue(), Mask); - IsUnary = true; - break; - case X86ISD::PSHUFB: { - IsUnary = true; - SDValue MaskNode = N->getOperand(1); - while (MaskNode->getOpcode() == ISD::BITCAST) - MaskNode = MaskNode->getOperand(0); +// Lower BUILD_VECTOR operation for v8i1 and v16i1 types. +SDValue +X86TargetLowering::LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG) const { - if (MaskNode->getOpcode() == ISD::BUILD_VECTOR) { - // If we have a build-vector, then things are easy. - EVT VT = MaskNode.getValueType(); - assert(VT.isVector() && - "Can't produce a non-vector with a build_vector!"); - if (!VT.isInteger()) - return false; + MVT VT = Op.getSimpleValueType(); + assert((VT.getVectorElementType() == MVT::i1) && (VT.getSizeInBits() <= 16) && + "Unexpected type in LowerBUILD_VECTORvXi1!"); - int NumBytesPerElement = VT.getVectorElementType().getSizeInBits() / 8; + SDLoc dl(Op); + if (ISD::isBuildVectorAllZeros(Op.getNode())) { + SDValue Cst = DAG.getTargetConstant(0, MVT::i1); + SmallVector Ops(VT.getVectorNumElements(), Cst); + return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); + } - SmallVector RawMask; - for (int i = 0, e = MaskNode->getNumOperands(); i < e; ++i) { - SDValue Op = MaskNode->getOperand(i); - if (Op->getOpcode() == ISD::UNDEF) { - RawMask.push_back((uint64_t)SM_SentinelUndef); - continue; - } - auto *CN = dyn_cast(Op.getNode()); - if (!CN) - return false; - APInt MaskElement = CN->getAPIntValue(); + if (ISD::isBuildVectorAllOnes(Op.getNode())) { + SDValue Cst = DAG.getTargetConstant(1, MVT::i1); + SmallVector Ops(VT.getVectorNumElements(), Cst); + return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); + } - // We now have to decode the element which could be any integer size and - // extract each byte of it. - for (int j = 0; j < NumBytesPerElement; ++j) { - // Note that this is x86 and so always little endian: the low byte is - // the first byte of the mask. - RawMask.push_back(MaskElement.getLoBits(8).getZExtValue()); - MaskElement = MaskElement.lshr(8); - } - } - DecodePSHUFBMask(RawMask, Mask); - break; + bool AllContants = true; + uint64_t Immediate = 0; + int NonConstIdx = -1; + bool IsSplat = true; + unsigned NumNonConsts = 0; + unsigned NumConsts = 0; + for (unsigned idx = 0, e = Op.getNumOperands(); idx < e; ++idx) { + SDValue In = Op.getOperand(idx); + if (In.getOpcode() == ISD::UNDEF) + continue; + if (!isa(In)) { + AllContants = false; + NonConstIdx = idx; + NumNonConsts++; + } else { + NumConsts++; + if (cast(In)->getZExtValue()) + Immediate |= (1ULL << idx); } + if (In != Op.getOperand(0)) + IsSplat = false; + } - auto *MaskLoad = dyn_cast(MaskNode); - if (!MaskLoad) - return false; - - SDValue Ptr = MaskLoad->getBasePtr(); - if (Ptr->getOpcode() == X86ISD::Wrapper) - Ptr = Ptr->getOperand(0); - - auto *MaskCP = dyn_cast(Ptr); - if (!MaskCP || MaskCP->isMachineConstantPoolEntry()) - return false; + if (AllContants) { + SDValue FullMask = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, + DAG.getConstant(Immediate, MVT::i16)); + return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, FullMask, + DAG.getIntPtrConstant(0)); + } - if (auto *C = dyn_cast(MaskCP->getConstVal())) { - DecodePSHUFBMask(C, Mask); - if (Mask.empty()) - return false; - break; + if (NumNonConsts == 1 && NonConstIdx != 0) { + SDValue DstVec; + if (NumConsts) { + SDValue VecAsImm = DAG.getConstant(Immediate, + MVT::getIntegerVT(VT.getSizeInBits())); + DstVec = DAG.getNode(ISD::BITCAST, dl, VT, VecAsImm); } - - return false; - } - case X86ISD::VPERMI: - ImmN = N->getOperand(N->getNumOperands()-1); - DecodeVPERMMask(cast(ImmN)->getZExtValue(), Mask); - IsUnary = true; - break; - case X86ISD::MOVSS: - case X86ISD::MOVSD: - DecodeScalarMoveMask(VT, /* IsLoad */ false, Mask); - break; - case X86ISD::VPERM2X128: - ImmN = N->getOperand(N->getNumOperands()-1); - DecodeVPERM2X128Mask(VT, cast(ImmN)->getZExtValue(), Mask); - if (Mask.empty()) return false; - break; - case X86ISD::MOVSLDUP: - DecodeMOVSLDUPMask(VT, Mask); - IsUnary = true; - break; - case X86ISD::MOVSHDUP: - DecodeMOVSHDUPMask(VT, Mask); - IsUnary = true; - break; - case X86ISD::MOVDDUP: - DecodeMOVDDUPMask(VT, Mask); - IsUnary = true; - break; - case X86ISD::MOVLHPD: - case X86ISD::MOVLPD: - case X86ISD::MOVLPS: - // Not yet implemented - return false; - default: llvm_unreachable("unknown target shuffle node"); + else + DstVec = DAG.getUNDEF(VT); + return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DstVec, + Op.getOperand(NonConstIdx), + DAG.getIntPtrConstant(NonConstIdx)); } - - // If we have a fake unary shuffle, the shuffle mask is spread across two - // inputs that are actually the same node. Re-map the mask to always point - // into the first input. - if (IsFakeUnary) - for (int &M : Mask) - if (M >= (int)Mask.size()) - M -= Mask.size(); - - return true; + if (!IsSplat && (NonConstIdx != 0)) + llvm_unreachable("Unsupported BUILD_VECTOR operation"); + MVT SelectVT = (VT == MVT::v16i1)? MVT::i16 : MVT::i8; + SDValue Select; + if (IsSplat) + Select = DAG.getNode(ISD::SELECT, dl, SelectVT, Op.getOperand(0), + DAG.getConstant(-1, SelectVT), + DAG.getConstant(0, SelectVT)); + else + Select = DAG.getNode(ISD::SELECT, dl, SelectVT, Op.getOperand(0), + DAG.getConstant((Immediate | 1), SelectVT), + DAG.getConstant(Immediate, SelectVT)); + return DAG.getNode(ISD::BITCAST, dl, VT, Select); } -/// getShuffleScalarElt - Returns the scalar element that will make up the ith -/// element of the result of the vector shuffle. -static SDValue getShuffleScalarElt(SDNode *N, unsigned Index, SelectionDAG &DAG, - unsigned Depth) { - if (Depth == 6) - return SDValue(); // Limit search depth. - - SDValue V = SDValue(N, 0); - EVT VT = V.getValueType(); - unsigned Opcode = V.getOpcode(); +/// \brief Return true if \p N implements a horizontal binop and return the +/// operands for the horizontal binop into V0 and V1. +/// +/// This is a helper function of PerformBUILD_VECTORCombine. +/// This function checks that the build_vector \p N in input implements a +/// horizontal operation. Parameter \p Opcode defines the kind of horizontal +/// operation to match. +/// For example, if \p Opcode is equal to ISD::ADD, then this function +/// checks if \p N implements a horizontal arithmetic add; if instead \p Opcode +/// is equal to ISD::SUB, then this function checks if this is a horizontal +/// arithmetic sub. +/// +/// This function only analyzes elements of \p N whose indices are +/// in range [BaseIdx, LastIdx). +static bool isHorizontalBinOp(const BuildVectorSDNode *N, unsigned Opcode, + SelectionDAG &DAG, + unsigned BaseIdx, unsigned LastIdx, + SDValue &V0, SDValue &V1) { + EVT VT = N->getValueType(0); - // Recurse into ISD::VECTOR_SHUFFLE node to find scalars. - if (const ShuffleVectorSDNode *SV = dyn_cast(N)) { - int Elt = SV->getMaskElt(Index); + assert(BaseIdx * 2 <= LastIdx && "Invalid Indices in input!"); + assert(VT.isVector() && VT.getVectorNumElements() >= LastIdx && + "Invalid Vector in input!"); - if (Elt < 0) - return DAG.getUNDEF(VT.getVectorElementType()); - - unsigned NumElems = VT.getVectorNumElements(); - SDValue NewV = (Elt < (int)NumElems) ? SV->getOperand(0) - : SV->getOperand(1); - return getShuffleScalarElt(NewV.getNode(), Elt % NumElems, DAG, Depth+1); - } - - // Recurse into target specific vector shuffles to find scalars. - if (isTargetShuffle(Opcode)) { - MVT ShufVT = V.getSimpleValueType(); - unsigned NumElems = ShufVT.getVectorNumElements(); - SmallVector ShuffleMask; - bool IsUnary; + bool IsCommutable = (Opcode == ISD::ADD || Opcode == ISD::FADD); + bool CanFold = true; + unsigned ExpectedVExtractIdx = BaseIdx; + unsigned NumElts = LastIdx - BaseIdx; + V0 = DAG.getUNDEF(VT); + V1 = DAG.getUNDEF(VT); - if (!getTargetShuffleMask(N, ShufVT, ShuffleMask, IsUnary)) - return SDValue(); + // Check if N implements a horizontal binop. + for (unsigned i = 0, e = NumElts; i != e && CanFold; ++i) { + SDValue Op = N->getOperand(i + BaseIdx); - int Elt = ShuffleMask[Index]; - if (Elt < 0) - return DAG.getUNDEF(ShufVT.getVectorElementType()); + // Skip UNDEFs. + if (Op->getOpcode() == ISD::UNDEF) { + // Update the expected vector extract index. + if (i * 2 == NumElts) + ExpectedVExtractIdx = BaseIdx; + ExpectedVExtractIdx += 2; + continue; + } - SDValue NewV = (Elt < (int)NumElems) ? N->getOperand(0) - : N->getOperand(1); - return getShuffleScalarElt(NewV.getNode(), Elt % NumElems, DAG, - Depth+1); - } + CanFold = Op->getOpcode() == Opcode && Op->hasOneUse(); - // Actual nodes that may contain scalar elements - if (Opcode == ISD::BITCAST) { - V = V.getOperand(0); - EVT SrcVT = V.getValueType(); - unsigned NumElems = VT.getVectorNumElements(); + if (!CanFold) + break; - if (!SrcVT.isVector() || SrcVT.getVectorNumElements() != NumElems) - return SDValue(); - } + SDValue Op0 = Op.getOperand(0); + SDValue Op1 = Op.getOperand(1); - if (V.getOpcode() == ISD::SCALAR_TO_VECTOR) - return (Index == 0) ? V.getOperand(0) - : DAG.getUNDEF(VT.getVectorElementType()); + // Try to match the following pattern: + // (BINOP (extract_vector_elt A, I), (extract_vector_elt A, I+1)) + CanFold = (Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && + Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT && + Op0.getOperand(0) == Op1.getOperand(0) && + isa(Op0.getOperand(1)) && + isa(Op1.getOperand(1))); + if (!CanFold) + break; - if (V.getOpcode() == ISD::BUILD_VECTOR) - return V.getOperand(Index); + unsigned I0 = cast(Op0.getOperand(1))->getZExtValue(); + unsigned I1 = cast(Op1.getOperand(1))->getZExtValue(); - return SDValue(); -} + if (i * 2 < NumElts) { + if (V0.getOpcode() == ISD::UNDEF) + V0 = Op0.getOperand(0); + } else { + if (V1.getOpcode() == ISD::UNDEF) + V1 = Op0.getOperand(0); + if (i * 2 == NumElts) + ExpectedVExtractIdx = BaseIdx; + } -/// getNumOfConsecutiveZeros - Return the number of elements of a vector -/// shuffle operation which come from a consecutively from a zero. The -/// search can start in two different directions, from left or right. -/// We count undefs as zeros until PreferredNum is reached. -static unsigned getNumOfConsecutiveZeros(ShuffleVectorSDNode *SVOp, - unsigned NumElems, bool ZerosFromLeft, - SelectionDAG &DAG, - unsigned PreferredNum = -1U) { - unsigned NumZeros = 0; - for (unsigned i = 0; i != NumElems; ++i) { - unsigned Index = ZerosFromLeft ? i : NumElems - i - 1; - SDValue Elt = getShuffleScalarElt(SVOp, Index, DAG, 0); - if (!Elt.getNode()) - break; + SDValue Expected = (i * 2 < NumElts) ? V0 : V1; + if (I0 == ExpectedVExtractIdx) + CanFold = I1 == I0 + 1 && Op0.getOperand(0) == Expected; + else if (IsCommutable && I1 == ExpectedVExtractIdx) { + // Try to match the following dag sequence: + // (BINOP (extract_vector_elt A, I+1), (extract_vector_elt A, I)) + CanFold = I0 == I1 + 1 && Op1.getOperand(0) == Expected; + } else + CanFold = false; - if (X86::isZeroNode(Elt)) - ++NumZeros; - else if (Elt.getOpcode() == ISD::UNDEF) // Undef as zero up to PreferredNum. - NumZeros = std::min(NumZeros + 1, PreferredNum); - else - break; + ExpectedVExtractIdx += 2; } - return NumZeros; + return CanFold; } -/// isShuffleMaskConsecutive - Check if the shuffle mask indicies [MaskI, MaskE) -/// correspond consecutively to elements from one of the vector operands, -/// starting from its index OpIdx. Also tell OpNum which source vector operand. -static -bool isShuffleMaskConsecutive(ShuffleVectorSDNode *SVOp, - unsigned MaskI, unsigned MaskE, unsigned OpIdx, - unsigned NumElems, unsigned &OpNum) { - bool SeenV1 = false; - bool SeenV2 = false; - - for (unsigned i = MaskI; i != MaskE; ++i, ++OpIdx) { - int Idx = SVOp->getMaskElt(i); - // Ignore undef indicies - if (Idx < 0) - continue; +/// \brief Emit a sequence of two 128-bit horizontal add/sub followed by +/// a concat_vector. +/// +/// This is a helper function of PerformBUILD_VECTORCombine. +/// This function expects two 256-bit vectors called V0 and V1. +/// At first, each vector is split into two separate 128-bit vectors. +/// Then, the resulting 128-bit vectors are used to implement two +/// horizontal binary operations. +/// +/// The kind of horizontal binary operation is defined by \p X86Opcode. +/// +/// \p Mode specifies how the 128-bit parts of V0 and V1 are passed in input to +/// the two new horizontal binop. +/// When Mode is set, the first horizontal binop dag node would take as input +/// the lower 128-bit of V0 and the upper 128-bit of V0. The second +/// horizontal binop dag node would take as input the lower 128-bit of V1 +/// and the upper 128-bit of V1. +/// Example: +/// HADD V0_LO, V0_HI +/// HADD V1_LO, V1_HI +/// +/// Otherwise, the first horizontal binop dag node takes as input the lower +/// 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop +/// dag node takes the the upper 128-bit of V0 and the upper 128-bit of V1. +/// Example: +/// HADD V0_LO, V1_LO +/// HADD V0_HI, V1_HI +/// +/// If \p isUndefLO is set, then the algorithm propagates UNDEF to the lower +/// 128-bits of the result. If \p isUndefHI is set, then UNDEF is propagated to +/// the upper 128-bits of the result. +static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1, + SDLoc DL, SelectionDAG &DAG, + unsigned X86Opcode, bool Mode, + bool isUndefLO, bool isUndefHI) { + EVT VT = V0.getValueType(); + assert(VT.is256BitVector() && VT == V1.getValueType() && + "Invalid nodes in input!"); - if (Idx < (int)NumElems) - SeenV1 = true; - else - SeenV2 = true; + unsigned NumElts = VT.getVectorNumElements(); + SDValue V0_LO = Extract128BitVector(V0, 0, DAG, DL); + SDValue V0_HI = Extract128BitVector(V0, NumElts/2, DAG, DL); + SDValue V1_LO = Extract128BitVector(V1, 0, DAG, DL); + SDValue V1_HI = Extract128BitVector(V1, NumElts/2, DAG, DL); + EVT NewVT = V0_LO.getValueType(); - // Only accept consecutive elements from the same vector - if ((Idx % NumElems != OpIdx) || (SeenV1 && SeenV2)) - return false; + SDValue LO = DAG.getUNDEF(NewVT); + SDValue HI = DAG.getUNDEF(NewVT); + + if (Mode) { + // Don't emit a horizontal binop if the result is expected to be UNDEF. + if (!isUndefLO && V0->getOpcode() != ISD::UNDEF) + LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V0_HI); + if (!isUndefHI && V1->getOpcode() != ISD::UNDEF) + HI = DAG.getNode(X86Opcode, DL, NewVT, V1_LO, V1_HI); + } else { + // Don't emit a horizontal binop if the result is expected to be UNDEF. + if (!isUndefLO && (V0_LO->getOpcode() != ISD::UNDEF || + V1_LO->getOpcode() != ISD::UNDEF)) + LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V1_LO); + + if (!isUndefHI && (V0_HI->getOpcode() != ISD::UNDEF || + V1_HI->getOpcode() != ISD::UNDEF)) + HI = DAG.getNode(X86Opcode, DL, NewVT, V0_HI, V1_HI); } - OpNum = SeenV1 ? 0 : 1; - return true; + return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI); } -/// isVectorShiftRight - Returns true if the shuffle can be implemented as a -/// logical left shift of a vector. -static bool isVectorShiftRight(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG, - bool &isLeft, SDValue &ShVal, unsigned &ShAmt) { - unsigned NumElems = - SVOp->getSimpleValueType(0).getVectorNumElements(); - unsigned NumZeros = getNumOfConsecutiveZeros( - SVOp, NumElems, false /* check zeros from right */, DAG, - SVOp->getMaskElt(0)); - unsigned OpSrc; +/// \brief Try to fold a build_vector that performs an 'addsub' into the +/// sequence of 'vadd + vsub + blendi'. +static SDValue matchAddSub(const BuildVectorSDNode *BV, SelectionDAG &DAG, + const X86Subtarget *Subtarget) { + SDLoc DL(BV); + EVT VT = BV->getValueType(0); + unsigned NumElts = VT.getVectorNumElements(); + SDValue InVec0 = DAG.getUNDEF(VT); + SDValue InVec1 = DAG.getUNDEF(VT); - if (!NumZeros) - return false; + assert((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v4f32 || + VT == MVT::v2f64) && "build_vector with an invalid type found!"); - // Considering the elements in the mask that are not consecutive zeros, - // check if they consecutively come from only one of the source vectors. - // - // V1 = {X, A, B, C} 0 - // \ \ \ / - // vector_shuffle V1, V2 <1, 2, 3, X> - // - if (!isShuffleMaskConsecutive(SVOp, - 0, // Mask Start Index - NumElems-NumZeros, // Mask End Index(exclusive) - NumZeros, // Where to start looking in the src vector - NumElems, // Number of elements in vector - OpSrc)) // Which source operand ? - return false; + // Odd-numbered elements in the input build vector are obtained from + // adding two integer/float elements. + // Even-numbered elements in the input build vector are obtained from + // subtracting two integer/float elements. + unsigned ExpectedOpcode = ISD::FSUB; + unsigned NextExpectedOpcode = ISD::FADD; + bool AddFound = false; + bool SubFound = false; - isLeft = false; - ShAmt = NumZeros; - ShVal = SVOp->getOperand(OpSrc); - return true; -} + for (unsigned i = 0, e = NumElts; i != e; ++i) { + SDValue Op = BV->getOperand(i); -/// isVectorShiftLeft - Returns true if the shuffle can be implemented as a -/// logical left shift of a vector. -static bool isVectorShiftLeft(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG, - bool &isLeft, SDValue &ShVal, unsigned &ShAmt) { - unsigned NumElems = - SVOp->getSimpleValueType(0).getVectorNumElements(); - unsigned NumZeros = getNumOfConsecutiveZeros( - SVOp, NumElems, true /* check zeros from left */, DAG, - NumElems - SVOp->getMaskElt(NumElems - 1) - 1); - unsigned OpSrc; + // Skip 'undef' values. + unsigned Opcode = Op.getOpcode(); + if (Opcode == ISD::UNDEF) { + std::swap(ExpectedOpcode, NextExpectedOpcode); + continue; + } - if (!NumZeros) - return false; + // Early exit if we found an unexpected opcode. + if (Opcode != ExpectedOpcode) + return SDValue(); - // Considering the elements in the mask that are not consecutive zeros, - // check if they consecutively come from only one of the source vectors. - // - // 0 { A, B, X, X } = V2 - // / \ / / - // vector_shuffle V1, V2 - // - if (!isShuffleMaskConsecutive(SVOp, - NumZeros, // Mask Start Index - NumElems, // Mask End Index(exclusive) - 0, // Where to start looking in the src vector - NumElems, // Number of elements in vector - OpSrc)) // Which source operand ? - return false; + SDValue Op0 = Op.getOperand(0); + SDValue Op1 = Op.getOperand(1); - isLeft = true; - ShAmt = NumZeros; - ShVal = SVOp->getOperand(OpSrc); - return true; -} + // Try to match the following pattern: + // (BINOP (extract_vector_elt A, i), (extract_vector_elt B, i)) + // Early exit if we cannot match that sequence. + if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT || + Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT || + !isa(Op0.getOperand(1)) || + !isa(Op1.getOperand(1)) || + Op0.getOperand(1) != Op1.getOperand(1)) + return SDValue(); -/// isVectorShift - Returns true if the shuffle can be implemented as a -/// logical left or right shift of a vector. -static bool isVectorShift(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG, - bool &isLeft, SDValue &ShVal, unsigned &ShAmt) { - // Although the logic below support any bitwidth size, there are no - // shift instructions which handle more than 128-bit vectors. - if (!SVOp->getSimpleValueType(0).is128BitVector()) - return false; + unsigned I0 = cast(Op0.getOperand(1))->getZExtValue(); + if (I0 != i) + return SDValue(); - if (isVectorShiftLeft(SVOp, DAG, isLeft, ShVal, ShAmt) || - isVectorShiftRight(SVOp, DAG, isLeft, ShVal, ShAmt)) - return true; + // We found a valid add/sub node. Update the information accordingly. + if (i & 1) + AddFound = true; + else + SubFound = true; - return false; -} + // Update InVec0 and InVec1. + if (InVec0.getOpcode() == ISD::UNDEF) + InVec0 = Op0.getOperand(0); + if (InVec1.getOpcode() == ISD::UNDEF) + InVec1 = Op1.getOperand(0); -/// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8. -/// -static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros, - unsigned NumNonZero, unsigned NumZero, - SelectionDAG &DAG, - const X86Subtarget* Subtarget, - const TargetLowering &TLI) { - if (NumNonZero > 8) - return SDValue(); + // Make sure that operands in input to each add/sub node always + // come from a same pair of vectors. + if (InVec0 != Op0.getOperand(0)) { + if (ExpectedOpcode == ISD::FSUB) + return SDValue(); - SDLoc dl(Op); - SDValue V; - bool First = true; - for (unsigned i = 0; i < 16; ++i) { - bool ThisIsNonZero = (NonZeros & (1 << i)) != 0; - if (ThisIsNonZero && First) { - if (NumZero) - V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl); - else - V = DAG.getUNDEF(MVT::v8i16); - First = false; + // FADD is commutable. Try to commute the operands + // and then test again. + std::swap(Op0, Op1); + if (InVec0 != Op0.getOperand(0)) + return SDValue(); } - if ((i & 1) != 0) { - SDValue ThisElt, LastElt; - bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0; - if (LastIsNonZero) { - LastElt = DAG.getNode(ISD::ZERO_EXTEND, dl, - MVT::i16, Op.getOperand(i-1)); - } - if (ThisIsNonZero) { - ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i)); - ThisElt = DAG.getNode(ISD::SHL, dl, MVT::i16, - ThisElt, DAG.getConstant(8, MVT::i8)); - if (LastIsNonZero) - ThisElt = DAG.getNode(ISD::OR, dl, MVT::i16, ThisElt, LastElt); - } else - ThisElt = LastElt; + if (InVec1 != Op1.getOperand(0)) + return SDValue(); - if (ThisElt.getNode()) - V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, ThisElt, - DAG.getIntPtrConstant(i/2)); - } + // Update the pair of expected opcodes. + std::swap(ExpectedOpcode, NextExpectedOpcode); } - return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V); -} - -/// LowerBuildVectorv8i16 - Custom lower build_vector of v8i16. -/// -static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros, - unsigned NumNonZero, unsigned NumZero, - SelectionDAG &DAG, - const X86Subtarget* Subtarget, - const TargetLowering &TLI) { - if (NumNonZero > 4) - return SDValue(); - - SDLoc dl(Op); - SDValue V; - bool First = true; - for (unsigned i = 0; i < 8; ++i) { - bool isNonZero = (NonZeros & (1 << i)) != 0; - if (isNonZero) { - if (First) { - if (NumZero) - V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl); - else - V = DAG.getUNDEF(MVT::v8i16); - First = false; - } - V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, - MVT::v8i16, V, Op.getOperand(i), - DAG.getIntPtrConstant(i)); - } - } + // Don't try to fold this build_vector into an ADDSUB if the inputs are undef. + if (AddFound && SubFound && InVec0.getOpcode() != ISD::UNDEF && + InVec1.getOpcode() != ISD::UNDEF) + return DAG.getNode(X86ISD::ADDSUB, DL, VT, InVec0, InVec1); - return V; + return SDValue(); } -/// LowerBuildVectorv4x32 - Custom lower build_vector of v4i32 or v4f32. -static SDValue LowerBuildVectorv4x32(SDValue Op, SelectionDAG &DAG, - const X86Subtarget *Subtarget, - const TargetLowering &TLI) { - // Find all zeroable elements. - std::bitset<4> Zeroable; - for (int i=0; i < 4; ++i) { - SDValue Elt = Op->getOperand(i); - Zeroable[i] = (Elt.getOpcode() == ISD::UNDEF || X86::isZeroNode(Elt)); - } - assert(Zeroable.size() - Zeroable.count() > 1 && - "We expect at least two non-zero elements!"); +static SDValue PerformBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG, + const X86Subtarget *Subtarget) { + SDLoc DL(N); + EVT VT = N->getValueType(0); + unsigned NumElts = VT.getVectorNumElements(); + BuildVectorSDNode *BV = cast(N); + SDValue InVec0, InVec1; - // We only know how to deal with build_vector nodes where elements are either - // zeroable or extract_vector_elt with constant index. - SDValue FirstNonZero; - unsigned FirstNonZeroIdx; - for (unsigned i=0; i < 4; ++i) { - if (Zeroable[i]) - continue; - SDValue Elt = Op->getOperand(i); - if (Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT || - !isa(Elt.getOperand(1))) - return SDValue(); - // Make sure that this node is extracting from a 128-bit vector. - MVT VT = Elt.getOperand(0).getSimpleValueType(); - if (!VT.is128BitVector()) - return SDValue(); - if (!FirstNonZero.getNode()) { - FirstNonZero = Elt; - FirstNonZeroIdx = i; - } + // Try to match an ADDSUB. + if ((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || + (Subtarget->hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) { + SDValue Value = matchAddSub(BV, DAG, Subtarget); + if (Value.getNode()) + return Value; } - assert(FirstNonZero.getNode() && "Unexpected build vector of all zeros!"); - SDValue V1 = FirstNonZero.getOperand(0); - MVT VT = V1.getSimpleValueType(); - - // See if this build_vector can be lowered as a blend with zero. - SDValue Elt; - unsigned EltMaskIdx, EltIdx; - int Mask[4]; - for (EltIdx = 0; EltIdx < 4; ++EltIdx) { - if (Zeroable[EltIdx]) { - // The zero vector will be on the right hand side. - Mask[EltIdx] = EltIdx+4; - continue; - } + // Try to match horizontal ADD/SUB. + unsigned NumUndefsLO = 0; + unsigned NumUndefsHI = 0; + unsigned Half = NumElts/2; - Elt = Op->getOperand(EltIdx); - // By construction, Elt is a EXTRACT_VECTOR_ELT with constant index. - EltMaskIdx = cast(Elt.getOperand(1))->getZExtValue(); - if (Elt.getOperand(0) != V1 || EltMaskIdx != EltIdx) - break; - Mask[EltIdx] = EltIdx; - } + // Count the number of UNDEF operands in the build_vector in input. + for (unsigned i = 0, e = Half; i != e; ++i) + if (BV->getOperand(i)->getOpcode() == ISD::UNDEF) + NumUndefsLO++; - if (EltIdx == 4) { - // Let the shuffle legalizer deal with blend operations. - SDValue VZero = getZeroVector(VT, Subtarget, DAG, SDLoc(Op)); - if (V1.getSimpleValueType() != VT) - V1 = DAG.getNode(ISD::BITCAST, SDLoc(V1), VT, V1); - return DAG.getVectorShuffle(VT, SDLoc(V1), V1, VZero, &Mask[0]); - } + for (unsigned i = Half, e = NumElts; i != e; ++i) + if (BV->getOperand(i)->getOpcode() == ISD::UNDEF) + NumUndefsHI++; - // See if we can lower this build_vector to a INSERTPS. - if (!Subtarget->hasSSE41()) + // Early exit if this is either a build_vector of all UNDEFs or all the + // operands but one are UNDEF. + if (NumUndefsLO + NumUndefsHI + 1 >= NumElts) return SDValue(); - SDValue V2 = Elt.getOperand(0); - if (Elt == FirstNonZero && EltIdx == FirstNonZeroIdx) - V1 = SDValue(); + if ((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget->hasSSE3()) { + // Try to match an SSE3 float HADD/HSUB. + if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, NumElts, InVec0, InVec1)) + return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1); - bool CanFold = true; - for (unsigned i = EltIdx + 1; i < 4 && CanFold; ++i) { - if (Zeroable[i]) - continue; + if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, NumElts, InVec0, InVec1)) + return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1); + } else if ((VT == MVT::v4i32 || VT == MVT::v8i16) && Subtarget->hasSSSE3()) { + // Try to match an SSSE3 integer HADD/HSUB. + if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1)) + return DAG.getNode(X86ISD::HADD, DL, VT, InVec0, InVec1); - SDValue Current = Op->getOperand(i); - SDValue SrcVector = Current->getOperand(0); - if (!V1.getNode()) - V1 = SrcVector; - CanFold = SrcVector == V1 && - cast(Current.getOperand(1))->getZExtValue() == i; + if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, NumElts, InVec0, InVec1)) + return DAG.getNode(X86ISD::HSUB, DL, VT, InVec0, InVec1); } - if (!CanFold) + if (!Subtarget->hasAVX()) return SDValue(); - assert(V1.getNode() && "Expected at least two non-zero elements!"); - if (V1.getSimpleValueType() != MVT::v4f32) - V1 = DAG.getNode(ISD::BITCAST, SDLoc(V1), MVT::v4f32, V1); - if (V2.getSimpleValueType() != MVT::v4f32) - V2 = DAG.getNode(ISD::BITCAST, SDLoc(V2), MVT::v4f32, V2); + if ((VT == MVT::v8f32 || VT == MVT::v4f64)) { + // Try to match an AVX horizontal add/sub of packed single/double + // precision floating point values from 256-bit vectors. + SDValue InVec2, InVec3; + if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, Half, InVec0, InVec1) && + isHorizontalBinOp(BV, ISD::FADD, DAG, Half, NumElts, InVec2, InVec3) && + ((InVec0.getOpcode() == ISD::UNDEF || + InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) && + ((InVec1.getOpcode() == ISD::UNDEF || + InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3)) + return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1); - // Ok, we can emit an INSERTPS instruction. - unsigned ZMask = Zeroable.to_ulong(); + if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, Half, InVec0, InVec1) && + isHorizontalBinOp(BV, ISD::FSUB, DAG, Half, NumElts, InVec2, InVec3) && + ((InVec0.getOpcode() == ISD::UNDEF || + InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) && + ((InVec1.getOpcode() == ISD::UNDEF || + InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3)) + return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1); + } else if (VT == MVT::v8i32 || VT == MVT::v16i16) { + // Try to match an AVX2 horizontal add/sub of signed integers. + SDValue InVec2, InVec3; + unsigned X86Opcode; + bool CanFold = true; - unsigned InsertPSMask = EltMaskIdx << 6 | EltIdx << 4 | ZMask; - assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!"); - SDValue Result = DAG.getNode(X86ISD::INSERTPS, SDLoc(Op), MVT::v4f32, V1, V2, - DAG.getIntPtrConstant(InsertPSMask)); - return DAG.getNode(ISD::BITCAST, SDLoc(Op), VT, Result); -} + if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, Half, InVec0, InVec1) && + isHorizontalBinOp(BV, ISD::ADD, DAG, Half, NumElts, InVec2, InVec3) && + ((InVec0.getOpcode() == ISD::UNDEF || + InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) && + ((InVec1.getOpcode() == ISD::UNDEF || + InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3)) + X86Opcode = X86ISD::HADD; + else if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, Half, InVec0, InVec1) && + isHorizontalBinOp(BV, ISD::SUB, DAG, Half, NumElts, InVec2, InVec3) && + ((InVec0.getOpcode() == ISD::UNDEF || + InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) && + ((InVec1.getOpcode() == ISD::UNDEF || + InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3)) + X86Opcode = X86ISD::HSUB; + else + CanFold = false; -/// Return a vector logical shift node. -static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, - unsigned NumBits, SelectionDAG &DAG, - const TargetLowering &TLI, SDLoc dl) { - assert(VT.is128BitVector() && "Unknown type for VShift"); - MVT ShVT = MVT::v2i64; - unsigned Opc = isLeft ? X86ISD::VSHLDQ : X86ISD::VSRLDQ; - SrcOp = DAG.getNode(ISD::BITCAST, dl, ShVT, SrcOp); - MVT ScalarShiftTy = TLI.getScalarShiftAmountTy(SrcOp.getValueType()); - assert(NumBits % 8 == 0 && "Only support byte sized shifts"); - SDValue ShiftVal = DAG.getConstant(NumBits/8, ScalarShiftTy); - return DAG.getNode(ISD::BITCAST, dl, VT, - DAG.getNode(Opc, dl, ShVT, SrcOp, ShiftVal)); -} + if (CanFold) { + // Fold this build_vector into a single horizontal add/sub. + // Do this only if the target has AVX2. + if (Subtarget->hasAVX2()) + return DAG.getNode(X86Opcode, DL, VT, InVec0, InVec1); -static SDValue -LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, SDLoc dl, SelectionDAG &DAG) { + // Do not try to expand this build_vector into a pair of horizontal + // add/sub if we can emit a pair of scalar add/sub. + if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half) + return SDValue(); - // Check if the scalar load can be widened into a vector load. And if - // the address is "base + cst" see if the cst can be "absorbed" into - // the shuffle mask. - if (LoadSDNode *LD = dyn_cast(SrcOp)) { - SDValue Ptr = LD->getBasePtr(); - if (!ISD::isNormalLoad(LD) || LD->isVolatile()) - return SDValue(); - EVT PVT = LD->getValueType(0); - if (PVT != MVT::i32 && PVT != MVT::f32) + // Convert this build_vector into a pair of horizontal binop followed by + // a concat vector. + bool isUndefLO = NumUndefsLO == Half; + bool isUndefHI = NumUndefsHI == Half; + return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, false, + isUndefLO, isUndefHI); + } + } + + if ((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 || + VT == MVT::v16i16) && Subtarget->hasAVX()) { + unsigned X86Opcode; + if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1)) + X86Opcode = X86ISD::HADD; + else if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, NumElts, InVec0, InVec1)) + X86Opcode = X86ISD::HSUB; + else if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, NumElts, InVec0, InVec1)) + X86Opcode = X86ISD::FHADD; + else if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, NumElts, InVec0, InVec1)) + X86Opcode = X86ISD::FHSUB; + else return SDValue(); - int FI = -1; - int64_t Offset = 0; - if (FrameIndexSDNode *FINode = dyn_cast(Ptr)) { - FI = FINode->getIndex(); - Offset = 0; - } else if (DAG.isBaseWithConstantOffset(Ptr) && - isa(Ptr.getOperand(0))) { - FI = cast(Ptr.getOperand(0))->getIndex(); - Offset = Ptr.getConstantOperandVal(1); - Ptr = Ptr.getOperand(0); - } else { + // Don't try to expand this build_vector into a pair of horizontal add/sub + // if we can simply emit a pair of scalar add/sub. + if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half) return SDValue(); - } - // FIXME: 256-bit vector instructions don't require a strict alignment, - // improve this code to support it better. - unsigned RequiredAlign = VT.getSizeInBits()/8; - SDValue Chain = LD->getChain(); - // Make sure the stack object alignment is at least 16 or 32. - MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); - if (DAG.InferPtrAlignment(Ptr) < RequiredAlign) { - if (MFI->isFixedObjectIndex(FI)) { - // Can't change the alignment. FIXME: It's possible to compute - // the exact stack offset and reference FI + adjust offset instead. - // If someone *really* cares about this. That's the way to implement it. - return SDValue(); - } else { - MFI->setObjectAlignment(FI, RequiredAlign); - } - } + // Convert this build_vector into two horizontal add/sub followed by + // a concat vector. + bool isUndefLO = NumUndefsLO == Half; + bool isUndefHI = NumUndefsHI == Half; + return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, true, + isUndefLO, isUndefHI); + } - // (Offset % 16 or 32) must be multiple of 4. Then address is then - // Ptr + (Offset & ~15). - if (Offset < 0) - return SDValue(); - if ((Offset % RequiredAlign) & 3) - return SDValue(); - int64_t StartOffset = Offset & ~(RequiredAlign-1); - if (StartOffset) - Ptr = DAG.getNode(ISD::ADD, SDLoc(Ptr), Ptr.getValueType(), - Ptr,DAG.getConstant(StartOffset, Ptr.getValueType())); + return SDValue(); +} - int EltNo = (Offset - StartOffset) >> 2; - unsigned NumElems = VT.getVectorNumElements(); +SDValue +X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { + SDLoc dl(Op); - EVT NVT = EVT::getVectorVT(*DAG.getContext(), PVT, NumElems); - SDValue V1 = DAG.getLoad(NVT, dl, Chain, Ptr, - LD->getPointerInfo().getWithOffset(StartOffset), - false, false, false, 0); + MVT VT = Op.getSimpleValueType(); + MVT ExtVT = VT.getVectorElementType(); + unsigned NumElems = Op.getNumOperands(); - SmallVector Mask(NumElems, EltNo); + // Generate vectors for predicate vectors. + if (VT.getScalarType() == MVT::i1 && Subtarget->hasAVX512()) + return LowerBUILD_VECTORvXi1(Op, DAG); - return DAG.getVectorShuffle(NVT, dl, V1, DAG.getUNDEF(NVT), &Mask[0]); + // Vectors containing all zeros can be matched by pxor and xorps later + if (ISD::isBuildVectorAllZeros(Op.getNode())) { + // Canonicalize this to <4 x i32> to 1) ensure the zero vectors are CSE'd + // and 2) ensure that i64 scalars are eliminated on x86-32 hosts. + if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) + return Op; + + return getZeroVector(VT, Subtarget, DAG, dl); } - return SDValue(); -} + // Vectors containing all ones can be matched by pcmpeqd on 128-bit width + // vectors or broken into v4i32 operations on 256-bit vectors. AVX2 can use + // vpcmpeqd on 256-bit vectors. + if (Subtarget->hasSSE2() && ISD::isBuildVectorAllOnes(Op.getNode())) { + if (VT == MVT::v4i32 || (VT == MVT::v8i32 && Subtarget->hasInt256())) + return Op; -/// Given the initializing elements 'Elts' of a vector of type 'VT', see if the -/// elements can be replaced by a single large load which has the same value as -/// a build_vector or insert_subvector whose loaded operands are 'Elts'. -/// -/// Example: -> zextload a -/// -/// FIXME: we'd also like to handle the case where the last elements are zero -/// rather than undef via VZEXT_LOAD, but we do not detect that case today. -/// There's even a handy isZeroNode for that purpose. -static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef Elts, - SDLoc &DL, SelectionDAG &DAG, - bool isAfterLegalize) { - unsigned NumElems = Elts.size(); + if (!VT.is512BitVector()) + return getOnesVector(VT, Subtarget->hasInt256(), DAG, dl); + } - LoadSDNode *LDBase = nullptr; - unsigned LastLoadedElt = -1U; + SDValue Broadcast = LowerVectorBroadcast(Op, Subtarget, DAG); + if (Broadcast.getNode()) + return Broadcast; - // For each element in the initializer, see if we've found a load or an undef. - // If we don't find an initial load element, or later load elements are - // non-consecutive, bail out. + unsigned EVTBits = ExtVT.getSizeInBits(); + + unsigned NumZero = 0; + unsigned NumNonZero = 0; + unsigned NonZeros = 0; + bool IsAllConstants = true; + SmallSet Values; for (unsigned i = 0; i < NumElems; ++i) { - SDValue Elt = Elts[i]; - // Look through a bitcast. - if (Elt.getNode() && Elt.getOpcode() == ISD::BITCAST) - Elt = Elt.getOperand(0); - if (!Elt.getNode() || - (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.getNode()))) - return SDValue(); - if (!LDBase) { - if (Elt.getNode()->getOpcode() == ISD::UNDEF) - return SDValue(); - LDBase = cast(Elt.getNode()); - LastLoadedElt = i; - continue; - } + SDValue Elt = Op.getOperand(i); if (Elt.getOpcode() == ISD::UNDEF) continue; - - LoadSDNode *LD = cast(Elt); - EVT LdVT = Elt.getValueType(); - // Each loaded element must be the correct fractional portion of the - // requested vector load. - if (LdVT.getSizeInBits() != VT.getSizeInBits() / NumElems) - return SDValue(); - if (!DAG.isConsecutiveLoad(LD, LDBase, LdVT.getSizeInBits() / 8, i)) - return SDValue(); - LastLoadedElt = i; + Values.insert(Elt); + if (Elt.getOpcode() != ISD::Constant && + Elt.getOpcode() != ISD::ConstantFP) + IsAllConstants = false; + if (X86::isZeroNode(Elt)) + NumZero++; + else { + NonZeros |= (1 << i); + NumNonZero++; + } } - // If we have found an entire vector of loads and undefs, then return a large - // load of the entire vector width starting at the base pointer. If we found - // consecutive loads for the low half, generate a vzext_load node. - if (LastLoadedElt == NumElems - 1) { - assert(LDBase && "Did not find base load for merging consecutive loads"); - EVT EltVT = LDBase->getValueType(0); - // Ensure that the input vector size for the merged loads matches the - // cumulative size of the input elements. - if (VT.getSizeInBits() != EltVT.getSizeInBits() * NumElems) - return SDValue(); - - if (isAfterLegalize && - !DAG.getTargetLoweringInfo().isOperationLegal(ISD::LOAD, VT)) - return SDValue(); + // All undef vector. Return an UNDEF. All zero vectors were handled above. + if (NumNonZero == 0) + return DAG.getUNDEF(VT); - SDValue NewLd = SDValue(); + // Special case for single non-zero, non-undef, element. + if (NumNonZero == 1) { + unsigned Idx = countTrailingZeros(NonZeros); + SDValue Item = Op.getOperand(Idx); - NewLd = DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(), - LDBase->getPointerInfo(), LDBase->isVolatile(), - LDBase->isNonTemporal(), LDBase->isInvariant(), - LDBase->getAlignment()); + // If this is an insertion of an i64 value on x86-32, and if the top bits of + // the value are obviously zero, truncate the value to i32 and do the + // insertion that way. Only do this if the value is non-constant or if the + // value is a constant being inserted into element 0. It is cheaper to do + // a constant pool load than it is to do a movd + shuffle. + if (ExtVT == MVT::i64 && !Subtarget->is64Bit() && + (!IsAllConstants || Idx == 0)) { + if (DAG.MaskedValueIsZero(Item, APInt::getBitsSet(64, 32, 64))) { + // Handle SSE only. + assert(VT == MVT::v2i64 && "Expected an SSE value type!"); + EVT VecVT = MVT::v4i32; - if (LDBase->hasAnyUseOfValue(1)) { - SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, - SDValue(LDBase, 1), - SDValue(NewLd.getNode(), 1)); - DAG.ReplaceAllUsesOfValueWith(SDValue(LDBase, 1), NewChain); - DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(LDBase, 1), - SDValue(NewLd.getNode(), 1)); + // Truncate the value (which may itself be a constant) to i32, and + // convert it to a vector with movd (S2V+shuffle to zero extend). + Item = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Item); + Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Item); + return DAG.getNode( + ISD::BITCAST, dl, VT, + getShuffleVectorZeroOrUndef(Item, Idx * 2, true, Subtarget, DAG)); + } } - return NewLd; - } + // If we have a constant or non-constant insertion into the low element of + // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into + // the rest of the elements. This will be matched as movd/movq/movss/movsd + // depending on what the source datatype is. + if (Idx == 0) { + if (NumZero == 0) + return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); - //TODO: The code below fires only for for loading the low v2i32 / v2f32 - //of a v4i32 / v4f32. It's probably worth generalizing. - EVT EltVT = VT.getVectorElementType(); - if (NumElems == 4 && LastLoadedElt == 1 && (EltVT.getSizeInBits() == 32) && - DAG.getTargetLoweringInfo().isTypeLegal(MVT::v2i64)) { - SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other); - SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() }; - SDValue ResNode = - DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, MVT::i64, - LDBase->getPointerInfo(), - LDBase->getAlignment(), - false/*isVolatile*/, true/*ReadMem*/, - false/*WriteMem*/); + if (ExtVT == MVT::i32 || ExtVT == MVT::f32 || ExtVT == MVT::f64 || + (ExtVT == MVT::i64 && Subtarget->is64Bit())) { + if (VT.is256BitVector() || VT.is512BitVector()) { + SDValue ZeroVec = getZeroVector(VT, Subtarget, DAG, dl); + return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, ZeroVec, + Item, DAG.getIntPtrConstant(0)); + } + assert(VT.is128BitVector() && "Expected an SSE value type!"); + Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); + // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector. + return getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG); + } - // Make sure the newly-created LOAD is in the same position as LDBase in - // terms of dependency. We create a TokenFactor for LDBase and ResNode, and - // update uses of LDBase's output chain to use the TokenFactor. - if (LDBase->hasAnyUseOfValue(1)) { - SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, - SDValue(LDBase, 1), SDValue(ResNode.getNode(), 1)); - DAG.ReplaceAllUsesOfValueWith(SDValue(LDBase, 1), NewChain); - DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(LDBase, 1), - SDValue(ResNode.getNode(), 1)); + if (ExtVT == MVT::i16 || ExtVT == MVT::i8) { + Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item); + Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Item); + if (VT.is256BitVector()) { + SDValue ZeroVec = getZeroVector(MVT::v8i32, Subtarget, DAG, dl); + Item = Insert128BitVector(ZeroVec, Item, 0, DAG, dl); + } else { + assert(VT.is128BitVector() && "Expected an SSE value type!"); + Item = getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG); + } + return DAG.getNode(ISD::BITCAST, dl, VT, Item); + } } - return DAG.getNode(ISD::BITCAST, DL, VT, ResNode); + // Is it a vector logical left shift? + if (NumElems == 2 && Idx == 1 && + X86::isZeroNode(Op.getOperand(0)) && + !X86::isZeroNode(Op.getOperand(1))) { + unsigned NumBits = VT.getSizeInBits(); + return getVShift(true, VT, + DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, + VT, Op.getOperand(1)), + NumBits/2, DAG, *this, dl); + } + + if (IsAllConstants) // Otherwise, it's better to do a constpool load. + return SDValue(); + + // Otherwise, if this is a vector with i32 or f32 elements, and the element + // is a non-constant being inserted into an element other than the low one, + // we can't use a constant pool load. Instead, use SCALAR_TO_VECTOR (aka + // movd/movss) to move this into the low element, then shuffle it into + // place. + if (EVTBits == 32) { + Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); + return getShuffleVectorZeroOrUndef(Item, Idx, NumZero > 0, Subtarget, DAG); + } } - return SDValue(); -} -/// LowerVectorBroadcast - Attempt to use the vbroadcast instruction -/// to generate a splat value for the following cases: -/// 1. A splat BUILD_VECTOR which uses a single scalar load, or a constant. -/// 2. A splat shuffle which uses a scalar_to_vector node which comes from -/// a scalar load, or a constant. -/// The VBROADCAST node is returned when a pattern is found, -/// or SDValue() otherwise. -static SDValue LowerVectorBroadcast(SDValue Op, const X86Subtarget* Subtarget, - SelectionDAG &DAG) { - // VBROADCAST requires AVX. - // TODO: Splats could be generated for non-AVX CPUs using SSE - // instructions, but there's less potential gain for only 128-bit vectors. - if (!Subtarget->hasAVX()) + // Splat is obviously ok. Let legalizer expand it to a shuffle. + if (Values.size() == 1) { + if (EVTBits == 32) { + // Instead of a shuffle like this: + // shuffle (scalar_to_vector (load (ptr + 4))), undef, <0, 0, 0, 0> + // Check if it's possible to issue this instead. + // shuffle (vload ptr)), undef, <1, 1, 1, 1> + unsigned Idx = countTrailingZeros(NonZeros); + SDValue Item = Op.getOperand(Idx); + if (Op.getNode()->isOnlyUserOf(Item.getNode())) + return LowerAsSplatVectorLoad(Item, VT, dl, DAG); + } return SDValue(); + } - MVT VT = Op.getSimpleValueType(); - SDLoc dl(Op); - - assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) && - "Unsupported vector type for broadcast."); + // A vector full of immediates; various special cases are already + // handled, so this is best done with a single constant-pool load. + if (IsAllConstants) + return SDValue(); - SDValue Ld; - bool ConstSplatVal; + // For AVX-length vectors, see if we can use a vector load to get all of the + // elements, otherwise build the individual 128-bit pieces and use + // shuffles to put them in place. + if (VT.is256BitVector() || VT.is512BitVector()) { + SmallVector V(Op->op_begin(), Op->op_begin() + NumElems); - switch (Op.getOpcode()) { - default: - // Unknown pattern found. - return SDValue(); + // Check for a build vector of consecutive loads. + if (SDValue LD = EltsFromConsecutiveLoads(VT, V, dl, DAG, false)) + return LD; - case ISD::BUILD_VECTOR: { - auto *BVOp = cast(Op.getNode()); - BitVector UndefElements; - SDValue Splat = BVOp->getSplatValue(&UndefElements); + EVT HVT = EVT::getVectorVT(*DAG.getContext(), ExtVT, NumElems/2); - // We need a splat of a single value to use broadcast, and it doesn't - // make any sense if the value is only in one element of the vector. - if (!Splat || (VT.getVectorNumElements() - UndefElements.count()) <= 1) - return SDValue(); + // Build both the lower and upper subvector. + SDValue Lower = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT, + makeArrayRef(&V[0], NumElems/2)); + SDValue Upper = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT, + makeArrayRef(&V[NumElems / 2], NumElems/2)); - Ld = Splat; - ConstSplatVal = (Ld.getOpcode() == ISD::Constant || - Ld.getOpcode() == ISD::ConstantFP); + // Recreate the wider vector with the lower and upper part. + if (VT.is256BitVector()) + return Concat128BitVectors(Lower, Upper, VT, NumElems, DAG, dl); + return Concat256BitVectors(Lower, Upper, VT, NumElems, DAG, dl); + } - // Make sure that all of the users of a non-constant load are from the - // BUILD_VECTOR node. - if (!ConstSplatVal && !BVOp->isOnlyUserOf(Ld.getNode())) - return SDValue(); - break; + // Let legalizer expand 2-wide build_vectors. + if (EVTBits == 64) { + if (NumNonZero == 1) { + // One half is zero or undef. + unsigned Idx = countTrailingZeros(NonZeros); + SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, + Op.getOperand(Idx)); + return getShuffleVectorZeroOrUndef(V2, Idx, true, Subtarget, DAG); } + return SDValue(); + } - case ISD::VECTOR_SHUFFLE: { - ShuffleVectorSDNode *SVOp = cast(Op); + // If element VT is < 32 bits, convert it to inserts into a zero vector. + if (EVTBits == 8 && NumElems == 16) { + SDValue V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG, + Subtarget, *this); + if (V.getNode()) return V; + } - // Shuffles must have a splat mask where the first element is - // broadcasted. - if ((!SVOp->isSplat()) || SVOp->getMaskElt(0) != 0) - return SDValue(); + if (EVTBits == 16 && NumElems == 8) { + SDValue V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG, + Subtarget, *this); + if (V.getNode()) return V; + } - SDValue Sc = Op.getOperand(0); - if (Sc.getOpcode() != ISD::SCALAR_TO_VECTOR && - Sc.getOpcode() != ISD::BUILD_VECTOR) { + // If element VT is == 32 bits and has 4 elems, try to generate an INSERTPS + if (EVTBits == 32 && NumElems == 4) { + SDValue V = LowerBuildVectorv4x32(Op, DAG, Subtarget, *this); + if (V.getNode()) + return V; + } - if (!Subtarget->hasInt256()) - return SDValue(); + // If element VT is == 32 bits, turn it into a number of shuffles. + SmallVector V(NumElems); + if (NumElems == 4 && NumZero > 0) { + for (unsigned i = 0; i < 4; ++i) { + bool isZero = !(NonZeros & (1 << i)); + if (isZero) + V[i] = getZeroVector(VT, Subtarget, DAG, dl); + else + V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); + } - // Use the register form of the broadcast instruction available on AVX2. - if (VT.getSizeInBits() >= 256) - Sc = Extract128BitVector(Sc, 0, DAG, dl); - return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Sc); + for (unsigned i = 0; i < 2; ++i) { + switch ((NonZeros & (0x3 << i*2)) >> (i*2)) { + default: break; + case 0: + V[i] = V[i*2]; // Must be a zero vector. + break; + case 1: + V[i] = getMOVL(DAG, dl, VT, V[i*2+1], V[i*2]); + break; + case 2: + V[i] = getMOVL(DAG, dl, VT, V[i*2], V[i*2+1]); + break; + case 3: + V[i] = getUnpackl(DAG, dl, VT, V[i*2], V[i*2+1]); + break; } + } - Ld = Sc.getOperand(0); - ConstSplatVal = (Ld.getOpcode() == ISD::Constant || - Ld.getOpcode() == ISD::ConstantFP); + bool Reverse1 = (NonZeros & 0x3) == 2; + bool Reverse2 = ((NonZeros & (0x3 << 2)) >> 2) == 2; + int MaskVec[] = { + Reverse1 ? 1 : 0, + Reverse1 ? 0 : 1, + static_cast(Reverse2 ? NumElems+1 : NumElems), + static_cast(Reverse2 ? NumElems : NumElems+1) + }; + return DAG.getVectorShuffle(VT, dl, V[0], V[1], &MaskVec[0]); + } - // The scalar_to_vector node and the suspected - // load node must have exactly one user. - // Constants may have multiple users. + if (Values.size() > 1 && VT.is128BitVector()) { + // Check for a build vector of consecutive loads. + for (unsigned i = 0; i < NumElems; ++i) + V[i] = Op.getOperand(i); - // AVX-512 has register version of the broadcast - bool hasRegVer = Subtarget->hasAVX512() && VT.is512BitVector() && - Ld.getValueType().getSizeInBits() >= 32; - if (!ConstSplatVal && ((!Sc.hasOneUse() || !Ld.hasOneUse()) && - !hasRegVer)) - return SDValue(); - break; - } - } + // Check for elements which are consecutive loads. + SDValue LD = EltsFromConsecutiveLoads(VT, V, dl, DAG, false); + if (LD.getNode()) + return LD; - unsigned ScalarSize = Ld.getValueType().getSizeInBits(); - bool IsGE256 = (VT.getSizeInBits() >= 256); + // Check for a build vector from mostly shuffle plus few inserting. + SDValue Sh = buildFromShuffleMostly(Op, DAG); + if (Sh.getNode()) + return Sh; - // When optimizing for size, generate up to 5 extra bytes for a broadcast - // instruction to save 8 or more bytes of constant pool data. - // TODO: If multiple splats are generated to load the same constant, - // it may be detrimental to overall size. There needs to be a way to detect - // that condition to know if this is truly a size win. - const Function *F = DAG.getMachineFunction().getFunction(); - bool OptForSize = F->hasFnAttribute(Attribute::OptimizeForSize); + // For SSE 4.1, use insertps to put the high elements into the low element. + if (Subtarget->hasSSE41()) { + SDValue Result; + if (Op.getOperand(0).getOpcode() != ISD::UNDEF) + Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0)); + else + Result = DAG.getUNDEF(VT); - // Handle broadcasting a single constant scalar from the constant pool - // into a vector. - // On Sandybridge (no AVX2), it is still better to load a constant vector - // from the constant pool and not to broadcast it from a scalar. - // But override that restriction when optimizing for size. - // TODO: Check if splatting is recommended for other AVX-capable CPUs. - if (ConstSplatVal && (Subtarget->hasAVX2() || OptForSize)) { - EVT CVT = Ld.getValueType(); - assert(!CVT.isVector() && "Must not broadcast a vector type"); - - // Splat f32, i32, v4f64, v4i64 in all cases with AVX2. - // For size optimization, also splat v2f64 and v2i64, and for size opt - // with AVX2, also splat i8 and i16. - // With pattern matching, the VBROADCAST node may become a VMOVDDUP. - if (ScalarSize == 32 || (IsGE256 && ScalarSize == 64) || - (OptForSize && (ScalarSize == 64 || Subtarget->hasAVX2()))) { - const Constant *C = nullptr; - if (ConstantSDNode *CI = dyn_cast(Ld)) - C = CI->getConstantIntValue(); - else if (ConstantFPSDNode *CF = dyn_cast(Ld)) - C = CF->getConstantFPValue(); + for (unsigned i = 1; i < NumElems; ++i) { + if (Op.getOperand(i).getOpcode() == ISD::UNDEF) continue; + Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result, + Op.getOperand(i), DAG.getIntPtrConstant(i)); + } + return Result; + } - assert(C && "Invalid constant type"); + // Otherwise, expand into a number of unpckl*, start by extending each of + // our (non-undef) elements to the full vector width with the element in the + // bottom slot of the vector (which generates no code for SSE). + for (unsigned i = 0; i < NumElems; ++i) { + if (Op.getOperand(i).getOpcode() != ISD::UNDEF) + V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); + else + V[i] = DAG.getUNDEF(VT); + } - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - SDValue CP = DAG.getConstantPool(C, TLI.getPointerTy()); - unsigned Alignment = cast(CP)->getAlignment(); - Ld = DAG.getLoad(CVT, dl, DAG.getEntryNode(), CP, - MachinePointerInfo::getConstantPool(), - false, false, false, Alignment); + // Next, we iteratively mix elements, e.g. for v4f32: + // Step 1: unpcklps 0, 2 ==> X: + // : unpcklps 1, 3 ==> Y: + // Step 2: unpcklps X, Y ==> <3, 2, 1, 0> + unsigned EltStride = NumElems >> 1; + while (EltStride != 0) { + for (unsigned i = 0; i < EltStride; ++i) { + // If V[i+EltStride] is undef and this is the first round of mixing, + // then it is safe to just drop this shuffle: V[i] is already in the + // right place, the one element (since it's the first round) being + // inserted as undef can be dropped. This isn't safe for successive + // rounds because they will permute elements within both vectors. + if (V[i+EltStride].getOpcode() == ISD::UNDEF && + EltStride == NumElems/2) + continue; - return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); + V[i] = getUnpackl(DAG, dl, VT, V[i], V[i + EltStride]); + } + EltStride >>= 1; } + return V[0]; } + return SDValue(); +} - bool IsLoad = ISD::isNormalLoad(Ld.getNode()); - - // Handle AVX2 in-register broadcasts. - if (!IsLoad && Subtarget->hasInt256() && - (ScalarSize == 32 || (IsGE256 && ScalarSize == 64))) - return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); +// LowerAVXCONCAT_VECTORS - 256-bit AVX can use the vinsertf128 instruction +// to create 256-bit vectors from two other 128-bit ones. +static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) { + SDLoc dl(Op); + MVT ResVT = Op.getSimpleValueType(); - // The scalar source must be a normal load. - if (!IsLoad) - return SDValue(); + assert((ResVT.is256BitVector() || + ResVT.is512BitVector()) && "Value type must be 256-/512-bit wide"); - if (ScalarSize == 32 || (IsGE256 && ScalarSize == 64) || - (Subtarget->hasVLX() && ScalarSize == 64)) - return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); + SDValue V1 = Op.getOperand(0); + SDValue V2 = Op.getOperand(1); + unsigned NumElems = ResVT.getVectorNumElements(); + if(ResVT.is256BitVector()) + return Concat128BitVectors(V1, V2, ResVT, NumElems, DAG, dl); - // The integer check is needed for the 64-bit into 128-bit so it doesn't match - // double since there is no vbroadcastsd xmm - if (Subtarget->hasInt256() && Ld.getValueType().isInteger()) { - if (ScalarSize == 8 || ScalarSize == 16 || ScalarSize == 64) - return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); + if (Op.getNumOperands() == 4) { + MVT HalfVT = MVT::getVectorVT(ResVT.getScalarType(), + ResVT.getVectorNumElements()/2); + SDValue V3 = Op.getOperand(2); + SDValue V4 = Op.getOperand(3); + return Concat256BitVectors(Concat128BitVectors(V1, V2, HalfVT, NumElems/2, DAG, dl), + Concat128BitVectors(V3, V4, HalfVT, NumElems/2, DAG, dl), ResVT, NumElems, DAG, dl); } - - // Unsupported broadcast. - return SDValue(); + return Concat256BitVectors(V1, V2, ResVT, NumElems, DAG, dl); } -/// \brief For an EXTRACT_VECTOR_ELT with a constant index return the real -/// underlying vector and index. -/// -/// Modifies \p ExtractedFromVec to the real vector and returns the real -/// index. -static int getUnderlyingExtractedFromVec(SDValue &ExtractedFromVec, - SDValue ExtIdx) { - int Idx = cast(ExtIdx)->getZExtValue(); - if (!isa(ExtractedFromVec)) - return Idx; +static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) { + MVT LLVM_ATTRIBUTE_UNUSED VT = Op.getSimpleValueType(); + assert((VT.is256BitVector() && Op.getNumOperands() == 2) || + (VT.is512BitVector() && (Op.getNumOperands() == 2 || + Op.getNumOperands() == 4))); - // For 256-bit vectors, LowerEXTRACT_VECTOR_ELT_SSE4 may have already - // lowered this: - // (extract_vector_elt (v8f32 %vreg1), Constant<6>) - // to: - // (extract_vector_elt (vector_shuffle<2,u,u,u> - // (extract_subvector (v8f32 %vreg0), Constant<4>), - // undef) - // Constant<0>) - // In this case the vector is the extract_subvector expression and the index - // is 2, as specified by the shuffle. - ShuffleVectorSDNode *SVOp = cast(ExtractedFromVec); - SDValue ShuffleVec = SVOp->getOperand(0); - MVT ShuffleVecVT = ShuffleVec.getSimpleValueType(); - assert(ShuffleVecVT.getVectorElementType() == - ExtractedFromVec.getSimpleValueType().getVectorElementType()); + // AVX can use the vinsertf128 instruction to create 256-bit vectors + // from two other 128-bit ones. - int ShuffleIdx = SVOp->getMaskElt(Idx); - if (isUndefOrInRange(ShuffleIdx, 0, ShuffleVecVT.getVectorNumElements())) { - ExtractedFromVec = ShuffleVec; - return ShuffleIdx; - } - return Idx; + // 512-bit vector may contain 2 256-bit vectors or 4 128-bit vectors + return LowerAVXCONCAT_VECTORS(Op, DAG); } -static SDValue buildFromShuffleMostly(SDValue Op, SelectionDAG &DAG) { - MVT VT = Op.getSimpleValueType(); - // Skip if insert_vec_elt is not supported. - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - if (!TLI.isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT)) - return SDValue(); +//===----------------------------------------------------------------------===// +// Vector shuffle lowering +// +// This is an experimental code path for lowering vector shuffles on x86. It is +// designed to handle arbitrary vector shuffles and blends, gracefully +// degrading performance as necessary. It works hard to recognize idiomatic +// shuffles and lower them to optimal instruction patterns without leaving +// a framework that allows reasonably efficient handling of all vector shuffle +// patterns. +//===----------------------------------------------------------------------===// - SDLoc DL(Op); - unsigned NumElems = Op.getNumOperands(); +/// \brief Tiny helper function to identify a no-op mask. +/// +/// This is a somewhat boring predicate function. It checks whether the mask +/// array input, which is assumed to be a single-input shuffle mask of the kind +/// used by the X86 shuffle instructions (not a fully general +/// ShuffleVectorSDNode mask) requires any shuffles to occur. Both undef and an +/// in-place shuffle are 'no-op's. +static bool isNoopShuffleMask(ArrayRef Mask) { + for (int i = 0, Size = Mask.size(); i < Size; ++i) + if (Mask[i] != -1 && Mask[i] != i) + return false; + return true; +} - SDValue VecIn1; - SDValue VecIn2; - SmallVector InsertIndices; - SmallVector Mask(NumElems, -1); +/// \brief Helper function to classify a mask as a single-input mask. +/// +/// This isn't a generic single-input test because in the vector shuffle +/// lowering we canonicalize single inputs to be the first input operand. This +/// means we can more quickly test for a single input by only checking whether +/// an input from the second operand exists. We also assume that the size of +/// mask corresponds to the size of the input vectors which isn't true in the +/// fully general case. +static bool isSingleInputShuffleMask(ArrayRef Mask) { + for (int M : Mask) + if (M >= (int)Mask.size()) + return false; + return true; +} - for (unsigned i = 0; i != NumElems; ++i) { - unsigned Opc = Op.getOperand(i).getOpcode(); +/// \brief Test whether there are elements crossing 128-bit lanes in this +/// shuffle mask. +/// +/// X86 divides up its shuffles into in-lane and cross-lane shuffle operations +/// and we routinely test for these. +static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef Mask) { + int LaneSize = 128 / VT.getScalarSizeInBits(); + int Size = Mask.size(); + for (int i = 0; i < Size; ++i) + if (Mask[i] >= 0 && (Mask[i] % Size) / LaneSize != i / LaneSize) + return true; + return false; +} - if (Opc == ISD::UNDEF) +/// \brief Test whether a shuffle mask is equivalent within each 128-bit lane. +/// +/// This checks a shuffle mask to see if it is performing the same +/// 128-bit lane-relative shuffle in each 128-bit lane. This trivially implies +/// that it is also not lane-crossing. It may however involve a blend from the +/// same lane of a second vector. +/// +/// The specific repeated shuffle mask is populated in \p RepeatedMask, as it is +/// non-trivial to compute in the face of undef lanes. The representation is +/// *not* suitable for use with existing 128-bit shuffles as it will contain +/// entries from both V1 and V2 inputs to the wider mask. +static bool +is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef Mask, + SmallVectorImpl &RepeatedMask) { + int LaneSize = 128 / VT.getScalarSizeInBits(); + RepeatedMask.resize(LaneSize, -1); + int Size = Mask.size(); + for (int i = 0; i < Size; ++i) { + if (Mask[i] < 0) continue; + if ((Mask[i] % Size) / LaneSize != i / LaneSize) + // This entry crosses lanes, so there is no way to model this shuffle. + return false; - if (Opc != ISD::EXTRACT_VECTOR_ELT) { - // Quit if more than 1 elements need inserting. - if (InsertIndices.size() > 1) - return SDValue(); - - InsertIndices.push_back(i); - continue; - } + // Ok, handle the in-lane shuffles by detecting if and when they repeat. + if (RepeatedMask[i % LaneSize] == -1) + // This is the first non-undef entry in this slot of a 128-bit lane. + RepeatedMask[i % LaneSize] = + Mask[i] < Size ? Mask[i] % LaneSize : Mask[i] % LaneSize + Size; + else if (RepeatedMask[i % LaneSize] + (i / LaneSize) * LaneSize != Mask[i]) + // Found a mismatch with the repeated mask. + return false; + } + return true; +} - SDValue ExtractedFromVec = Op.getOperand(i).getOperand(0); - SDValue ExtIdx = Op.getOperand(i).getOperand(1); - // Quit if non-constant index. - if (!isa(ExtIdx)) - return SDValue(); - int Idx = getUnderlyingExtractedFromVec(ExtractedFromVec, ExtIdx); - - // Quit if extracted from vector of different type. - if (ExtractedFromVec.getValueType() != VT) - return SDValue(); +/// \brief Base case helper for testing a single mask element. +static bool isShuffleEquivalentImpl(SDValue V1, SDValue V2, + BuildVectorSDNode *BV1, + BuildVectorSDNode *BV2, ArrayRef Mask, + int i, int Arg) { + int Size = Mask.size(); + if (Mask[i] != -1 && Mask[i] != Arg) { + auto *MaskBV = Mask[i] < Size ? BV1 : BV2; + auto *ArgsBV = Arg < Size ? BV1 : BV2; + if (!MaskBV || !ArgsBV || + MaskBV->getOperand(Mask[i] % Size) != ArgsBV->getOperand(Arg % Size)) + return false; + } + return true; +} - if (!VecIn1.getNode()) - VecIn1 = ExtractedFromVec; - else if (VecIn1 != ExtractedFromVec) { - if (!VecIn2.getNode()) - VecIn2 = ExtractedFromVec; - else if (VecIn2 != ExtractedFromVec) - // Quit if more than 2 vectors to shuffle - return SDValue(); - } +/// \brief Recursive helper to peel off and test each mask element. +template +static bool isShuffleEquivalentImpl(SDValue V1, SDValue V2, + BuildVectorSDNode *BV1, + BuildVectorSDNode *BV2, ArrayRef Mask, + int i, int Arg, Ts... Args) { + if (!isShuffleEquivalentImpl(V1, V2, BV1, BV2, Mask, i, Arg)) + return false; - if (ExtractedFromVec == VecIn1) - Mask[i] = Idx; - else if (ExtractedFromVec == VecIn2) - Mask[i] = Idx + NumElems; - } + return isShuffleEquivalentImpl(V1, V2, BV1, BV2, Mask, i + 1, Args...); +} - if (!VecIn1.getNode()) - return SDValue(); +/// \brief Checks whether a shuffle mask is equivalent to an explicit list of +/// arguments. +/// +/// This is a fast way to test a shuffle mask against a fixed pattern: +/// +/// if (isShuffleEquivalent(Mask, 3, 2, 1, 0)) { ... } +/// +/// It returns true if the mask is exactly as wide as the argument list, and +/// each element of the mask is either -1 (signifying undef) or the value given +/// in the argument. +template +static bool isShuffleEquivalent(SDValue V1, SDValue V2, ArrayRef Mask, + Ts... Args) { + if (Mask.size() != sizeof...(Args)) + return false; - VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT); - SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, &Mask[0]); - for (unsigned i = 0, e = InsertIndices.size(); i != e; ++i) { - unsigned Idx = InsertIndices[i]; - NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx), - DAG.getIntPtrConstant(Idx)); - } + // If the values are build vectors, we can look through them to find + // equivalent inputs that make the shuffles equivalent. + auto *BV1 = dyn_cast(V1); + auto *BV2 = dyn_cast(V2); - return NV; + // Recursively peel off arguments and test them against the mask. + return isShuffleEquivalentImpl(V1, V2, BV1, BV2, Mask, 0, Args...); } -// Lower BUILD_VECTOR operation for v8i1 and v16i1 types. -SDValue -X86TargetLowering::LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG) const { +/// \brief Get a 4-lane 8-bit shuffle immediate for a mask. +/// +/// This helper function produces an 8-bit shuffle immediate corresponding to +/// the ubiquitous shuffle encoding scheme used in x86 instructions for +/// shuffling 4 lanes. It can be used with most of the PSHUF instructions for +/// example. +/// +/// NB: We rely heavily on "undef" masks preserving the input lane. +static SDValue getV4X86ShuffleImm8ForMask(ArrayRef Mask, + SelectionDAG &DAG) { + assert(Mask.size() == 4 && "Only 4-lane shuffle masks"); + assert(Mask[0] >= -1 && Mask[0] < 4 && "Out of bound mask element!"); + assert(Mask[1] >= -1 && Mask[1] < 4 && "Out of bound mask element!"); + assert(Mask[2] >= -1 && Mask[2] < 4 && "Out of bound mask element!"); + assert(Mask[3] >= -1 && Mask[3] < 4 && "Out of bound mask element!"); - MVT VT = Op.getSimpleValueType(); - assert((VT.getVectorElementType() == MVT::i1) && (VT.getSizeInBits() <= 16) && - "Unexpected type in LowerBUILD_VECTORvXi1!"); + unsigned Imm = 0; + Imm |= (Mask[0] == -1 ? 0 : Mask[0]) << 0; + Imm |= (Mask[1] == -1 ? 1 : Mask[1]) << 2; + Imm |= (Mask[2] == -1 ? 2 : Mask[2]) << 4; + Imm |= (Mask[3] == -1 ? 3 : Mask[3]) << 6; + return DAG.getConstant(Imm, MVT::i8); +} - SDLoc dl(Op); - if (ISD::isBuildVectorAllZeros(Op.getNode())) { - SDValue Cst = DAG.getTargetConstant(0, MVT::i1); - SmallVector Ops(VT.getVectorNumElements(), Cst); - return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); +/// \brief Try to emit a blend instruction for a shuffle using bit math. +/// +/// This is used as a fallback approach when first class blend instructions are +/// unavailable. Currently it is only suitable for integer vectors, but could +/// be generalized for floating point vectors if desirable. +static SDValue lowerVectorShuffleAsBitBlend(SDLoc DL, MVT VT, SDValue V1, + SDValue V2, ArrayRef Mask, + SelectionDAG &DAG) { + assert(VT.isInteger() && "Only supports integer vector types!"); + MVT EltVT = VT.getScalarType(); + int NumEltBits = EltVT.getSizeInBits(); + SDValue Zero = DAG.getConstant(0, EltVT); + SDValue AllOnes = DAG.getConstant(APInt::getAllOnesValue(NumEltBits), EltVT); + SmallVector MaskOps; + for (int i = 0, Size = Mask.size(); i < Size; ++i) { + if (Mask[i] != -1 && Mask[i] != i && Mask[i] != i + Size) + return SDValue(); // Shuffled input! + MaskOps.push_back(Mask[i] < Size ? AllOnes : Zero); } - if (ISD::isBuildVectorAllOnes(Op.getNode())) { - SDValue Cst = DAG.getTargetConstant(1, MVT::i1); - SmallVector Ops(VT.getVectorNumElements(), Cst); - return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); - } + SDValue V1Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, VT, MaskOps); + V1 = DAG.getNode(ISD::AND, DL, VT, V1, V1Mask); + // We have to cast V2 around. + MVT MaskVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64); + V2 = DAG.getNode(ISD::BITCAST, DL, VT, + DAG.getNode(X86ISD::ANDNP, DL, MaskVT, + DAG.getNode(ISD::BITCAST, DL, MaskVT, V1Mask), + DAG.getNode(ISD::BITCAST, DL, MaskVT, V2))); + return DAG.getNode(ISD::OR, DL, VT, V1, V2); +} - bool AllContants = true; - uint64_t Immediate = 0; - int NonConstIdx = -1; - bool IsSplat = true; - unsigned NumNonConsts = 0; - unsigned NumConsts = 0; - for (unsigned idx = 0, e = Op.getNumOperands(); idx < e; ++idx) { - SDValue In = Op.getOperand(idx); - if (In.getOpcode() == ISD::UNDEF) +/// \brief Try to emit a blend instruction for a shuffle. +/// +/// This doesn't do any checks for the availability of instructions for blending +/// these values. It relies on the availability of the X86ISD::BLENDI pattern to +/// be matched in the backend with the type given. What it does check for is +/// that the shuffle mask is in fact a blend. +static SDValue lowerVectorShuffleAsBlend(SDLoc DL, MVT VT, SDValue V1, + SDValue V2, ArrayRef Mask, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + unsigned BlendMask = 0; + for (int i = 0, Size = Mask.size(); i < Size; ++i) { + if (Mask[i] >= Size) { + if (Mask[i] != i + Size) + return SDValue(); // Shuffled V2 input! + BlendMask |= 1u << i; continue; - if (!isa(In)) { - AllContants = false; - NonConstIdx = idx; - NumNonConsts++; - } else { - NumConsts++; - if (cast(In)->getZExtValue()) - Immediate |= (1ULL << idx); } - if (In != Op.getOperand(0)) - IsSplat = false; + if (Mask[i] >= 0 && Mask[i] != i) + return SDValue(); // Shuffled V1 input! } + switch (VT.SimpleTy) { + case MVT::v2f64: + case MVT::v4f32: + case MVT::v4f64: + case MVT::v8f32: + return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2, + DAG.getConstant(BlendMask, MVT::i8)); - if (AllContants) { - SDValue FullMask = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, - DAG.getConstant(Immediate, MVT::i16)); - return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, FullMask, - DAG.getIntPtrConstant(0)); - } + case MVT::v4i64: + case MVT::v8i32: + assert(Subtarget->hasAVX2() && "256-bit integer blends require AVX2!"); + // FALLTHROUGH + case MVT::v2i64: + case MVT::v4i32: + // If we have AVX2 it is faster to use VPBLENDD when the shuffle fits into + // that instruction. + if (Subtarget->hasAVX2()) { + // Scale the blend by the number of 32-bit dwords per element. + int Scale = VT.getScalarSizeInBits() / 32; + BlendMask = 0; + for (int i = 0, Size = Mask.size(); i < Size; ++i) + if (Mask[i] >= Size) + for (int j = 0; j < Scale; ++j) + BlendMask |= 1u << (i * Scale + j); - if (NumNonConsts == 1 && NonConstIdx != 0) { - SDValue DstVec; - if (NumConsts) { - SDValue VecAsImm = DAG.getConstant(Immediate, - MVT::getIntegerVT(VT.getSizeInBits())); - DstVec = DAG.getNode(ISD::BITCAST, dl, VT, VecAsImm); + MVT BlendVT = VT.getSizeInBits() > 128 ? MVT::v8i32 : MVT::v4i32; + V1 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V1); + V2 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V2); + return DAG.getNode(ISD::BITCAST, DL, VT, + DAG.getNode(X86ISD::BLENDI, DL, BlendVT, V1, V2, + DAG.getConstant(BlendMask, MVT::i8))); } - else - DstVec = DAG.getUNDEF(VT); - return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DstVec, - Op.getOperand(NonConstIdx), - DAG.getIntPtrConstant(NonConstIdx)); + // FALLTHROUGH + case MVT::v8i16: { + // For integer shuffles we need to expand the mask and cast the inputs to + // v8i16s prior to blending. + int Scale = 8 / VT.getVectorNumElements(); + BlendMask = 0; + for (int i = 0, Size = Mask.size(); i < Size; ++i) + if (Mask[i] >= Size) + for (int j = 0; j < Scale; ++j) + BlendMask |= 1u << (i * Scale + j); + + V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1); + V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V2); + return DAG.getNode(ISD::BITCAST, DL, VT, + DAG.getNode(X86ISD::BLENDI, DL, MVT::v8i16, V1, V2, + DAG.getConstant(BlendMask, MVT::i8))); } - if (!IsSplat && (NonConstIdx != 0)) - llvm_unreachable("Unsupported BUILD_VECTOR operation"); - MVT SelectVT = (VT == MVT::v16i1)? MVT::i16 : MVT::i8; - SDValue Select; - if (IsSplat) - Select = DAG.getNode(ISD::SELECT, dl, SelectVT, Op.getOperand(0), - DAG.getConstant(-1, SelectVT), - DAG.getConstant(0, SelectVT)); - else - Select = DAG.getNode(ISD::SELECT, dl, SelectVT, Op.getOperand(0), - DAG.getConstant((Immediate | 1), SelectVT), - DAG.getConstant(Immediate, SelectVT)); - return DAG.getNode(ISD::BITCAST, dl, VT, Select); -} -/// \brief Return true if \p N implements a horizontal binop and return the -/// operands for the horizontal binop into V0 and V1. -/// -/// This is a helper function of PerformBUILD_VECTORCombine. -/// This function checks that the build_vector \p N in input implements a -/// horizontal operation. Parameter \p Opcode defines the kind of horizontal -/// operation to match. -/// For example, if \p Opcode is equal to ISD::ADD, then this function -/// checks if \p N implements a horizontal arithmetic add; if instead \p Opcode -/// is equal to ISD::SUB, then this function checks if this is a horizontal -/// arithmetic sub. -/// -/// This function only analyzes elements of \p N whose indices are -/// in range [BaseIdx, LastIdx). -static bool isHorizontalBinOp(const BuildVectorSDNode *N, unsigned Opcode, - SelectionDAG &DAG, - unsigned BaseIdx, unsigned LastIdx, - SDValue &V0, SDValue &V1) { - EVT VT = N->getValueType(0); + case MVT::v16i16: { + assert(Subtarget->hasAVX2() && "256-bit integer blends require AVX2!"); + SmallVector RepeatedMask; + if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) { + // We can lower these with PBLENDW which is mirrored across 128-bit lanes. + assert(RepeatedMask.size() == 8 && "Repeated mask size doesn't match!"); + BlendMask = 0; + for (int i = 0; i < 8; ++i) + if (RepeatedMask[i] >= 16) + BlendMask |= 1u << i; + return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2, + DAG.getConstant(BlendMask, MVT::i8)); + } + } + // FALLTHROUGH + case MVT::v16i8: + case MVT::v32i8: { + // Scale the blend by the number of bytes per element. + int Scale = VT.getScalarSizeInBits() / 8; - assert(BaseIdx * 2 <= LastIdx && "Invalid Indices in input!"); - assert(VT.isVector() && VT.getVectorNumElements() >= LastIdx && - "Invalid Vector in input!"); + // This form of blend is always done on bytes. Compute the byte vector + // type. + MVT BlendVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); - bool IsCommutable = (Opcode == ISD::ADD || Opcode == ISD::FADD); - bool CanFold = true; - unsigned ExpectedVExtractIdx = BaseIdx; - unsigned NumElts = LastIdx - BaseIdx; - V0 = DAG.getUNDEF(VT); - V1 = DAG.getUNDEF(VT); + // Compute the VSELECT mask. Note that VSELECT is really confusing in the + // mix of LLVM's code generator and the x86 backend. We tell the code + // generator that boolean values in the elements of an x86 vector register + // are -1 for true and 0 for false. We then use the LLVM semantics of 'true' + // mapping a select to operand #1, and 'false' mapping to operand #2. The + // reality in x86 is that vector masks (pre-AVX-512) use only the high bit + // of the element (the remaining are ignored) and 0 in that high bit would + // mean operand #1 while 1 in the high bit would mean operand #2. So while + // the LLVM model for boolean values in vector elements gets the relevant + // bit set, it is set backwards and over constrained relative to x86's + // actual model. + SmallVector VSELECTMask; + for (int i = 0, Size = Mask.size(); i < Size; ++i) + for (int j = 0; j < Scale; ++j) + VSELECTMask.push_back( + Mask[i] < 0 ? DAG.getUNDEF(MVT::i8) + : DAG.getConstant(Mask[i] < Size ? -1 : 0, MVT::i8)); - // Check if N implements a horizontal binop. - for (unsigned i = 0, e = NumElts; i != e && CanFold; ++i) { - SDValue Op = N->getOperand(i + BaseIdx); + V1 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V1); + V2 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V2); + return DAG.getNode( + ISD::BITCAST, DL, VT, + DAG.getNode(ISD::VSELECT, DL, BlendVT, + DAG.getNode(ISD::BUILD_VECTOR, DL, BlendVT, VSELECTMask), + V1, V2)); + } - // Skip UNDEFs. - if (Op->getOpcode() == ISD::UNDEF) { - // Update the expected vector extract index. - if (i * 2 == NumElts) - ExpectedVExtractIdx = BaseIdx; - ExpectedVExtractIdx += 2; - continue; - } + default: + llvm_unreachable("Not a supported integer vector type!"); + } +} - CanFold = Op->getOpcode() == Opcode && Op->hasOneUse(); +/// \brief Try to lower as a blend of elements from two inputs followed by +/// a single-input permutation. +/// +/// This matches the pattern where we can blend elements from two inputs and +/// then reduce the shuffle to a single-input permutation. +static SDValue lowerVectorShuffleAsBlendAndPermute(SDLoc DL, MVT VT, SDValue V1, + SDValue V2, + ArrayRef Mask, + SelectionDAG &DAG) { + // We build up the blend mask while checking whether a blend is a viable way + // to reduce the shuffle. + SmallVector BlendMask(Mask.size(), -1); + SmallVector PermuteMask(Mask.size(), -1); - if (!CanFold) - break; + for (int i = 0, Size = Mask.size(); i < Size; ++i) { + if (Mask[i] < 0) + continue; - SDValue Op0 = Op.getOperand(0); - SDValue Op1 = Op.getOperand(1); + assert(Mask[i] < Size * 2 && "Shuffle input is out of bounds."); - // Try to match the following pattern: - // (BINOP (extract_vector_elt A, I), (extract_vector_elt A, I+1)) - CanFold = (Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && - Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT && - Op0.getOperand(0) == Op1.getOperand(0) && - isa(Op0.getOperand(1)) && - isa(Op1.getOperand(1))); - if (!CanFold) - break; + if (BlendMask[Mask[i] % Size] == -1) + BlendMask[Mask[i] % Size] = Mask[i]; + else if (BlendMask[Mask[i] % Size] != Mask[i]) + return SDValue(); // Can't blend in the needed input! - unsigned I0 = cast(Op0.getOperand(1))->getZExtValue(); - unsigned I1 = cast(Op1.getOperand(1))->getZExtValue(); + PermuteMask[i] = Mask[i] % Size; + } - if (i * 2 < NumElts) { - if (V0.getOpcode() == ISD::UNDEF) - V0 = Op0.getOperand(0); - } else { - if (V1.getOpcode() == ISD::UNDEF) - V1 = Op0.getOperand(0); - if (i * 2 == NumElts) - ExpectedVExtractIdx = BaseIdx; - } + SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask); + return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), PermuteMask); +} - SDValue Expected = (i * 2 < NumElts) ? V0 : V1; - if (I0 == ExpectedVExtractIdx) - CanFold = I1 == I0 + 1 && Op0.getOperand(0) == Expected; - else if (IsCommutable && I1 == ExpectedVExtractIdx) { - // Try to match the following dag sequence: - // (BINOP (extract_vector_elt A, I+1), (extract_vector_elt A, I)) - CanFold = I0 == I1 + 1 && Op1.getOperand(0) == Expected; - } else - CanFold = false; +/// \brief Generic routine to decompose a shuffle and blend into indepndent +/// blends and permutes. +/// +/// This matches the extremely common pattern for handling combined +/// shuffle+blend operations on newer X86 ISAs where we have very fast blend +/// operations. It will try to pick the best arrangement of shuffles and +/// blends. +static SDValue lowerVectorShuffleAsDecomposedShuffleBlend(SDLoc DL, MVT VT, + SDValue V1, + SDValue V2, + ArrayRef Mask, + SelectionDAG &DAG) { + // Shuffle the input elements into the desired positions in V1 and V2 and + // blend them together. + SmallVector V1Mask(Mask.size(), -1); + SmallVector V2Mask(Mask.size(), -1); + SmallVector BlendMask(Mask.size(), -1); + for (int i = 0, Size = Mask.size(); i < Size; ++i) + if (Mask[i] >= 0 && Mask[i] < Size) { + V1Mask[i] = Mask[i]; + BlendMask[i] = i; + } else if (Mask[i] >= Size) { + V2Mask[i] = Mask[i] - Size; + BlendMask[i] = i + Size; + } - ExpectedVExtractIdx += 2; - } + // Try to lower with the simpler initial blend strategy unless one of the + // input shuffles would be a no-op. We prefer to shuffle inputs as the + // shuffle may be able to fold with a load or other benefit. However, when + // we'll have to do 2x as many shuffles in order to achieve this, blending + // first is a better strategy. + if (!isNoopShuffleMask(V1Mask) && !isNoopShuffleMask(V2Mask)) + if (SDValue BlendPerm = + lowerVectorShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask, DAG)) + return BlendPerm; - return CanFold; + V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask); + V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask); + return DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask); } -/// \brief Emit a sequence of two 128-bit horizontal add/sub followed by -/// a concat_vector. -/// -/// This is a helper function of PerformBUILD_VECTORCombine. -/// This function expects two 256-bit vectors called V0 and V1. -/// At first, each vector is split into two separate 128-bit vectors. -/// Then, the resulting 128-bit vectors are used to implement two -/// horizontal binary operations. -/// -/// The kind of horizontal binary operation is defined by \p X86Opcode. +/// \brief Try to lower a vector shuffle as a byte rotation. /// -/// \p Mode specifies how the 128-bit parts of V0 and V1 are passed in input to -/// the two new horizontal binop. -/// When Mode is set, the first horizontal binop dag node would take as input -/// the lower 128-bit of V0 and the upper 128-bit of V0. The second -/// horizontal binop dag node would take as input the lower 128-bit of V1 -/// and the upper 128-bit of V1. -/// Example: -/// HADD V0_LO, V0_HI -/// HADD V1_LO, V1_HI +/// SSSE3 has a generic PALIGNR instruction in x86 that will do an arbitrary +/// byte-rotation of the concatenation of two vectors; pre-SSSE3 can use +/// a PSRLDQ/PSLLDQ/POR pattern to get a similar effect. This routine will +/// try to generically lower a vector shuffle through such an pattern. It +/// does not check for the profitability of lowering either as PALIGNR or +/// PSRLDQ/PSLLDQ/POR, only whether the mask is valid to lower in that form. +/// This matches shuffle vectors that look like: /// -/// Otherwise, the first horizontal binop dag node takes as input the lower -/// 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop -/// dag node takes the the upper 128-bit of V0 and the upper 128-bit of V1. -/// Example: -/// HADD V0_LO, V1_LO -/// HADD V0_HI, V1_HI +/// v8i16 [11, 12, 13, 14, 15, 0, 1, 2] /// -/// If \p isUndefLO is set, then the algorithm propagates UNDEF to the lower -/// 128-bits of the result. If \p isUndefHI is set, then UNDEF is propagated to -/// the upper 128-bits of the result. -static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1, - SDLoc DL, SelectionDAG &DAG, - unsigned X86Opcode, bool Mode, - bool isUndefLO, bool isUndefHI) { - EVT VT = V0.getValueType(); - assert(VT.is256BitVector() && VT == V1.getValueType() && - "Invalid nodes in input!"); - - unsigned NumElts = VT.getVectorNumElements(); - SDValue V0_LO = Extract128BitVector(V0, 0, DAG, DL); - SDValue V0_HI = Extract128BitVector(V0, NumElts/2, DAG, DL); - SDValue V1_LO = Extract128BitVector(V1, 0, DAG, DL); - SDValue V1_HI = Extract128BitVector(V1, NumElts/2, DAG, DL); - EVT NewVT = V0_LO.getValueType(); +/// Essentially it concatenates V1 and V2, shifts right by some number of +/// elements, and takes the low elements as the result. Note that while this is +/// specified as a *right shift* because x86 is little-endian, it is a *left +/// rotate* of the vector lanes. +static SDValue lowerVectorShuffleAsByteRotate(SDLoc DL, MVT VT, SDValue V1, + SDValue V2, + ArrayRef Mask, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!"); - SDValue LO = DAG.getUNDEF(NewVT); - SDValue HI = DAG.getUNDEF(NewVT); + int NumElts = Mask.size(); + int NumLanes = VT.getSizeInBits() / 128; + int NumLaneElts = NumElts / NumLanes; - if (Mode) { - // Don't emit a horizontal binop if the result is expected to be UNDEF. - if (!isUndefLO && V0->getOpcode() != ISD::UNDEF) - LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V0_HI); - if (!isUndefHI && V1->getOpcode() != ISD::UNDEF) - HI = DAG.getNode(X86Opcode, DL, NewVT, V1_LO, V1_HI); - } else { - // Don't emit a horizontal binop if the result is expected to be UNDEF. - if (!isUndefLO && (V0_LO->getOpcode() != ISD::UNDEF || - V1_LO->getOpcode() != ISD::UNDEF)) - LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V1_LO); + // We need to detect various ways of spelling a rotation: + // [11, 12, 13, 14, 15, 0, 1, 2] + // [-1, 12, 13, 14, -1, -1, 1, -1] + // [-1, -1, -1, -1, -1, -1, 1, 2] + // [ 3, 4, 5, 6, 7, 8, 9, 10] + // [-1, 4, 5, 6, -1, -1, 9, -1] + // [-1, 4, 5, 6, -1, -1, -1, -1] + int Rotation = 0; + SDValue Lo, Hi; + for (int l = 0; l < NumElts; l += NumLaneElts) { + for (int i = 0; i < NumLaneElts; ++i) { + if (Mask[l + i] == -1) + continue; + assert(Mask[l + i] >= 0 && "Only -1 is a valid negative mask element!"); - if (!isUndefHI && (V0_HI->getOpcode() != ISD::UNDEF || - V1_HI->getOpcode() != ISD::UNDEF)) - HI = DAG.getNode(X86Opcode, DL, NewVT, V0_HI, V1_HI); - } + // Get the mod-Size index and lane correct it. + int LaneIdx = (Mask[l + i] % NumElts) - l; + // Make sure it was in this lane. + if (LaneIdx < 0 || LaneIdx >= NumLaneElts) + return SDValue(); - return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI); -} + // Determine where a rotated vector would have started. + int StartIdx = i - LaneIdx; + if (StartIdx == 0) + // The identity rotation isn't interesting, stop. + return SDValue(); -/// \brief Try to fold a build_vector that performs an 'addsub' into the -/// sequence of 'vadd + vsub + blendi'. -static SDValue matchAddSub(const BuildVectorSDNode *BV, SelectionDAG &DAG, - const X86Subtarget *Subtarget) { - SDLoc DL(BV); - EVT VT = BV->getValueType(0); - unsigned NumElts = VT.getVectorNumElements(); - SDValue InVec0 = DAG.getUNDEF(VT); - SDValue InVec1 = DAG.getUNDEF(VT); + // If we found the tail of a vector the rotation must be the missing + // front. If we found the head of a vector, it must be how much of the + // head. + int CandidateRotation = StartIdx < 0 ? -StartIdx : NumLaneElts - StartIdx; - assert((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v4f32 || - VT == MVT::v2f64) && "build_vector with an invalid type found!"); + if (Rotation == 0) + Rotation = CandidateRotation; + else if (Rotation != CandidateRotation) + // The rotations don't match, so we can't match this mask. + return SDValue(); - // Odd-numbered elements in the input build vector are obtained from - // adding two integer/float elements. - // Even-numbered elements in the input build vector are obtained from - // subtracting two integer/float elements. - unsigned ExpectedOpcode = ISD::FSUB; - unsigned NextExpectedOpcode = ISD::FADD; - bool AddFound = false; - bool SubFound = false; + // Compute which value this mask is pointing at. + SDValue MaskV = Mask[l + i] < NumElts ? V1 : V2; - for (unsigned i = 0, e = NumElts; i != e; ++i) { - SDValue Op = BV->getOperand(i); + // Compute which of the two target values this index should be assigned + // to. This reflects whether the high elements are remaining or the low + // elements are remaining. + SDValue &TargetV = StartIdx < 0 ? Hi : Lo; - // Skip 'undef' values. - unsigned Opcode = Op.getOpcode(); - if (Opcode == ISD::UNDEF) { - std::swap(ExpectedOpcode, NextExpectedOpcode); - continue; + // Either set up this value if we've not encountered it before, or check + // that it remains consistent. + if (!TargetV) + TargetV = MaskV; + else if (TargetV != MaskV) + // This may be a rotation, but it pulls from the inputs in some + // unsupported interleaving. + return SDValue(); } + } - // Early exit if we found an unexpected opcode. - if (Opcode != ExpectedOpcode) - return SDValue(); + // Check that we successfully analyzed the mask, and normalize the results. + assert(Rotation != 0 && "Failed to locate a viable rotation!"); + assert((Lo || Hi) && "Failed to find a rotated input vector!"); + if (!Lo) + Lo = Hi; + else if (!Hi) + Hi = Lo; - SDValue Op0 = Op.getOperand(0); - SDValue Op1 = Op.getOperand(1); + // The actual rotate instruction rotates bytes, so we need to scale the + // rotation based on how many bytes are in the vector lane. + int Scale = 16 / NumLaneElts; - // Try to match the following pattern: - // (BINOP (extract_vector_elt A, i), (extract_vector_elt B, i)) - // Early exit if we cannot match that sequence. - if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT || - Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT || - !isa(Op0.getOperand(1)) || - !isa(Op1.getOperand(1)) || - Op0.getOperand(1) != Op1.getOperand(1)) - return SDValue(); + // SSSE3 targets can use the palignr instruction. + if (Subtarget->hasSSSE3()) { + // Cast the inputs to i8 vector of correct length to match PALIGNR. + MVT AlignVT = MVT::getVectorVT(MVT::i8, 16 * NumLanes); + Lo = DAG.getNode(ISD::BITCAST, DL, AlignVT, Lo); + Hi = DAG.getNode(ISD::BITCAST, DL, AlignVT, Hi); - unsigned I0 = cast(Op0.getOperand(1))->getZExtValue(); - if (I0 != i) - return SDValue(); + return DAG.getNode(ISD::BITCAST, DL, VT, + DAG.getNode(X86ISD::PALIGNR, DL, AlignVT, Hi, Lo, + DAG.getConstant(Rotation * Scale, MVT::i8))); + } - // We found a valid add/sub node. Update the information accordingly. - if (i & 1) - AddFound = true; - else - SubFound = true; + assert(VT.getSizeInBits() == 128 && + "Rotate-based lowering only supports 128-bit lowering!"); + assert(Mask.size() <= 16 && + "Can shuffle at most 16 bytes in a 128-bit vector!"); - // Update InVec0 and InVec1. - if (InVec0.getOpcode() == ISD::UNDEF) - InVec0 = Op0.getOperand(0); - if (InVec1.getOpcode() == ISD::UNDEF) - InVec1 = Op1.getOperand(0); + // Default SSE2 implementation + int LoByteShift = 16 - Rotation * Scale; + int HiByteShift = Rotation * Scale; - // Make sure that operands in input to each add/sub node always - // come from a same pair of vectors. - if (InVec0 != Op0.getOperand(0)) { - if (ExpectedOpcode == ISD::FSUB) - return SDValue(); + // Cast the inputs to v2i64 to match PSLLDQ/PSRLDQ. + Lo = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Lo); + Hi = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Hi); - // FADD is commutable. Try to commute the operands - // and then test again. - std::swap(Op0, Op1); - if (InVec0 != Op0.getOperand(0)) - return SDValue(); + SDValue LoShift = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v2i64, Lo, + DAG.getConstant(LoByteShift, MVT::i8)); + SDValue HiShift = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v2i64, Hi, + DAG.getConstant(HiByteShift, MVT::i8)); + return DAG.getNode(ISD::BITCAST, DL, VT, + DAG.getNode(ISD::OR, DL, MVT::v2i64, LoShift, HiShift)); +} + +/// \brief Compute whether each element of a shuffle is zeroable. +/// +/// A "zeroable" vector shuffle element is one which can be lowered to zero. +/// Either it is an undef element in the shuffle mask, the element of the input +/// referenced is undef, or the element of the input referenced is known to be +/// zero. Many x86 shuffles can zero lanes cheaply and we often want to handle +/// as many lanes with this technique as possible to simplify the remaining +/// shuffle. +static SmallBitVector computeZeroableShuffleElements(ArrayRef Mask, + SDValue V1, SDValue V2) { + SmallBitVector Zeroable(Mask.size(), false); + + while (V1.getOpcode() == ISD::BITCAST) + V1 = V1->getOperand(0); + while (V2.getOpcode() == ISD::BITCAST) + V2 = V2->getOperand(0); + + bool V1IsZero = ISD::isBuildVectorAllZeros(V1.getNode()); + bool V2IsZero = ISD::isBuildVectorAllZeros(V2.getNode()); + + for (int i = 0, Size = Mask.size(); i < Size; ++i) { + int M = Mask[i]; + // Handle the easy cases. + if (M < 0 || (M >= 0 && M < Size && V1IsZero) || (M >= Size && V2IsZero)) { + Zeroable[i] = true; + continue; } - if (InVec1 != Op1.getOperand(0)) - return SDValue(); + // If this is an index into a build_vector node (which has the same number + // of elements), dig out the input value and use it. + SDValue V = M < Size ? V1 : V2; + if (V.getOpcode() != ISD::BUILD_VECTOR || Size != (int)V.getNumOperands()) + continue; - // Update the pair of expected opcodes. - std::swap(ExpectedOpcode, NextExpectedOpcode); + SDValue Input = V.getOperand(M % Size); + // The UNDEF opcode check really should be dead code here, but not quite + // worth asserting on (it isn't invalid, just unexpected). + if (Input.getOpcode() == ISD::UNDEF || X86::isZeroNode(Input)) + Zeroable[i] = true; } - // Don't try to fold this build_vector into an ADDSUB if the inputs are undef. - if (AddFound && SubFound && InVec0.getOpcode() != ISD::UNDEF && - InVec1.getOpcode() != ISD::UNDEF) - return DAG.getNode(X86ISD::ADDSUB, DL, VT, InVec0, InVec1); - - return SDValue(); + return Zeroable; } -static SDValue PerformBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG, - const X86Subtarget *Subtarget) { - SDLoc DL(N); - EVT VT = N->getValueType(0); - unsigned NumElts = VT.getVectorNumElements(); - BuildVectorSDNode *BV = cast(N); - SDValue InVec0, InVec1; +/// \brief Try to emit a bitmask instruction for a shuffle. +/// +/// This handles cases where we can model a blend exactly as a bitmask due to +/// one of the inputs being zeroable. +static SDValue lowerVectorShuffleAsBitMask(SDLoc DL, MVT VT, SDValue V1, + SDValue V2, ArrayRef Mask, + SelectionDAG &DAG) { + MVT EltVT = VT.getScalarType(); + int NumEltBits = EltVT.getSizeInBits(); + MVT IntEltVT = MVT::getIntegerVT(NumEltBits); + SDValue Zero = DAG.getConstant(0, IntEltVT); + SDValue AllOnes = DAG.getConstant(APInt::getAllOnesValue(NumEltBits), IntEltVT); + if (EltVT.isFloatingPoint()) { + Zero = DAG.getNode(ISD::BITCAST, DL, EltVT, Zero); + AllOnes = DAG.getNode(ISD::BITCAST, DL, EltVT, AllOnes); + } + SmallVector VMaskOps(Mask.size(), Zero); + SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2); + SDValue V; + for (int i = 0, Size = Mask.size(); i < Size; ++i) { + if (Zeroable[i]) + continue; + if (Mask[i] % Size != i) + return SDValue(); // Not a blend. + if (!V) + V = Mask[i] < Size ? V1 : V2; + else if (V != (Mask[i] < Size ? V1 : V2)) + return SDValue(); // Can only let one input through the mask. - // Try to match an ADDSUB. - if ((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || - (Subtarget->hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) { - SDValue Value = matchAddSub(BV, DAG, Subtarget); - if (Value.getNode()) - return Value; + VMaskOps[i] = AllOnes; } + if (!V) + return SDValue(); // No non-zeroable elements! - // Try to match horizontal ADD/SUB. - unsigned NumUndefsLO = 0; - unsigned NumUndefsHI = 0; - unsigned Half = NumElts/2; - - // Count the number of UNDEF operands in the build_vector in input. - for (unsigned i = 0, e = Half; i != e; ++i) - if (BV->getOperand(i)->getOpcode() == ISD::UNDEF) - NumUndefsLO++; - - for (unsigned i = Half, e = NumElts; i != e; ++i) - if (BV->getOperand(i)->getOpcode() == ISD::UNDEF) - NumUndefsHI++; - - // Early exit if this is either a build_vector of all UNDEFs or all the - // operands but one are UNDEF. - if (NumUndefsLO + NumUndefsHI + 1 >= NumElts) - return SDValue(); - - if ((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget->hasSSE3()) { - // Try to match an SSE3 float HADD/HSUB. - if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, NumElts, InVec0, InVec1)) - return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1); - - if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, NumElts, InVec0, InVec1)) - return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1); - } else if ((VT == MVT::v4i32 || VT == MVT::v8i16) && Subtarget->hasSSSE3()) { - // Try to match an SSSE3 integer HADD/HSUB. - if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1)) - return DAG.getNode(X86ISD::HADD, DL, VT, InVec0, InVec1); - - if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, NumElts, InVec0, InVec1)) - return DAG.getNode(X86ISD::HSUB, DL, VT, InVec0, InVec1); - } - - if (!Subtarget->hasAVX()) - return SDValue(); + SDValue VMask = DAG.getNode(ISD::BUILD_VECTOR, DL, VT, VMaskOps); + V = DAG.getNode(VT.isFloatingPoint() + ? (unsigned) X86ISD::FAND : (unsigned) ISD::AND, + DL, VT, V, VMask); + return V; +} - if ((VT == MVT::v8f32 || VT == MVT::v4f64)) { - // Try to match an AVX horizontal add/sub of packed single/double - // precision floating point values from 256-bit vectors. - SDValue InVec2, InVec3; - if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, Half, InVec0, InVec1) && - isHorizontalBinOp(BV, ISD::FADD, DAG, Half, NumElts, InVec2, InVec3) && - ((InVec0.getOpcode() == ISD::UNDEF || - InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) && - ((InVec1.getOpcode() == ISD::UNDEF || - InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3)) - return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1); +/// \brief Try to lower a vector shuffle as a bit shift (shifts in zeros). +/// +/// Attempts to match a shuffle mask against the PSLL(W/D/Q/DQ) and +/// PSRL(W/D/Q/DQ) SSE2 and AVX2 logical bit-shift instructions. The function +/// matches elements from one of the input vectors shuffled to the left or +/// right with zeroable elements 'shifted in'. It handles both the strictly +/// bit-wise element shifts and the byte shift across an entire 128-bit double +/// quad word lane. +/// +/// PSHL : (little-endian) left bit shift. +/// [ zz, 0, zz, 2 ] +/// [ -1, 4, zz, -1 ] +/// PSRL : (little-endian) right bit shift. +/// [ 1, zz, 3, zz] +/// [ -1, -1, 7, zz] +/// PSLLDQ : (little-endian) left byte shift +/// [ zz, 0, 1, 2, 3, 4, 5, 6] +/// [ zz, zz, -1, -1, 2, 3, 4, -1] +/// [ zz, zz, zz, zz, zz, zz, -1, 1] +/// PSRLDQ : (little-endian) right byte shift +/// [ 5, 6, 7, zz, zz, zz, zz, zz] +/// [ -1, 5, 6, 7, zz, zz, zz, zz] +/// [ 1, 2, -1, -1, -1, -1, zz, zz] +static SDValue lowerVectorShuffleAsShift(SDLoc DL, MVT VT, SDValue V1, + SDValue V2, ArrayRef Mask, + SelectionDAG &DAG) { + SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2); - if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, Half, InVec0, InVec1) && - isHorizontalBinOp(BV, ISD::FSUB, DAG, Half, NumElts, InVec2, InVec3) && - ((InVec0.getOpcode() == ISD::UNDEF || - InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) && - ((InVec1.getOpcode() == ISD::UNDEF || - InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3)) - return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1); - } else if (VT == MVT::v8i32 || VT == MVT::v16i16) { - // Try to match an AVX2 horizontal add/sub of signed integers. - SDValue InVec2, InVec3; - unsigned X86Opcode; - bool CanFold = true; + int Size = Mask.size(); + assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size"); - if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, Half, InVec0, InVec1) && - isHorizontalBinOp(BV, ISD::ADD, DAG, Half, NumElts, InVec2, InVec3) && - ((InVec0.getOpcode() == ISD::UNDEF || - InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) && - ((InVec1.getOpcode() == ISD::UNDEF || - InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3)) - X86Opcode = X86ISD::HADD; - else if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, Half, InVec0, InVec1) && - isHorizontalBinOp(BV, ISD::SUB, DAG, Half, NumElts, InVec2, InVec3) && - ((InVec0.getOpcode() == ISD::UNDEF || - InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) && - ((InVec1.getOpcode() == ISD::UNDEF || - InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3)) - X86Opcode = X86ISD::HSUB; - else - CanFold = false; + auto CheckZeros = [&](int Shift, int Scale, bool Left) { + for (int i = 0; i < Size; i += Scale) + for (int j = 0; j < Shift; ++j) + if (!Zeroable[i + j + (Left ? 0 : (Scale - Shift))]) + return false; - if (CanFold) { - // Fold this build_vector into a single horizontal add/sub. - // Do this only if the target has AVX2. - if (Subtarget->hasAVX2()) - return DAG.getNode(X86Opcode, DL, VT, InVec0, InVec1); + return true; + }; - // Do not try to expand this build_vector into a pair of horizontal - // add/sub if we can emit a pair of scalar add/sub. - if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half) + auto MatchShift = [&](int Shift, int Scale, bool Left, SDValue V) { + for (int i = 0; i != Size; i += Scale) { + unsigned Pos = Left ? i + Shift : i; + unsigned Low = Left ? i : i + Shift; + unsigned Len = Scale - Shift; + if (!isSequentialOrUndefInRange(Mask, Pos, Len, + Low + (V == V1 ? 0 : Size))) return SDValue(); - - // Convert this build_vector into a pair of horizontal binop followed by - // a concat vector. - bool isUndefLO = NumUndefsLO == Half; - bool isUndefHI = NumUndefsHI == Half; - return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, false, - isUndefLO, isUndefHI); } - } - - if ((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 || - VT == MVT::v16i16) && Subtarget->hasAVX()) { - unsigned X86Opcode; - if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1)) - X86Opcode = X86ISD::HADD; - else if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, NumElts, InVec0, InVec1)) - X86Opcode = X86ISD::HSUB; - else if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, NumElts, InVec0, InVec1)) - X86Opcode = X86ISD::FHADD; - else if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, NumElts, InVec0, InVec1)) - X86Opcode = X86ISD::FHSUB; - else - return SDValue(); - // Don't try to expand this build_vector into a pair of horizontal add/sub - // if we can simply emit a pair of scalar add/sub. - if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half) - return SDValue(); + int ShiftEltBits = VT.getScalarSizeInBits() * Scale; + bool ByteShift = ShiftEltBits > 64; + unsigned OpCode = Left ? (ByteShift ? X86ISD::VSHLDQ : X86ISD::VSHLI) + : (ByteShift ? X86ISD::VSRLDQ : X86ISD::VSRLI); + int ShiftAmt = Shift * VT.getScalarSizeInBits() / (ByteShift ? 8 : 1); - // Convert this build_vector into two horizontal add/sub followed by - // a concat vector. - bool isUndefLO = NumUndefsLO == Half; - bool isUndefHI = NumUndefsHI == Half; - return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, true, - isUndefLO, isUndefHI); - } + // Normalize the scale for byte shifts to still produce an i64 element + // type. + Scale = ByteShift ? Scale / 2 : Scale; - return SDValue(); -} + // We need to round trip through the appropriate type for the shift. + MVT ShiftSVT = MVT::getIntegerVT(VT.getScalarSizeInBits() * Scale); + MVT ShiftVT = MVT::getVectorVT(ShiftSVT, Size / Scale); + assert(DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) && + "Illegal integer vector type"); + V = DAG.getNode(ISD::BITCAST, DL, ShiftVT, V); -SDValue -X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { - SDLoc dl(Op); + V = DAG.getNode(OpCode, DL, ShiftVT, V, DAG.getConstant(ShiftAmt, MVT::i8)); + return DAG.getNode(ISD::BITCAST, DL, VT, V); + }; - MVT VT = Op.getSimpleValueType(); - MVT ExtVT = VT.getVectorElementType(); - unsigned NumElems = Op.getNumOperands(); + // SSE/AVX supports logical shifts up to 64-bit integers - so we can just + // keep doubling the size of the integer elements up to that. We can + // then shift the elements of the integer vector by whole multiples of + // their width within the elements of the larger integer vector. Test each + // multiple to see if we can find a match with the moved element indices + // and that the shifted in elements are all zeroable. + for (int Scale = 2; Scale * VT.getScalarSizeInBits() <= 128; Scale *= 2) + for (int Shift = 1; Shift != Scale; ++Shift) + for (bool Left : {true, false}) + if (CheckZeros(Shift, Scale, Left)) + for (SDValue V : {V1, V2}) + if (SDValue Match = MatchShift(Shift, Scale, Left, V)) + return Match; - // Generate vectors for predicate vectors. - if (VT.getScalarType() == MVT::i1 && Subtarget->hasAVX512()) - return LowerBUILD_VECTORvXi1(Op, DAG); + // no match + return SDValue(); +} - // Vectors containing all zeros can be matched by pxor and xorps later - if (ISD::isBuildVectorAllZeros(Op.getNode())) { - // Canonicalize this to <4 x i32> to 1) ensure the zero vectors are CSE'd - // and 2) ensure that i64 scalars are eliminated on x86-32 hosts. - if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) - return Op; +/// \brief Lower a vector shuffle as a zero or any extension. +/// +/// Given a specific number of elements, element bit width, and extension +/// stride, produce either a zero or any extension based on the available +/// features of the subtarget. +static SDValue lowerVectorShuffleAsSpecificZeroOrAnyExtend( + SDLoc DL, MVT VT, int Scale, bool AnyExt, SDValue InputV, + const X86Subtarget *Subtarget, SelectionDAG &DAG) { + assert(Scale > 1 && "Need a scale to extend."); + int NumElements = VT.getVectorNumElements(); + int EltBits = VT.getScalarSizeInBits(); + assert((EltBits == 8 || EltBits == 16 || EltBits == 32) && + "Only 8, 16, and 32 bit elements can be extended."); + assert(Scale * EltBits <= 64 && "Cannot zero extend past 64 bits."); - return getZeroVector(VT, Subtarget, DAG, dl); + // Found a valid zext mask! Try various lowering strategies based on the + // input type and available ISA extensions. + if (Subtarget->hasSSE41()) { + MVT ExtVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Scale), + NumElements / Scale); + return DAG.getNode(ISD::BITCAST, DL, VT, + DAG.getNode(X86ISD::VZEXT, DL, ExtVT, InputV)); } - // Vectors containing all ones can be matched by pcmpeqd on 128-bit width - // vectors or broken into v4i32 operations on 256-bit vectors. AVX2 can use - // vpcmpeqd on 256-bit vectors. - if (Subtarget->hasSSE2() && ISD::isBuildVectorAllOnes(Op.getNode())) { - if (VT == MVT::v4i32 || (VT == MVT::v8i32 && Subtarget->hasInt256())) - return Op; - - if (!VT.is512BitVector()) - return getOnesVector(VT, Subtarget->hasInt256(), DAG, dl); + // For any extends we can cheat for larger element sizes and use shuffle + // instructions that can fold with a load and/or copy. + if (AnyExt && EltBits == 32) { + int PSHUFDMask[4] = {0, -1, 1, -1}; + return DAG.getNode( + ISD::BITCAST, DL, VT, + DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, + DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, InputV), + getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG))); } - - SDValue Broadcast = LowerVectorBroadcast(Op, Subtarget, DAG); - if (Broadcast.getNode()) - return Broadcast; - - unsigned EVTBits = ExtVT.getSizeInBits(); - - unsigned NumZero = 0; - unsigned NumNonZero = 0; - unsigned NonZeros = 0; - bool IsAllConstants = true; - SmallSet Values; - for (unsigned i = 0; i < NumElems; ++i) { - SDValue Elt = Op.getOperand(i); - if (Elt.getOpcode() == ISD::UNDEF) - continue; - Values.insert(Elt); - if (Elt.getOpcode() != ISD::Constant && - Elt.getOpcode() != ISD::ConstantFP) - IsAllConstants = false; - if (X86::isZeroNode(Elt)) - NumZero++; - else { - NonZeros |= (1 << i); - NumNonZero++; - } + if (AnyExt && EltBits == 16 && Scale > 2) { + int PSHUFDMask[4] = {0, -1, 0, -1}; + InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, + DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, InputV), + getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG)); + int PSHUFHWMask[4] = {1, -1, -1, -1}; + return DAG.getNode( + ISD::BITCAST, DL, VT, + DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16, + DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, InputV), + getV4X86ShuffleImm8ForMask(PSHUFHWMask, DAG))); } - // All undef vector. Return an UNDEF. All zero vectors were handled above. - if (NumNonZero == 0) - return DAG.getUNDEF(VT); - - // Special case for single non-zero, non-undef, element. - if (NumNonZero == 1) { - unsigned Idx = countTrailingZeros(NonZeros); - SDValue Item = Op.getOperand(Idx); + // If this would require more than 2 unpack instructions to expand, use + // pshufb when available. We can only use more than 2 unpack instructions + // when zero extending i8 elements which also makes it easier to use pshufb. + if (Scale > 4 && EltBits == 8 && Subtarget->hasSSSE3()) { + assert(NumElements == 16 && "Unexpected byte vector width!"); + SDValue PSHUFBMask[16]; + for (int i = 0; i < 16; ++i) + PSHUFBMask[i] = + DAG.getConstant((i % Scale == 0) ? i / Scale : 0x80, MVT::i8); + InputV = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, InputV); + return DAG.getNode(ISD::BITCAST, DL, VT, + DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV, + DAG.getNode(ISD::BUILD_VECTOR, DL, + MVT::v16i8, PSHUFBMask))); + } - // If this is an insertion of an i64 value on x86-32, and if the top bits of - // the value are obviously zero, truncate the value to i32 and do the - // insertion that way. Only do this if the value is non-constant or if the - // value is a constant being inserted into element 0. It is cheaper to do - // a constant pool load than it is to do a movd + shuffle. - if (ExtVT == MVT::i64 && !Subtarget->is64Bit() && - (!IsAllConstants || Idx == 0)) { - if (DAG.MaskedValueIsZero(Item, APInt::getBitsSet(64, 32, 64))) { - // Handle SSE only. - assert(VT == MVT::v2i64 && "Expected an SSE value type!"); - EVT VecVT = MVT::v4i32; - unsigned VecElts = 4; + // Otherwise emit a sequence of unpacks. + do { + MVT InputVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits), NumElements); + SDValue Ext = AnyExt ? DAG.getUNDEF(InputVT) + : getZeroVector(InputVT, Subtarget, DAG, DL); + InputV = DAG.getNode(ISD::BITCAST, DL, InputVT, InputV); + InputV = DAG.getNode(X86ISD::UNPCKL, DL, InputVT, InputV, Ext); + Scale /= 2; + EltBits *= 2; + NumElements /= 2; + } while (Scale > 1); + return DAG.getNode(ISD::BITCAST, DL, VT, InputV); +} - // Truncate the value (which may itself be a constant) to i32, and - // convert it to a vector with movd (S2V+shuffle to zero extend). - Item = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Item); - Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Item); +/// \brief Try to lower a vector shuffle as a zero extension on any microarch. +/// +/// This routine will try to do everything in its power to cleverly lower +/// a shuffle which happens to match the pattern of a zero extend. It doesn't +/// check for the profitability of this lowering, it tries to aggressively +/// match this pattern. It will use all of the micro-architectural details it +/// can to emit an efficient lowering. It handles both blends with all-zero +/// inputs to explicitly zero-extend and undef-lanes (sometimes undef due to +/// masking out later). +/// +/// The reason we have dedicated lowering for zext-style shuffles is that they +/// are both incredibly common and often quite performance sensitive. +static SDValue lowerVectorShuffleAsZeroOrAnyExtend( + SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef Mask, + const X86Subtarget *Subtarget, SelectionDAG &DAG) { + SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2); - // If using the new shuffle lowering, just directly insert this. - if (ExperimentalVectorShuffleLowering) - return DAG.getNode( - ISD::BITCAST, dl, VT, - getShuffleVectorZeroOrUndef(Item, Idx * 2, true, Subtarget, DAG)); - - Item = getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG); - - // Now we have our 32-bit value zero extended in the low element of - // a vector. If Idx != 0, swizzle it into place. - if (Idx != 0) { - SmallVector Mask; - Mask.push_back(Idx); - for (unsigned i = 1; i != VecElts; ++i) - Mask.push_back(i); - Item = DAG.getVectorShuffle(VecVT, dl, Item, DAG.getUNDEF(VecVT), - &Mask[0]); - } - return DAG.getNode(ISD::BITCAST, dl, VT, Item); - } - } + int Bits = VT.getSizeInBits(); + int NumElements = VT.getVectorNumElements(); + assert(VT.getScalarSizeInBits() <= 32 && + "Exceeds 32-bit integer zero extension limit"); + assert((int)Mask.size() == NumElements && "Unexpected shuffle mask size"); - // If we have a constant or non-constant insertion into the low element of - // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into - // the rest of the elements. This will be matched as movd/movq/movss/movsd - // depending on what the source datatype is. - if (Idx == 0) { - if (NumZero == 0) - return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); + // Define a helper function to check a particular ext-scale and lower to it if + // valid. + auto Lower = [&](int Scale) -> SDValue { + SDValue InputV; + bool AnyExt = true; + for (int i = 0; i < NumElements; ++i) { + if (Mask[i] == -1) + continue; // Valid anywhere but doesn't tell us anything. + if (i % Scale != 0) { + // Each of the extended elements need to be zeroable. + if (!Zeroable[i]) + return SDValue(); - if (ExtVT == MVT::i32 || ExtVT == MVT::f32 || ExtVT == MVT::f64 || - (ExtVT == MVT::i64 && Subtarget->is64Bit())) { - if (VT.is256BitVector() || VT.is512BitVector()) { - SDValue ZeroVec = getZeroVector(VT, Subtarget, DAG, dl); - return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, ZeroVec, - Item, DAG.getIntPtrConstant(0)); - } - assert(VT.is128BitVector() && "Expected an SSE value type!"); - Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); - // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector. - return getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG); + // We no longer are in the anyext case. + AnyExt = false; + continue; } - if (ExtVT == MVT::i16 || ExtVT == MVT::i8) { - Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item); - Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Item); - if (VT.is256BitVector()) { - SDValue ZeroVec = getZeroVector(MVT::v8i32, Subtarget, DAG, dl); - Item = Insert128BitVector(ZeroVec, Item, 0, DAG, dl); - } else { - assert(VT.is128BitVector() && "Expected an SSE value type!"); - Item = getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG); - } - return DAG.getNode(ISD::BITCAST, dl, VT, Item); - } - } + // Each of the base elements needs to be consecutive indices into the + // same input vector. + SDValue V = Mask[i] < NumElements ? V1 : V2; + if (!InputV) + InputV = V; + else if (InputV != V) + return SDValue(); // Flip-flopping inputs. - // Is it a vector logical left shift? - if (NumElems == 2 && Idx == 1 && - X86::isZeroNode(Op.getOperand(0)) && - !X86::isZeroNode(Op.getOperand(1))) { - unsigned NumBits = VT.getSizeInBits(); - return getVShift(true, VT, - DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, - VT, Op.getOperand(1)), - NumBits/2, DAG, *this, dl); + if (Mask[i] % NumElements != i / Scale) + return SDValue(); // Non-consecutive strided elements. } - if (IsAllConstants) // Otherwise, it's better to do a constpool load. + // If we fail to find an input, we have a zero-shuffle which should always + // have already been handled. + // FIXME: Maybe handle this here in case during blending we end up with one? + if (!InputV) return SDValue(); - // Otherwise, if this is a vector with i32 or f32 elements, and the element - // is a non-constant being inserted into an element other than the low one, - // we can't use a constant pool load. Instead, use SCALAR_TO_VECTOR (aka - // movd/movss) to move this into the low element, then shuffle it into - // place. - if (EVTBits == 32) { - Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); + return lowerVectorShuffleAsSpecificZeroOrAnyExtend( + DL, VT, Scale, AnyExt, InputV, Subtarget, DAG); + }; - // If using the new shuffle lowering, just directly insert this. - if (ExperimentalVectorShuffleLowering) - return getShuffleVectorZeroOrUndef(Item, Idx, NumZero > 0, Subtarget, DAG); + // The widest scale possible for extending is to a 64-bit integer. + assert(Bits % 64 == 0 && + "The number of bits in a vector must be divisible by 64 on x86!"); + int NumExtElements = Bits / 64; - // Turn it into a shuffle of zero and zero-extended scalar to vector. - Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, Subtarget, DAG); - SmallVector MaskVec; - for (unsigned i = 0; i != NumElems; ++i) - MaskVec.push_back(i == Idx ? 0 : 1); - return DAG.getVectorShuffle(VT, dl, Item, DAG.getUNDEF(VT), &MaskVec[0]); - } + // Each iteration, try extending the elements half as much, but into twice as + // many elements. + for (; NumExtElements < NumElements; NumExtElements *= 2) { + assert(NumElements % NumExtElements == 0 && + "The input vector size must be divisible by the extended size."); + if (SDValue V = Lower(NumElements / NumExtElements)) + return V; } - // Splat is obviously ok. Let legalizer expand it to a shuffle. - if (Values.size() == 1) { - if (EVTBits == 32) { - // Instead of a shuffle like this: - // shuffle (scalar_to_vector (load (ptr + 4))), undef, <0, 0, 0, 0> - // Check if it's possible to issue this instead. - // shuffle (vload ptr)), undef, <1, 1, 1, 1> - unsigned Idx = countTrailingZeros(NonZeros); - SDValue Item = Op.getOperand(Idx); - if (Op.getNode()->isOnlyUserOf(Item.getNode())) - return LowerAsSplatVectorLoad(Item, VT, dl, DAG); - } + // General extends failed, but 128-bit vectors may be able to use MOVQ. + if (Bits != 128) return SDValue(); - } - // A vector full of immediates; various special cases are already - // handled, so this is best done with a single constant-pool load. - if (IsAllConstants) + // Returns one of the source operands if the shuffle can be reduced to a + // MOVQ, copying the lower 64-bits and zero-extending to the upper 64-bits. + auto CanZExtLowHalf = [&]() { + for (int i = NumElements / 2; i != NumElements; ++i) + if (!Zeroable[i]) + return SDValue(); + if (isSequentialOrUndefInRange(Mask, 0, NumElements / 2, 0)) + return V1; + if (isSequentialOrUndefInRange(Mask, 0, NumElements / 2, NumElements)) + return V2; return SDValue(); + }; - // For AVX-length vectors, see if we can use a vector load to get all of the - // elements, otherwise build the individual 128-bit pieces and use - // shuffles to put them in place. - if (VT.is256BitVector() || VT.is512BitVector()) { - SmallVector V(Op->op_begin(), Op->op_begin() + NumElems); - - // Check for a build vector of consecutive loads. - if (SDValue LD = EltsFromConsecutiveLoads(VT, V, dl, DAG, false)) - return LD; - - EVT HVT = EVT::getVectorVT(*DAG.getContext(), ExtVT, NumElems/2); + if (SDValue V = CanZExtLowHalf()) { + V = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, V); + V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V); + return DAG.getNode(ISD::BITCAST, DL, VT, V); + } - // Build both the lower and upper subvector. - SDValue Lower = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT, - makeArrayRef(&V[0], NumElems/2)); - SDValue Upper = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT, - makeArrayRef(&V[NumElems / 2], NumElems/2)); + // No viable ext lowering found. + return SDValue(); +} - // Recreate the wider vector with the lower and upper part. - if (VT.is256BitVector()) - return Concat128BitVectors(Lower, Upper, VT, NumElems, DAG, dl); - return Concat256BitVectors(Lower, Upper, VT, NumElems, DAG, dl); - } - - // Let legalizer expand 2-wide build_vectors. - if (EVTBits == 64) { - if (NumNonZero == 1) { - // One half is zero or undef. - unsigned Idx = countTrailingZeros(NonZeros); - SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, - Op.getOperand(Idx)); - return getShuffleVectorZeroOrUndef(V2, Idx, true, Subtarget, DAG); - } +/// \brief Try to get a scalar value for a specific element of a vector. +/// +/// Looks through BUILD_VECTOR and SCALAR_TO_VECTOR nodes to find a scalar. +static SDValue getScalarValueForVectorElement(SDValue V, int Idx, + SelectionDAG &DAG) { + MVT VT = V.getSimpleValueType(); + MVT EltVT = VT.getVectorElementType(); + while (V.getOpcode() == ISD::BITCAST) + V = V.getOperand(0); + // If the bitcasts shift the element size, we can't extract an equivalent + // element from it. + MVT NewVT = V.getSimpleValueType(); + if (!NewVT.isVector() || NewVT.getScalarSizeInBits() != VT.getScalarSizeInBits()) return SDValue(); - } - - // If element VT is < 32 bits, convert it to inserts into a zero vector. - if (EVTBits == 8 && NumElems == 16) { - SDValue V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG, - Subtarget, *this); - if (V.getNode()) return V; - } - - if (EVTBits == 16 && NumElems == 8) { - SDValue V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG, - Subtarget, *this); - if (V.getNode()) return V; - } - - // If element VT is == 32 bits and has 4 elems, try to generate an INSERTPS - if (EVTBits == 32 && NumElems == 4) { - SDValue V = LowerBuildVectorv4x32(Op, DAG, Subtarget, *this); - if (V.getNode()) - return V; - } - - // If element VT is == 32 bits, turn it into a number of shuffles. - SmallVector V(NumElems); - if (NumElems == 4 && NumZero > 0) { - for (unsigned i = 0; i < 4; ++i) { - bool isZero = !(NonZeros & (1 << i)); - if (isZero) - V[i] = getZeroVector(VT, Subtarget, DAG, dl); - else - V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); - } - - for (unsigned i = 0; i < 2; ++i) { - switch ((NonZeros & (0x3 << i*2)) >> (i*2)) { - default: break; - case 0: - V[i] = V[i*2]; // Must be a zero vector. - break; - case 1: - V[i] = getMOVL(DAG, dl, VT, V[i*2+1], V[i*2]); - break; - case 2: - V[i] = getMOVL(DAG, dl, VT, V[i*2], V[i*2+1]); - break; - case 3: - V[i] = getUnpackl(DAG, dl, VT, V[i*2], V[i*2+1]); - break; - } - } - - bool Reverse1 = (NonZeros & 0x3) == 2; - bool Reverse2 = ((NonZeros & (0x3 << 2)) >> 2) == 2; - int MaskVec[] = { - Reverse1 ? 1 : 0, - Reverse1 ? 0 : 1, - static_cast(Reverse2 ? NumElems+1 : NumElems), - static_cast(Reverse2 ? NumElems : NumElems+1) - }; - return DAG.getVectorShuffle(VT, dl, V[0], V[1], &MaskVec[0]); - } - if (Values.size() > 1 && VT.is128BitVector()) { - // Check for a build vector of consecutive loads. - for (unsigned i = 0; i < NumElems; ++i) - V[i] = Op.getOperand(i); + if (V.getOpcode() == ISD::BUILD_VECTOR || + (Idx == 0 && V.getOpcode() == ISD::SCALAR_TO_VECTOR)) + return DAG.getNode(ISD::BITCAST, SDLoc(V), EltVT, V.getOperand(Idx)); - // Check for elements which are consecutive loads. - SDValue LD = EltsFromConsecutiveLoads(VT, V, dl, DAG, false); - if (LD.getNode()) - return LD; + return SDValue(); +} - // Check for a build vector from mostly shuffle plus few inserting. - SDValue Sh = buildFromShuffleMostly(Op, DAG); - if (Sh.getNode()) - return Sh; +/// \brief Helper to test for a load that can be folded with x86 shuffles. +/// +/// This is particularly important because the set of instructions varies +/// significantly based on whether the operand is a load or not. +static bool isShuffleFoldableLoad(SDValue V) { + while (V.getOpcode() == ISD::BITCAST) + V = V.getOperand(0); - // For SSE 4.1, use insertps to put the high elements into the low element. - if (Subtarget->hasSSE41()) { - SDValue Result; - if (Op.getOperand(0).getOpcode() != ISD::UNDEF) - Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0)); - else - Result = DAG.getUNDEF(VT); + return ISD::isNON_EXTLoad(V.getNode()); +} - for (unsigned i = 1; i < NumElems; ++i) { - if (Op.getOperand(i).getOpcode() == ISD::UNDEF) continue; - Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result, - Op.getOperand(i), DAG.getIntPtrConstant(i)); - } - return Result; - } +/// \brief Try to lower insertion of a single element into a zero vector. +/// +/// This is a common pattern that we have especially efficient patterns to lower +/// across all subtarget feature sets. +static SDValue lowerVectorShuffleAsElementInsertion( + MVT VT, SDLoc DL, SDValue V1, SDValue V2, ArrayRef Mask, + const X86Subtarget *Subtarget, SelectionDAG &DAG) { + SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2); + MVT ExtVT = VT; + MVT EltVT = VT.getVectorElementType(); - // Otherwise, expand into a number of unpckl*, start by extending each of - // our (non-undef) elements to the full vector width with the element in the - // bottom slot of the vector (which generates no code for SSE). - for (unsigned i = 0; i < NumElems; ++i) { - if (Op.getOperand(i).getOpcode() != ISD::UNDEF) - V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); - else - V[i] = DAG.getUNDEF(VT); + int V2Index = std::find_if(Mask.begin(), Mask.end(), + [&Mask](int M) { return M >= (int)Mask.size(); }) - + Mask.begin(); + bool IsV1Zeroable = true; + for (int i = 0, Size = Mask.size(); i < Size; ++i) + if (i != V2Index && !Zeroable[i]) { + IsV1Zeroable = false; + break; } - // Next, we iteratively mix elements, e.g. for v4f32: - // Step 1: unpcklps 0, 2 ==> X: - // : unpcklps 1, 3 ==> Y: - // Step 2: unpcklps X, Y ==> <3, 2, 1, 0> - unsigned EltStride = NumElems >> 1; - while (EltStride != 0) { - for (unsigned i = 0; i < EltStride; ++i) { - // If V[i+EltStride] is undef and this is the first round of mixing, - // then it is safe to just drop this shuffle: V[i] is already in the - // right place, the one element (since it's the first round) being - // inserted as undef can be dropped. This isn't safe for successive - // rounds because they will permute elements within both vectors. - if (V[i+EltStride].getOpcode() == ISD::UNDEF && - EltStride == NumElems/2) - continue; + // Check for a single input from a SCALAR_TO_VECTOR node. + // FIXME: All of this should be canonicalized into INSERT_VECTOR_ELT and + // all the smarts here sunk into that routine. However, the current + // lowering of BUILD_VECTOR makes that nearly impossible until the old + // vector shuffle lowering is dead. + if (SDValue V2S = getScalarValueForVectorElement( + V2, Mask[V2Index] - Mask.size(), DAG)) { + // We need to zext the scalar if it is smaller than an i32. + V2S = DAG.getNode(ISD::BITCAST, DL, EltVT, V2S); + if (EltVT == MVT::i8 || EltVT == MVT::i16) { + // Using zext to expand a narrow element won't work for non-zero + // insertions. + if (!IsV1Zeroable) + return SDValue(); - V[i] = getUnpackl(DAG, dl, VT, V[i], V[i + EltStride]); - } - EltStride >>= 1; + // Zero-extend directly to i32. + ExtVT = MVT::v4i32; + V2S = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, V2S); } - return V[0]; + V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S); + } else if (Mask[V2Index] != (int)Mask.size() || EltVT == MVT::i8 || + EltVT == MVT::i16) { + // Either not inserting from the low element of the input or the input + // element size is too small to use VZEXT_MOVL to clear the high bits. + return SDValue(); } - return SDValue(); -} - -// LowerAVXCONCAT_VECTORS - 256-bit AVX can use the vinsertf128 instruction -// to create 256-bit vectors from two other 128-bit ones. -static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) { - SDLoc dl(Op); - MVT ResVT = Op.getSimpleValueType(); - - assert((ResVT.is256BitVector() || - ResVT.is512BitVector()) && "Value type must be 256-/512-bit wide"); - SDValue V1 = Op.getOperand(0); - SDValue V2 = Op.getOperand(1); - unsigned NumElems = ResVT.getVectorNumElements(); - if(ResVT.is256BitVector()) - return Concat128BitVectors(V1, V2, ResVT, NumElems, DAG, dl); + if (!IsV1Zeroable) { + // If V1 can't be treated as a zero vector we have fewer options to lower + // this. We can't support integer vectors or non-zero targets cheaply, and + // the V1 elements can't be permuted in any way. + assert(VT == ExtVT && "Cannot change extended type when non-zeroable!"); + if (!VT.isFloatingPoint() || V2Index != 0) + return SDValue(); + SmallVector V1Mask(Mask.begin(), Mask.end()); + V1Mask[V2Index] = -1; + if (!isNoopShuffleMask(V1Mask)) + return SDValue(); + // This is essentially a special case blend operation, but if we have + // general purpose blend operations, they are always faster. Bail and let + // the rest of the lowering handle these as blends. + if (Subtarget->hasSSE41()) + return SDValue(); - if (Op.getNumOperands() == 4) { - MVT HalfVT = MVT::getVectorVT(ResVT.getScalarType(), - ResVT.getVectorNumElements()/2); - SDValue V3 = Op.getOperand(2); - SDValue V4 = Op.getOperand(3); - return Concat256BitVectors(Concat128BitVectors(V1, V2, HalfVT, NumElems/2, DAG, dl), - Concat128BitVectors(V3, V4, HalfVT, NumElems/2, DAG, dl), ResVT, NumElems, DAG, dl); + // Otherwise, use MOVSD or MOVSS. + assert((EltVT == MVT::f32 || EltVT == MVT::f64) && + "Only two types of floating point element types to handle!"); + return DAG.getNode(EltVT == MVT::f32 ? X86ISD::MOVSS : X86ISD::MOVSD, DL, + ExtVT, V1, V2); } - return Concat256BitVectors(V1, V2, ResVT, NumElems, DAG, dl); -} -static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) { - MVT LLVM_ATTRIBUTE_UNUSED VT = Op.getSimpleValueType(); - assert((VT.is256BitVector() && Op.getNumOperands() == 2) || - (VT.is512BitVector() && (Op.getNumOperands() == 2 || - Op.getNumOperands() == 4))); + // This lowering only works for the low element with floating point vectors. + if (VT.isFloatingPoint() && V2Index != 0) + return SDValue(); - // AVX can use the vinsertf128 instruction to create 256-bit vectors - // from two other 128-bit ones. + V2 = DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2); + if (ExtVT != VT) + V2 = DAG.getNode(ISD::BITCAST, DL, VT, V2); - // 512-bit vector may contain 2 256-bit vectors or 4 128-bit vectors - return LowerAVXCONCAT_VECTORS(Op, DAG); + if (V2Index != 0) { + // If we have 4 or fewer lanes we can cheaply shuffle the element into + // the desired position. Otherwise it is more efficient to do a vector + // shift left. We know that we can do a vector shift left because all + // the inputs are zero. + if (VT.isFloatingPoint() || VT.getVectorNumElements() <= 4) { + SmallVector V2Shuffle(Mask.size(), 1); + V2Shuffle[V2Index] = 0; + V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Shuffle); + } else { + V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, V2); + V2 = DAG.getNode( + X86ISD::VSHLDQ, DL, MVT::v2i64, V2, + DAG.getConstant( + V2Index * EltVT.getSizeInBits()/8, + DAG.getTargetLoweringInfo().getScalarShiftAmountTy(MVT::v2i64))); + V2 = DAG.getNode(ISD::BITCAST, DL, VT, V2); + } + } + return V2; } - -//===----------------------------------------------------------------------===// -// Vector shuffle lowering -// -// This is an experimental code path for lowering vector shuffles on x86. It is -// designed to handle arbitrary vector shuffles and blends, gracefully -// degrading performance as necessary. It works hard to recognize idiomatic -// shuffles and lower them to optimal instruction patterns without leaving -// a framework that allows reasonably efficient handling of all vector shuffle -// patterns. -//===----------------------------------------------------------------------===// - -/// \brief Tiny helper function to identify a no-op mask. +/// \brief Try to lower broadcast of a single element. /// -/// This is a somewhat boring predicate function. It checks whether the mask -/// array input, which is assumed to be a single-input shuffle mask of the kind -/// used by the X86 shuffle instructions (not a fully general -/// ShuffleVectorSDNode mask) requires any shuffles to occur. Both undef and an -/// in-place shuffle are 'no-op's. -static bool isNoopShuffleMask(ArrayRef Mask) { - for (int i = 0, Size = Mask.size(); i < Size; ++i) - if (Mask[i] != -1 && Mask[i] != i) - return false; - return true; -} +/// For convenience, this code also bundles all of the subtarget feature set +/// filtering. While a little annoying to re-dispatch on type here, there isn't +/// a convenient way to factor it out. +static SDValue lowerVectorShuffleAsBroadcast(MVT VT, SDLoc DL, SDValue V, + ArrayRef Mask, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + if (!Subtarget->hasAVX()) + return SDValue(); + if (VT.isInteger() && !Subtarget->hasAVX2()) + return SDValue(); -/// \brief Helper function to classify a mask as a single-input mask. -/// -/// This isn't a generic single-input test because in the vector shuffle -/// lowering we canonicalize single inputs to be the first input operand. This -/// means we can more quickly test for a single input by only checking whether -/// an input from the second operand exists. We also assume that the size of -/// mask corresponds to the size of the input vectors which isn't true in the -/// fully general case. -static bool isSingleInputShuffleMask(ArrayRef Mask) { + // Check that the mask is a broadcast. + int BroadcastIdx = -1; for (int M : Mask) - if (M >= (int)Mask.size()) - return false; - return true; -} + if (M >= 0 && BroadcastIdx == -1) + BroadcastIdx = M; + else if (M >= 0 && M != BroadcastIdx) + return SDValue(); -/// \brief Test whether there are elements crossing 128-bit lanes in this -/// shuffle mask. -/// -/// X86 divides up its shuffles into in-lane and cross-lane shuffle operations -/// and we routinely test for these. -static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef Mask) { - int LaneSize = 128 / VT.getScalarSizeInBits(); - int Size = Mask.size(); - for (int i = 0; i < Size; ++i) - if (Mask[i] >= 0 && (Mask[i] % Size) / LaneSize != i / LaneSize) - return true; - return false; -} + assert(BroadcastIdx < (int)Mask.size() && "We only expect to be called with " + "a sorted mask where the broadcast " + "comes from V1."); -/// \brief Test whether a shuffle mask is equivalent within each 128-bit lane. -/// -/// This checks a shuffle mask to see if it is performing the same -/// 128-bit lane-relative shuffle in each 128-bit lane. This trivially implies -/// that it is also not lane-crossing. It may however involve a blend from the -/// same lane of a second vector. -/// -/// The specific repeated shuffle mask is populated in \p RepeatedMask, as it is -/// non-trivial to compute in the face of undef lanes. The representation is -/// *not* suitable for use with existing 128-bit shuffles as it will contain -/// entries from both V1 and V2 inputs to the wider mask. -static bool -is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef Mask, - SmallVectorImpl &RepeatedMask) { - int LaneSize = 128 / VT.getScalarSizeInBits(); - RepeatedMask.resize(LaneSize, -1); - int Size = Mask.size(); - for (int i = 0; i < Size; ++i) { - if (Mask[i] < 0) + // Go up the chain of (vector) values to try and find a scalar load that + // we can combine with the broadcast. + for (;;) { + switch (V.getOpcode()) { + case ISD::CONCAT_VECTORS: { + int OperandSize = Mask.size() / V.getNumOperands(); + V = V.getOperand(BroadcastIdx / OperandSize); + BroadcastIdx %= OperandSize; continue; - if ((Mask[i] % Size) / LaneSize != i / LaneSize) - // This entry crosses lanes, so there is no way to model this shuffle. - return false; + } - // Ok, handle the in-lane shuffles by detecting if and when they repeat. - if (RepeatedMask[i % LaneSize] == -1) - // This is the first non-undef entry in this slot of a 128-bit lane. - RepeatedMask[i % LaneSize] = - Mask[i] < Size ? Mask[i] % LaneSize : Mask[i] % LaneSize + Size; - else if (RepeatedMask[i % LaneSize] + (i / LaneSize) * LaneSize != Mask[i]) - // Found a mismatch with the repeated mask. - return false; - } - return true; -} + case ISD::INSERT_SUBVECTOR: { + SDValue VOuter = V.getOperand(0), VInner = V.getOperand(1); + auto ConstantIdx = dyn_cast(V.getOperand(2)); + if (!ConstantIdx) + break; -/// \brief Base case helper for testing a single mask element. -static bool isShuffleEquivalentImpl(SDValue V1, SDValue V2, - BuildVectorSDNode *BV1, - BuildVectorSDNode *BV2, ArrayRef Mask, - int i, int Arg) { - int Size = Mask.size(); - if (Mask[i] != -1 && Mask[i] != Arg) { - auto *MaskBV = Mask[i] < Size ? BV1 : BV2; - auto *ArgsBV = Arg < Size ? BV1 : BV2; - if (!MaskBV || !ArgsBV || - MaskBV->getOperand(Mask[i] % Size) != ArgsBV->getOperand(Arg % Size)) - return false; + int BeginIdx = (int)ConstantIdx->getZExtValue(); + int EndIdx = + BeginIdx + (int)VInner.getValueType().getVectorNumElements(); + if (BroadcastIdx >= BeginIdx && BroadcastIdx < EndIdx) { + BroadcastIdx -= BeginIdx; + V = VInner; + } else { + V = VOuter; + } + continue; + } + } + break; } - return true; -} -/// \brief Recursive helper to peel off and test each mask element. -template -static bool isShuffleEquivalentImpl(SDValue V1, SDValue V2, - BuildVectorSDNode *BV1, - BuildVectorSDNode *BV2, ArrayRef Mask, - int i, int Arg, Ts... Args) { - if (!isShuffleEquivalentImpl(V1, V2, BV1, BV2, Mask, i, Arg)) - return false; + // Check if this is a broadcast of a scalar. We special case lowering + // for scalars so that we can more effectively fold with loads. + if (V.getOpcode() == ISD::BUILD_VECTOR || + (V.getOpcode() == ISD::SCALAR_TO_VECTOR && BroadcastIdx == 0)) { + V = V.getOperand(BroadcastIdx); - return isShuffleEquivalentImpl(V1, V2, BV1, BV2, Mask, i + 1, Args...); + // If the scalar isn't a load we can't broadcast from it in AVX1, only with + // AVX2. + if (!Subtarget->hasAVX2() && !isShuffleFoldableLoad(V)) + return SDValue(); + } else if (BroadcastIdx != 0 || !Subtarget->hasAVX2()) { + // We can't broadcast from a vector register w/o AVX2, and we can only + // broadcast from the zero-element of a vector register. + return SDValue(); + } + + return DAG.getNode(X86ISD::VBROADCAST, DL, VT, V); } -/// \brief Checks whether a shuffle mask is equivalent to an explicit list of -/// arguments. -/// -/// This is a fast way to test a shuffle mask against a fixed pattern: -/// -/// if (isShuffleEquivalent(Mask, 3, 2, 1, 0)) { ... } -/// -/// It returns true if the mask is exactly as wide as the argument list, and -/// each element of the mask is either -1 (signifying undef) or the value given -/// in the argument. -template -static bool isShuffleEquivalent(SDValue V1, SDValue V2, ArrayRef Mask, - Ts... Args) { - if (Mask.size() != sizeof...(Args)) - return false; +// Check for whether we can use INSERTPS to perform the shuffle. We only use +// INSERTPS when the V1 elements are already in the correct locations +// because otherwise we can just always use two SHUFPS instructions which +// are much smaller to encode than a SHUFPS and an INSERTPS. We can also +// perform INSERTPS if a single V1 element is out of place and all V2 +// elements are zeroable. +static SDValue lowerVectorShuffleAsInsertPS(SDValue Op, SDValue V1, SDValue V2, + ArrayRef Mask, + SelectionDAG &DAG) { + assert(Op.getSimpleValueType() == MVT::v4f32 && "Bad shuffle type!"); + assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); + assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"); - // If the values are build vectors, we can look through them to find - // equivalent inputs that make the shuffles equivalent. - auto *BV1 = dyn_cast(V1); - auto *BV2 = dyn_cast(V2); + SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2); - // Recursively peel off arguments and test them against the mask. - return isShuffleEquivalentImpl(V1, V2, BV1, BV2, Mask, 0, Args...); -} + unsigned ZMask = 0; + int V1DstIndex = -1; + int V2DstIndex = -1; + bool V1UsedInPlace = false; -/// \brief Get a 4-lane 8-bit shuffle immediate for a mask. -/// -/// This helper function produces an 8-bit shuffle immediate corresponding to -/// the ubiquitous shuffle encoding scheme used in x86 instructions for -/// shuffling 4 lanes. It can be used with most of the PSHUF instructions for -/// example. -/// -/// NB: We rely heavily on "undef" masks preserving the input lane. -static SDValue getV4X86ShuffleImm8ForMask(ArrayRef Mask, - SelectionDAG &DAG) { - assert(Mask.size() == 4 && "Only 4-lane shuffle masks"); - assert(Mask[0] >= -1 && Mask[0] < 4 && "Out of bound mask element!"); - assert(Mask[1] >= -1 && Mask[1] < 4 && "Out of bound mask element!"); - assert(Mask[2] >= -1 && Mask[2] < 4 && "Out of bound mask element!"); - assert(Mask[3] >= -1 && Mask[3] < 4 && "Out of bound mask element!"); + for (int i = 0; i < 4; ++i) { + // Synthesize a zero mask from the zeroable elements (includes undefs). + if (Zeroable[i]) { + ZMask |= 1 << i; + continue; + } - unsigned Imm = 0; - Imm |= (Mask[0] == -1 ? 0 : Mask[0]) << 0; - Imm |= (Mask[1] == -1 ? 1 : Mask[1]) << 2; - Imm |= (Mask[2] == -1 ? 2 : Mask[2]) << 4; - Imm |= (Mask[3] == -1 ? 3 : Mask[3]) << 6; - return DAG.getConstant(Imm, MVT::i8); -} - -/// \brief Try to emit a blend instruction for a shuffle using bit math. -/// -/// This is used as a fallback approach when first class blend instructions are -/// unavailable. Currently it is only suitable for integer vectors, but could -/// be generalized for floating point vectors if desirable. -static SDValue lowerVectorShuffleAsBitBlend(SDLoc DL, MVT VT, SDValue V1, - SDValue V2, ArrayRef Mask, - SelectionDAG &DAG) { - assert(VT.isInteger() && "Only supports integer vector types!"); - MVT EltVT = VT.getScalarType(); - int NumEltBits = EltVT.getSizeInBits(); - SDValue Zero = DAG.getConstant(0, EltVT); - SDValue AllOnes = DAG.getConstant(APInt::getAllOnesValue(NumEltBits), EltVT); - SmallVector MaskOps; - for (int i = 0, Size = Mask.size(); i < Size; ++i) { - if (Mask[i] != -1 && Mask[i] != i && Mask[i] != i + Size) - return SDValue(); // Shuffled input! - MaskOps.push_back(Mask[i] < Size ? AllOnes : Zero); - } - - SDValue V1Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, VT, MaskOps); - V1 = DAG.getNode(ISD::AND, DL, VT, V1, V1Mask); - // We have to cast V2 around. - MVT MaskVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64); - V2 = DAG.getNode(ISD::BITCAST, DL, VT, - DAG.getNode(X86ISD::ANDNP, DL, MaskVT, - DAG.getNode(ISD::BITCAST, DL, MaskVT, V1Mask), - DAG.getNode(ISD::BITCAST, DL, MaskVT, V2))); - return DAG.getNode(ISD::OR, DL, VT, V1, V2); -} - -/// \brief Try to emit a blend instruction for a shuffle. -/// -/// This doesn't do any checks for the availability of instructions for blending -/// these values. It relies on the availability of the X86ISD::BLENDI pattern to -/// be matched in the backend with the type given. What it does check for is -/// that the shuffle mask is in fact a blend. -static SDValue lowerVectorShuffleAsBlend(SDLoc DL, MVT VT, SDValue V1, - SDValue V2, ArrayRef Mask, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - unsigned BlendMask = 0; - for (int i = 0, Size = Mask.size(); i < Size; ++i) { - if (Mask[i] >= Size) { - if (Mask[i] != i + Size) - return SDValue(); // Shuffled V2 input! - BlendMask |= 1u << i; + // Flag if we use any V1 inputs in place. + if (i == Mask[i]) { + V1UsedInPlace = true; continue; } - if (Mask[i] >= 0 && Mask[i] != i) - return SDValue(); // Shuffled V1 input! - } - switch (VT.SimpleTy) { - case MVT::v2f64: - case MVT::v4f32: - case MVT::v4f64: - case MVT::v8f32: - return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2, - DAG.getConstant(BlendMask, MVT::i8)); - case MVT::v4i64: - case MVT::v8i32: - assert(Subtarget->hasAVX2() && "256-bit integer blends require AVX2!"); - // FALLTHROUGH - case MVT::v2i64: - case MVT::v4i32: - // If we have AVX2 it is faster to use VPBLENDD when the shuffle fits into - // that instruction. - if (Subtarget->hasAVX2()) { - // Scale the blend by the number of 32-bit dwords per element. - int Scale = VT.getScalarSizeInBits() / 32; - BlendMask = 0; - for (int i = 0, Size = Mask.size(); i < Size; ++i) - if (Mask[i] >= Size) - for (int j = 0; j < Scale; ++j) - BlendMask |= 1u << (i * Scale + j); + // We can only insert a single non-zeroable element. + if (V1DstIndex != -1 || V2DstIndex != -1) + return SDValue(); - MVT BlendVT = VT.getSizeInBits() > 128 ? MVT::v8i32 : MVT::v4i32; - V1 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V1); - V2 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V2); - return DAG.getNode(ISD::BITCAST, DL, VT, - DAG.getNode(X86ISD::BLENDI, DL, BlendVT, V1, V2, - DAG.getConstant(BlendMask, MVT::i8))); + if (Mask[i] < 4) { + // V1 input out of place for insertion. + V1DstIndex = i; + } else { + // V2 input for insertion. + V2DstIndex = i; } - // FALLTHROUGH - case MVT::v8i16: { - // For integer shuffles we need to expand the mask and cast the inputs to - // v8i16s prior to blending. - int Scale = 8 / VT.getVectorNumElements(); - BlendMask = 0; - for (int i = 0, Size = Mask.size(); i < Size; ++i) - if (Mask[i] >= Size) - for (int j = 0; j < Scale; ++j) - BlendMask |= 1u << (i * Scale + j); - - V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1); - V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V2); - return DAG.getNode(ISD::BITCAST, DL, VT, - DAG.getNode(X86ISD::BLENDI, DL, MVT::v8i16, V1, V2, - DAG.getConstant(BlendMask, MVT::i8))); } - case MVT::v16i16: { - assert(Subtarget->hasAVX2() && "256-bit integer blends require AVX2!"); - SmallVector RepeatedMask; - if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) { - // We can lower these with PBLENDW which is mirrored across 128-bit lanes. - assert(RepeatedMask.size() == 8 && "Repeated mask size doesn't match!"); - BlendMask = 0; - for (int i = 0; i < 8; ++i) - if (RepeatedMask[i] >= 16) - BlendMask |= 1u << i; - return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2, - DAG.getConstant(BlendMask, MVT::i8)); - } - } - // FALLTHROUGH - case MVT::v16i8: - case MVT::v32i8: { - // Scale the blend by the number of bytes per element. - int Scale = VT.getScalarSizeInBits() / 8; + // Don't bother if we have no (non-zeroable) element for insertion. + if (V1DstIndex == -1 && V2DstIndex == -1) + return SDValue(); - // This form of blend is always done on bytes. Compute the byte vector - // type. - MVT BlendVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); + // Determine element insertion src/dst indices. The src index is from the + // start of the inserted vector, not the start of the concatenated vector. + unsigned V2SrcIndex = 0; + if (V1DstIndex != -1) { + // If we have a V1 input out of place, we use V1 as the V2 element insertion + // and don't use the original V2 at all. + V2SrcIndex = Mask[V1DstIndex]; + V2DstIndex = V1DstIndex; + V2 = V1; + } else { + V2SrcIndex = Mask[V2DstIndex] - 4; + } - // Compute the VSELECT mask. Note that VSELECT is really confusing in the - // mix of LLVM's code generator and the x86 backend. We tell the code - // generator that boolean values in the elements of an x86 vector register - // are -1 for true and 0 for false. We then use the LLVM semantics of 'true' - // mapping a select to operand #1, and 'false' mapping to operand #2. The - // reality in x86 is that vector masks (pre-AVX-512) use only the high bit - // of the element (the remaining are ignored) and 0 in that high bit would - // mean operand #1 while 1 in the high bit would mean operand #2. So while - // the LLVM model for boolean values in vector elements gets the relevant - // bit set, it is set backwards and over constrained relative to x86's - // actual model. - SmallVector VSELECTMask; - for (int i = 0, Size = Mask.size(); i < Size; ++i) - for (int j = 0; j < Scale; ++j) - VSELECTMask.push_back( - Mask[i] < 0 ? DAG.getUNDEF(MVT::i8) - : DAG.getConstant(Mask[i] < Size ? -1 : 0, MVT::i8)); + // If no V1 inputs are used in place, then the result is created only from + // the zero mask and the V2 insertion - so remove V1 dependency. + if (!V1UsedInPlace) + V1 = DAG.getUNDEF(MVT::v4f32); - V1 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V1); - V2 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V2); - return DAG.getNode( - ISD::BITCAST, DL, VT, - DAG.getNode(ISD::VSELECT, DL, BlendVT, - DAG.getNode(ISD::BUILD_VECTOR, DL, BlendVT, VSELECTMask), - V1, V2)); - } + unsigned InsertPSMask = V2SrcIndex << 6 | V2DstIndex << 4 | ZMask; + assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!"); - default: - llvm_unreachable("Not a supported integer vector type!"); - } + // Insert the V2 element into the desired position. + SDLoc DL(Op); + return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2, + DAG.getConstant(InsertPSMask, MVT::i8)); } -/// \brief Try to lower as a blend of elements from two inputs followed by -/// a single-input permutation. +/// \brief Try to lower a shuffle as a permute of the inputs followed by an +/// UNPCK instruction. /// -/// This matches the pattern where we can blend elements from two inputs and -/// then reduce the shuffle to a single-input permutation. -static SDValue lowerVectorShuffleAsBlendAndPermute(SDLoc DL, MVT VT, SDValue V1, - SDValue V2, - ArrayRef Mask, - SelectionDAG &DAG) { - // We build up the blend mask while checking whether a blend is a viable way - // to reduce the shuffle. - SmallVector BlendMask(Mask.size(), -1); - SmallVector PermuteMask(Mask.size(), -1); +/// This specifically targets cases where we end up with alternating between +/// the two inputs, and so can permute them into something that feeds a single +/// UNPCK instruction. Note that this routine only targets integer vectors +/// because for floating point vectors we have a generalized SHUFPS lowering +/// strategy that handles everything that doesn't *exactly* match an unpack, +/// making this clever lowering unnecessary. +static SDValue lowerVectorShuffleAsUnpack(MVT VT, SDLoc DL, SDValue V1, + SDValue V2, ArrayRef Mask, + SelectionDAG &DAG) { + assert(!VT.isFloatingPoint() && + "This routine only supports integer vectors."); + assert(!isSingleInputShuffleMask(Mask) && + "This routine should only be used when blending two inputs."); + assert(Mask.size() >= 2 && "Single element masks are invalid."); - for (int i = 0, Size = Mask.size(); i < Size; ++i) { - if (Mask[i] < 0) - continue; + int Size = Mask.size(); - assert(Mask[i] < Size * 2 && "Shuffle input is out of bounds."); + int NumLoInputs = std::count_if(Mask.begin(), Mask.end(), [Size](int M) { + return M >= 0 && M % Size < Size / 2; + }); + int NumHiInputs = std::count_if( + Mask.begin(), Mask.end(), [Size](int M) { return M % Size >= Size / 2; }); - if (BlendMask[Mask[i] % Size] == -1) - BlendMask[Mask[i] % Size] = Mask[i]; - else if (BlendMask[Mask[i] % Size] != Mask[i]) - return SDValue(); // Can't blend in the needed input! + bool UnpackLo = NumLoInputs >= NumHiInputs; - PermuteMask[i] = Mask[i] % Size; - } + auto TryUnpack = [&](MVT UnpackVT, int Scale) { + SmallVector V1Mask(Mask.size(), -1); + SmallVector V2Mask(Mask.size(), -1); - SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask); - return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), PermuteMask); -} + for (int i = 0; i < Size; ++i) { + if (Mask[i] < 0) + continue; -/// \brief Generic routine to decompose a shuffle and blend into indepndent -/// blends and permutes. -/// -/// This matches the extremely common pattern for handling combined -/// shuffle+blend operations on newer X86 ISAs where we have very fast blend -/// operations. It will try to pick the best arrangement of shuffles and -/// blends. -static SDValue lowerVectorShuffleAsDecomposedShuffleBlend(SDLoc DL, MVT VT, - SDValue V1, - SDValue V2, - ArrayRef Mask, - SelectionDAG &DAG) { - // Shuffle the input elements into the desired positions in V1 and V2 and - // blend them together. - SmallVector V1Mask(Mask.size(), -1); - SmallVector V2Mask(Mask.size(), -1); - SmallVector BlendMask(Mask.size(), -1); - for (int i = 0, Size = Mask.size(); i < Size; ++i) - if (Mask[i] >= 0 && Mask[i] < Size) { - V1Mask[i] = Mask[i]; - BlendMask[i] = i; - } else if (Mask[i] >= Size) { - V2Mask[i] = Mask[i] - Size; - BlendMask[i] = i + Size; - } - - // Try to lower with the simpler initial blend strategy unless one of the - // input shuffles would be a no-op. We prefer to shuffle inputs as the - // shuffle may be able to fold with a load or other benefit. However, when - // we'll have to do 2x as many shuffles in order to achieve this, blending - // first is a better strategy. - if (!isNoopShuffleMask(V1Mask) && !isNoopShuffleMask(V2Mask)) - if (SDValue BlendPerm = - lowerVectorShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask, DAG)) - return BlendPerm; - - V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask); - V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask); - return DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask); -} - -/// \brief Try to lower a vector shuffle as a byte rotation. -/// -/// SSSE3 has a generic PALIGNR instruction in x86 that will do an arbitrary -/// byte-rotation of the concatenation of two vectors; pre-SSSE3 can use -/// a PSRLDQ/PSLLDQ/POR pattern to get a similar effect. This routine will -/// try to generically lower a vector shuffle through such an pattern. It -/// does not check for the profitability of lowering either as PALIGNR or -/// PSRLDQ/PSLLDQ/POR, only whether the mask is valid to lower in that form. -/// This matches shuffle vectors that look like: -/// -/// v8i16 [11, 12, 13, 14, 15, 0, 1, 2] -/// -/// Essentially it concatenates V1 and V2, shifts right by some number of -/// elements, and takes the low elements as the result. Note that while this is -/// specified as a *right shift* because x86 is little-endian, it is a *left -/// rotate* of the vector lanes. -static SDValue lowerVectorShuffleAsByteRotate(SDLoc DL, MVT VT, SDValue V1, - SDValue V2, - ArrayRef Mask, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!"); - - int NumElts = Mask.size(); - int NumLanes = VT.getSizeInBits() / 128; - int NumLaneElts = NumElts / NumLanes; - - // We need to detect various ways of spelling a rotation: - // [11, 12, 13, 14, 15, 0, 1, 2] - // [-1, 12, 13, 14, -1, -1, 1, -1] - // [-1, -1, -1, -1, -1, -1, 1, 2] - // [ 3, 4, 5, 6, 7, 8, 9, 10] - // [-1, 4, 5, 6, -1, -1, 9, -1] - // [-1, 4, 5, 6, -1, -1, -1, -1] - int Rotation = 0; - SDValue Lo, Hi; - for (int l = 0; l < NumElts; l += NumLaneElts) { - for (int i = 0; i < NumLaneElts; ++i) { - if (Mask[l + i] == -1) - continue; - assert(Mask[l + i] >= 0 && "Only -1 is a valid negative mask element!"); + // Each element of the unpack contains Scale elements from this mask. + int UnpackIdx = i / Scale; - // Get the mod-Size index and lane correct it. - int LaneIdx = (Mask[l + i] % NumElts) - l; - // Make sure it was in this lane. - if (LaneIdx < 0 || LaneIdx >= NumLaneElts) + // We only handle the case where V1 feeds the first slots of the unpack. + // We rely on canonicalization to ensure this is the case. + if ((UnpackIdx % 2 == 0) != (Mask[i] < Size)) return SDValue(); - // Determine where a rotated vector would have started. - int StartIdx = i - LaneIdx; - if (StartIdx == 0) - // The identity rotation isn't interesting, stop. - return SDValue(); + // Setup the mask for this input. The indexing is tricky as we have to + // handle the unpack stride. + SmallVectorImpl &VMask = (UnpackIdx % 2 == 0) ? V1Mask : V2Mask; + VMask[(UnpackIdx / 2) * Scale + i % Scale + (UnpackLo ? 0 : Size / 2)] = + Mask[i] % Size; + } - // If we found the tail of a vector the rotation must be the missing - // front. If we found the head of a vector, it must be how much of the - // head. - int CandidateRotation = StartIdx < 0 ? -StartIdx : NumLaneElts - StartIdx; + // If we will have to shuffle both inputs to use the unpack, check whether + // we can just unpack first and shuffle the result. If so, skip this unpack. + if ((NumLoInputs == 0 || NumHiInputs == 0) && !isNoopShuffleMask(V1Mask) && + !isNoopShuffleMask(V2Mask)) + return SDValue(); - if (Rotation == 0) - Rotation = CandidateRotation; - else if (Rotation != CandidateRotation) - // The rotations don't match, so we can't match this mask. - return SDValue(); + // Shuffle the inputs into place. + V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask); + V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask); - // Compute which value this mask is pointing at. - SDValue MaskV = Mask[l + i] < NumElts ? V1 : V2; + // Cast the inputs to the type we will use to unpack them. + V1 = DAG.getNode(ISD::BITCAST, DL, UnpackVT, V1); + V2 = DAG.getNode(ISD::BITCAST, DL, UnpackVT, V2); - // Compute which of the two target values this index should be assigned - // to. This reflects whether the high elements are remaining or the low - // elements are remaining. - SDValue &TargetV = StartIdx < 0 ? Hi : Lo; + // Unpack the inputs and cast the result back to the desired type. + return DAG.getNode(ISD::BITCAST, DL, VT, + DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, + DL, UnpackVT, V1, V2)); + }; - // Either set up this value if we've not encountered it before, or check - // that it remains consistent. - if (!TargetV) - TargetV = MaskV; - else if (TargetV != MaskV) - // This may be a rotation, but it pulls from the inputs in some - // unsupported interleaving. - return SDValue(); - } + // We try each unpack from the largest to the smallest to try and find one + // that fits this mask. + int OrigNumElements = VT.getVectorNumElements(); + int OrigScalarSize = VT.getScalarSizeInBits(); + for (int ScalarSize = 64; ScalarSize >= OrigScalarSize; ScalarSize /= 2) { + int Scale = ScalarSize / OrigScalarSize; + int NumElements = OrigNumElements / Scale; + MVT UnpackVT = MVT::getVectorVT(MVT::getIntegerVT(ScalarSize), NumElements); + if (SDValue Unpack = TryUnpack(UnpackVT, Scale)) + return Unpack; } - // Check that we successfully analyzed the mask, and normalize the results. - assert(Rotation != 0 && "Failed to locate a viable rotation!"); - assert((Lo || Hi) && "Failed to find a rotated input vector!"); - if (!Lo) - Lo = Hi; - else if (!Hi) - Hi = Lo; - - // The actual rotate instruction rotates bytes, so we need to scale the - // rotation based on how many bytes are in the vector lane. - int Scale = 16 / NumLaneElts; - - // SSSE3 targets can use the palignr instruction. - if (Subtarget->hasSSSE3()) { - // Cast the inputs to i8 vector of correct length to match PALIGNR. - MVT AlignVT = MVT::getVectorVT(MVT::i8, 16 * NumLanes); - Lo = DAG.getNode(ISD::BITCAST, DL, AlignVT, Lo); - Hi = DAG.getNode(ISD::BITCAST, DL, AlignVT, Hi); + // If none of the unpack-rooted lowerings worked (or were profitable) try an + // initial unpack. + if (NumLoInputs == 0 || NumHiInputs == 0) { + assert((NumLoInputs > 0 || NumHiInputs > 0) && + "We have to have *some* inputs!"); + int HalfOffset = NumLoInputs == 0 ? Size / 2 : 0; - return DAG.getNode(ISD::BITCAST, DL, VT, - DAG.getNode(X86ISD::PALIGNR, DL, AlignVT, Hi, Lo, - DAG.getConstant(Rotation * Scale, MVT::i8))); - } + // FIXME: We could consider the total complexity of the permute of each + // possible unpacking. Or at the least we should consider how many + // half-crossings are created. + // FIXME: We could consider commuting the unpacks. - assert(VT.getSizeInBits() == 128 && - "Rotate-based lowering only supports 128-bit lowering!"); - assert(Mask.size() <= 16 && - "Can shuffle at most 16 bytes in a 128-bit vector!"); + SmallVector PermMask; + PermMask.assign(Size, -1); + for (int i = 0; i < Size; ++i) { + if (Mask[i] < 0) + continue; - // Default SSE2 implementation - int LoByteShift = 16 - Rotation * Scale; - int HiByteShift = Rotation * Scale; + assert(Mask[i] % Size >= HalfOffset && "Found input from wrong half!"); - // Cast the inputs to v2i64 to match PSLLDQ/PSRLDQ. - Lo = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Lo); - Hi = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Hi); + PermMask[i] = + 2 * ((Mask[i] % Size) - HalfOffset) + (Mask[i] < Size ? 0 : 1); + } + return DAG.getVectorShuffle( + VT, DL, DAG.getNode(NumLoInputs == 0 ? X86ISD::UNPCKH : X86ISD::UNPCKL, + DL, VT, V1, V2), + DAG.getUNDEF(VT), PermMask); + } - SDValue LoShift = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v2i64, Lo, - DAG.getConstant(LoByteShift, MVT::i8)); - SDValue HiShift = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v2i64, Hi, - DAG.getConstant(HiByteShift, MVT::i8)); - return DAG.getNode(ISD::BITCAST, DL, VT, - DAG.getNode(ISD::OR, DL, MVT::v2i64, LoShift, HiShift)); + return SDValue(); } -/// \brief Compute whether each element of a shuffle is zeroable. +/// \brief Handle lowering of 2-lane 64-bit floating point shuffles. /// -/// A "zeroable" vector shuffle element is one which can be lowered to zero. -/// Either it is an undef element in the shuffle mask, the element of the input -/// referenced is undef, or the element of the input referenced is known to be -/// zero. Many x86 shuffles can zero lanes cheaply and we often want to handle -/// as many lanes with this technique as possible to simplify the remaining -/// shuffle. -static SmallBitVector computeZeroableShuffleElements(ArrayRef Mask, - SDValue V1, SDValue V2) { - SmallBitVector Zeroable(Mask.size(), false); +/// This is the basis function for the 2-lane 64-bit shuffles as we have full +/// support for floating point shuffles but not integer shuffles. These +/// instructions will incur a domain crossing penalty on some chips though so +/// it is better to avoid lowering through this for integer vectors where +/// possible. +static SDValue lowerV2F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); + assert(Op.getSimpleValueType() == MVT::v2f64 && "Bad shuffle type!"); + assert(V1.getSimpleValueType() == MVT::v2f64 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!"); + ShuffleVectorSDNode *SVOp = cast(Op); + ArrayRef Mask = SVOp->getMask(); + assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!"); - while (V1.getOpcode() == ISD::BITCAST) - V1 = V1->getOperand(0); - while (V2.getOpcode() == ISD::BITCAST) - V2 = V2->getOperand(0); + if (isSingleInputShuffleMask(Mask)) { + // Use low duplicate instructions for masks that match their pattern. + if (Subtarget->hasSSE3()) + if (isShuffleEquivalent(V1, V2, Mask, 0, 0)) + return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, V1); - bool V1IsZero = ISD::isBuildVectorAllZeros(V1.getNode()); - bool V2IsZero = ISD::isBuildVectorAllZeros(V2.getNode()); + // Straight shuffle of a single input vector. Simulate this by using the + // single input as both of the "inputs" to this instruction.. + unsigned SHUFPDMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1); - for (int i = 0, Size = Mask.size(); i < Size; ++i) { - int M = Mask[i]; - // Handle the easy cases. - if (M < 0 || (M >= 0 && M < Size && V1IsZero) || (M >= Size && V2IsZero)) { - Zeroable[i] = true; - continue; + if (Subtarget->hasAVX()) { + // If we have AVX, we can use VPERMILPS which will allow folding a load + // into the shuffle. + return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1, + DAG.getConstant(SHUFPDMask, MVT::i8)); } - // If this is an index into a build_vector node (which has the same number - // of elements), dig out the input value and use it. - SDValue V = M < Size ? V1 : V2; - if (V.getOpcode() != ISD::BUILD_VECTOR || Size != (int)V.getNumOperands()) - continue; + return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V1, + DAG.getConstant(SHUFPDMask, MVT::i8)); + } + assert(Mask[0] >= 0 && Mask[0] < 2 && "Non-canonicalized blend!"); + assert(Mask[1] >= 2 && "Non-canonicalized blend!"); - SDValue Input = V.getOperand(M % Size); - // The UNDEF opcode check really should be dead code here, but not quite - // worth asserting on (it isn't invalid, just unexpected). - if (Input.getOpcode() == ISD::UNDEF || X86::isZeroNode(Input)) - Zeroable[i] = true; + // If we have a single input, insert that into V1 if we can do so cheaply. + if ((Mask[0] >= 2) + (Mask[1] >= 2) == 1) { + if (SDValue Insertion = lowerVectorShuffleAsElementInsertion( + MVT::v2f64, DL, V1, V2, Mask, Subtarget, DAG)) + return Insertion; + // Try inverting the insertion since for v2 masks it is easy to do and we + // can't reliably sort the mask one way or the other. + int InverseMask[2] = {Mask[0] < 0 ? -1 : (Mask[0] ^ 2), + Mask[1] < 0 ? -1 : (Mask[1] ^ 2)}; + if (SDValue Insertion = lowerVectorShuffleAsElementInsertion( + MVT::v2f64, DL, V2, V1, InverseMask, Subtarget, DAG)) + return Insertion; } - return Zeroable; -} + // Try to use one of the special instruction patterns to handle two common + // blend patterns if a zero-blend above didn't work. + if (isShuffleEquivalent(V1, V2, Mask, 0, 3) || isShuffleEquivalent(V1, V2, Mask, 1, 3)) + if (SDValue V1S = getScalarValueForVectorElement(V1, Mask[0], DAG)) + // We can either use a special instruction to load over the low double or + // to move just the low double. + return DAG.getNode( + isShuffleFoldableLoad(V1S) ? X86ISD::MOVLPD : X86ISD::MOVSD, + DL, MVT::v2f64, V2, + DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S)); -/// \brief Try to emit a bitmask instruction for a shuffle. -/// -/// This handles cases where we can model a blend exactly as a bitmask due to -/// one of the inputs being zeroable. -static SDValue lowerVectorShuffleAsBitMask(SDLoc DL, MVT VT, SDValue V1, - SDValue V2, ArrayRef Mask, - SelectionDAG &DAG) { - MVT EltVT = VT.getScalarType(); - int NumEltBits = EltVT.getSizeInBits(); - MVT IntEltVT = MVT::getIntegerVT(NumEltBits); - SDValue Zero = DAG.getConstant(0, IntEltVT); - SDValue AllOnes = DAG.getConstant(APInt::getAllOnesValue(NumEltBits), IntEltVT); - if (EltVT.isFloatingPoint()) { - Zero = DAG.getNode(ISD::BITCAST, DL, EltVT, Zero); - AllOnes = DAG.getNode(ISD::BITCAST, DL, EltVT, AllOnes); - } - SmallVector VMaskOps(Mask.size(), Zero); - SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2); - SDValue V; - for (int i = 0, Size = Mask.size(); i < Size; ++i) { - if (Zeroable[i]) - continue; - if (Mask[i] % Size != i) - return SDValue(); // Not a blend. - if (!V) - V = Mask[i] < Size ? V1 : V2; - else if (V != (Mask[i] < Size ? V1 : V2)) - return SDValue(); // Can only let one input through the mask. + if (Subtarget->hasSSE41()) + if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask, + Subtarget, DAG)) + return Blend; - VMaskOps[i] = AllOnes; - } - if (!V) - return SDValue(); // No non-zeroable elements! + // Use dedicated unpack instructions for masks that match their pattern. + if (isShuffleEquivalent(V1, V2, Mask, 0, 2)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2f64, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, 1, 3)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v2f64, V1, V2); - SDValue VMask = DAG.getNode(ISD::BUILD_VECTOR, DL, VT, VMaskOps); - V = DAG.getNode(VT.isFloatingPoint() - ? (unsigned) X86ISD::FAND : (unsigned) ISD::AND, - DL, VT, V, VMask); - return V; + unsigned SHUFPDMask = (Mask[0] == 1) | (((Mask[1] - 2) == 1) << 1); + return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V2, + DAG.getConstant(SHUFPDMask, MVT::i8)); } -/// \brief Try to lower a vector shuffle as a bit shift (shifts in zeros). -/// -/// Attempts to match a shuffle mask against the PSLL(W/D/Q/DQ) and -/// PSRL(W/D/Q/DQ) SSE2 and AVX2 logical bit-shift instructions. The function -/// matches elements from one of the input vectors shuffled to the left or -/// right with zeroable elements 'shifted in'. It handles both the strictly -/// bit-wise element shifts and the byte shift across an entire 128-bit double -/// quad word lane. +/// \brief Handle lowering of 2-lane 64-bit integer shuffles. /// -/// PSHL : (little-endian) left bit shift. -/// [ zz, 0, zz, 2 ] -/// [ -1, 4, zz, -1 ] -/// PSRL : (little-endian) right bit shift. -/// [ 1, zz, 3, zz] -/// [ -1, -1, 7, zz] -/// PSLLDQ : (little-endian) left byte shift -/// [ zz, 0, 1, 2, 3, 4, 5, 6] -/// [ zz, zz, -1, -1, 2, 3, 4, -1] -/// [ zz, zz, zz, zz, zz, zz, -1, 1] -/// PSRLDQ : (little-endian) right byte shift -/// [ 5, 6, 7, zz, zz, zz, zz, zz] -/// [ -1, 5, 6, 7, zz, zz, zz, zz] -/// [ 1, 2, -1, -1, -1, -1, zz, zz] -static SDValue lowerVectorShuffleAsShift(SDLoc DL, MVT VT, SDValue V1, - SDValue V2, ArrayRef Mask, - SelectionDAG &DAG) { - SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2); +/// Tries to lower a 2-lane 64-bit shuffle using shuffle operations provided by +/// the integer unit to minimize domain crossing penalties. However, for blends +/// it falls back to the floating point shuffle operation with appropriate bit +/// casting. +static SDValue lowerV2I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); + assert(Op.getSimpleValueType() == MVT::v2i64 && "Bad shuffle type!"); + assert(V1.getSimpleValueType() == MVT::v2i64 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!"); + ShuffleVectorSDNode *SVOp = cast(Op); + ArrayRef Mask = SVOp->getMask(); + assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!"); - int Size = Mask.size(); - assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size"); + if (isSingleInputShuffleMask(Mask)) { + // Check for being able to broadcast a single element. + if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v2i64, DL, V1, + Mask, Subtarget, DAG)) + return Broadcast; - auto CheckZeros = [&](int Shift, int Scale, bool Left) { - for (int i = 0; i < Size; i += Scale) - for (int j = 0; j < Shift; ++j) - if (!Zeroable[i + j + (Left ? 0 : (Scale - Shift))]) - return false; + // Straight shuffle of a single input vector. For everything from SSE2 + // onward this has a single fast instruction with no scary immediates. + // We have to map the mask as it is actually a v4i32 shuffle instruction. + V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V1); + int WidenedMask[4] = { + std::max(Mask[0], 0) * 2, std::max(Mask[0], 0) * 2 + 1, + std::max(Mask[1], 0) * 2, std::max(Mask[1], 0) * 2 + 1}; + return DAG.getNode( + ISD::BITCAST, DL, MVT::v2i64, + DAG.getNode(X86ISD::PSHUFD, SDLoc(Op), MVT::v4i32, V1, + getV4X86ShuffleImm8ForMask(WidenedMask, DAG))); + } + assert(Mask[0] != -1 && "No undef lanes in multi-input v2 shuffles!"); + assert(Mask[1] != -1 && "No undef lanes in multi-input v2 shuffles!"); + assert(Mask[0] < 2 && "We sort V1 to be the first input."); + assert(Mask[1] >= 2 && "We sort V2 to be the second input."); - return true; - }; + // If we have a blend of two PACKUS operations an the blend aligns with the + // low and half halves, we can just merge the PACKUS operations. This is + // particularly important as it lets us merge shuffles that this routine itself + // creates. + auto GetPackNode = [](SDValue V) { + while (V.getOpcode() == ISD::BITCAST) + V = V.getOperand(0); - auto MatchShift = [&](int Shift, int Scale, bool Left, SDValue V) { - for (int i = 0; i != Size; i += Scale) { - unsigned Pos = Left ? i + Shift : i; - unsigned Low = Left ? i : i + Shift; - unsigned Len = Scale - Shift; - if (!isSequentialOrUndefInRange(Mask, Pos, Len, - Low + (V == V1 ? 0 : Size))) - return SDValue(); - } + return V.getOpcode() == X86ISD::PACKUS ? V : SDValue(); + }; + if (SDValue V1Pack = GetPackNode(V1)) + if (SDValue V2Pack = GetPackNode(V2)) + return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, + DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, + Mask[0] == 0 ? V1Pack.getOperand(0) + : V1Pack.getOperand(1), + Mask[1] == 2 ? V2Pack.getOperand(0) + : V2Pack.getOperand(1))); - int ShiftEltBits = VT.getScalarSizeInBits() * Scale; - bool ByteShift = ShiftEltBits > 64; - unsigned OpCode = Left ? (ByteShift ? X86ISD::VSHLDQ : X86ISD::VSHLI) - : (ByteShift ? X86ISD::VSRLDQ : X86ISD::VSRLI); - int ShiftAmt = Shift * VT.getScalarSizeInBits() / (ByteShift ? 8 : 1); + // Try to use shift instructions. + if (SDValue Shift = + lowerVectorShuffleAsShift(DL, MVT::v2i64, V1, V2, Mask, DAG)) + return Shift; - // Normalize the scale for byte shifts to still produce an i64 element - // type. - Scale = ByteShift ? Scale / 2 : Scale; + // When loading a scalar and then shuffling it into a vector we can often do + // the insertion cheaply. + if (SDValue Insertion = lowerVectorShuffleAsElementInsertion( + MVT::v2i64, DL, V1, V2, Mask, Subtarget, DAG)) + return Insertion; + // Try inverting the insertion since for v2 masks it is easy to do and we + // can't reliably sort the mask one way or the other. + int InverseMask[2] = {Mask[0] ^ 2, Mask[1] ^ 2}; + if (SDValue Insertion = lowerVectorShuffleAsElementInsertion( + MVT::v2i64, DL, V2, V1, InverseMask, Subtarget, DAG)) + return Insertion; - // We need to round trip through the appropriate type for the shift. - MVT ShiftSVT = MVT::getIntegerVT(VT.getScalarSizeInBits() * Scale); - MVT ShiftVT = MVT::getVectorVT(ShiftSVT, Size / Scale); - assert(DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) && - "Illegal integer vector type"); - V = DAG.getNode(ISD::BITCAST, DL, ShiftVT, V); + // We have different paths for blend lowering, but they all must use the + // *exact* same predicate. + bool IsBlendSupported = Subtarget->hasSSE41(); + if (IsBlendSupported) + if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask, + Subtarget, DAG)) + return Blend; - V = DAG.getNode(OpCode, DL, ShiftVT, V, DAG.getConstant(ShiftAmt, MVT::i8)); - return DAG.getNode(ISD::BITCAST, DL, VT, V); - }; + // Use dedicated unpack instructions for masks that match their pattern. + if (isShuffleEquivalent(V1, V2, Mask, 0, 2)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, 1, 3)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v2i64, V1, V2); - // SSE/AVX supports logical shifts up to 64-bit integers - so we can just - // keep doubling the size of the integer elements up to that. We can - // then shift the elements of the integer vector by whole multiples of - // their width within the elements of the larger integer vector. Test each - // multiple to see if we can find a match with the moved element indices - // and that the shifted in elements are all zeroable. - for (int Scale = 2; Scale * VT.getScalarSizeInBits() <= 128; Scale *= 2) - for (int Shift = 1; Shift != Scale; ++Shift) - for (bool Left : {true, false}) - if (CheckZeros(Shift, Scale, Left)) - for (SDValue V : {V1, V2}) - if (SDValue Match = MatchShift(Shift, Scale, Left, V)) - return Match; + // Try to use byte rotation instructions. + // Its more profitable for pre-SSSE3 to use shuffles/unpacks. + if (Subtarget->hasSSSE3()) + if (SDValue Rotate = lowerVectorShuffleAsByteRotate( + DL, MVT::v2i64, V1, V2, Mask, Subtarget, DAG)) + return Rotate; - // no match - return SDValue(); -} + // If we have direct support for blends, we should lower by decomposing into + // a permute. That will be faster than the domain cross. + if (IsBlendSupported) + return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v2i64, V1, V2, + Mask, DAG); -/// \brief Lower a vector shuffle as a zero or any extension. -/// -/// Given a specific number of elements, element bit width, and extension -/// stride, produce either a zero or any extension based on the available -/// features of the subtarget. -static SDValue lowerVectorShuffleAsSpecificZeroOrAnyExtend( - SDLoc DL, MVT VT, int Scale, bool AnyExt, SDValue InputV, - const X86Subtarget *Subtarget, SelectionDAG &DAG) { - assert(Scale > 1 && "Need a scale to extend."); - int NumElements = VT.getVectorNumElements(); - int EltBits = VT.getScalarSizeInBits(); - assert((EltBits == 8 || EltBits == 16 || EltBits == 32) && - "Only 8, 16, and 32 bit elements can be extended."); - assert(Scale * EltBits <= 64 && "Cannot zero extend past 64 bits."); - - // Found a valid zext mask! Try various lowering strategies based on the - // input type and available ISA extensions. - if (Subtarget->hasSSE41()) { - MVT ExtVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Scale), - NumElements / Scale); - return DAG.getNode(ISD::BITCAST, DL, VT, - DAG.getNode(X86ISD::VZEXT, DL, ExtVT, InputV)); - } + // We implement this with SHUFPD which is pretty lame because it will likely + // incur 2 cycles of stall for integer vectors on Nehalem and older chips. + // However, all the alternatives are still more cycles and newer chips don't + // have this problem. It would be really nice if x86 had better shuffles here. + V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V1); + V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V2); + return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, + DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask)); +} - // For any extends we can cheat for larger element sizes and use shuffle - // instructions that can fold with a load and/or copy. - if (AnyExt && EltBits == 32) { - int PSHUFDMask[4] = {0, -1, 1, -1}; - return DAG.getNode( - ISD::BITCAST, DL, VT, - DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, - DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, InputV), - getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG))); - } - if (AnyExt && EltBits == 16 && Scale > 2) { - int PSHUFDMask[4] = {0, -1, 0, -1}; - InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, - DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, InputV), - getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG)); - int PSHUFHWMask[4] = {1, -1, -1, -1}; - return DAG.getNode( - ISD::BITCAST, DL, VT, - DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16, - DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, InputV), - getV4X86ShuffleImm8ForMask(PSHUFHWMask, DAG))); - } +/// \brief Test whether this can be lowered with a single SHUFPS instruction. +/// +/// This is used to disable more specialized lowerings when the shufps lowering +/// will happen to be efficient. +static bool isSingleSHUFPSMask(ArrayRef Mask) { + // This routine only handles 128-bit shufps. + assert(Mask.size() == 4 && "Unsupported mask size!"); - // If this would require more than 2 unpack instructions to expand, use - // pshufb when available. We can only use more than 2 unpack instructions - // when zero extending i8 elements which also makes it easier to use pshufb. - if (Scale > 4 && EltBits == 8 && Subtarget->hasSSSE3()) { - assert(NumElements == 16 && "Unexpected byte vector width!"); - SDValue PSHUFBMask[16]; - for (int i = 0; i < 16; ++i) - PSHUFBMask[i] = - DAG.getConstant((i % Scale == 0) ? i / Scale : 0x80, MVT::i8); - InputV = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, InputV); - return DAG.getNode(ISD::BITCAST, DL, VT, - DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV, - DAG.getNode(ISD::BUILD_VECTOR, DL, - MVT::v16i8, PSHUFBMask))); - } + // To lower with a single SHUFPS we need to have the low half and high half + // each requiring a single input. + if (Mask[0] != -1 && Mask[1] != -1 && (Mask[0] < 4) != (Mask[1] < 4)) + return false; + if (Mask[2] != -1 && Mask[3] != -1 && (Mask[2] < 4) != (Mask[3] < 4)) + return false; - // Otherwise emit a sequence of unpacks. - do { - MVT InputVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits), NumElements); - SDValue Ext = AnyExt ? DAG.getUNDEF(InputVT) - : getZeroVector(InputVT, Subtarget, DAG, DL); - InputV = DAG.getNode(ISD::BITCAST, DL, InputVT, InputV); - InputV = DAG.getNode(X86ISD::UNPCKL, DL, InputVT, InputV, Ext); - Scale /= 2; - EltBits *= 2; - NumElements /= 2; - } while (Scale > 1); - return DAG.getNode(ISD::BITCAST, DL, VT, InputV); + return true; } -/// \brief Try to lower a vector shuffle as a zero extension on any microarch. -/// -/// This routine will try to do everything in its power to cleverly lower -/// a shuffle which happens to match the pattern of a zero extend. It doesn't -/// check for the profitability of this lowering, it tries to aggressively -/// match this pattern. It will use all of the micro-architectural details it -/// can to emit an efficient lowering. It handles both blends with all-zero -/// inputs to explicitly zero-extend and undef-lanes (sometimes undef due to -/// masking out later). +/// \brief Lower a vector shuffle using the SHUFPS instruction. /// -/// The reason we have dedicated lowering for zext-style shuffles is that they -/// are both incredibly common and often quite performance sensitive. -static SDValue lowerVectorShuffleAsZeroOrAnyExtend( - SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef Mask, - const X86Subtarget *Subtarget, SelectionDAG &DAG) { - SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2); +/// This is a helper routine dedicated to lowering vector shuffles using SHUFPS. +/// It makes no assumptions about whether this is the *best* lowering, it simply +/// uses it. +static SDValue lowerVectorShuffleWithSHUFPS(SDLoc DL, MVT VT, + ArrayRef Mask, SDValue V1, + SDValue V2, SelectionDAG &DAG) { + SDValue LowV = V1, HighV = V2; + int NewMask[4] = {Mask[0], Mask[1], Mask[2], Mask[3]}; - int Bits = VT.getSizeInBits(); - int NumElements = VT.getVectorNumElements(); - assert(VT.getScalarSizeInBits() <= 32 && - "Exceeds 32-bit integer zero extension limit"); - assert((int)Mask.size() == NumElements && "Unexpected shuffle mask size"); + int NumV2Elements = + std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; }); - // Define a helper function to check a particular ext-scale and lower to it if - // valid. - auto Lower = [&](int Scale) -> SDValue { - SDValue InputV; - bool AnyExt = true; - for (int i = 0; i < NumElements; ++i) { - if (Mask[i] == -1) - continue; // Valid anywhere but doesn't tell us anything. - if (i % Scale != 0) { - // Each of the extended elements need to be zeroable. - if (!Zeroable[i]) - return SDValue(); + if (NumV2Elements == 1) { + int V2Index = + std::find_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; }) - + Mask.begin(); - // We no longer are in the anyext case. - AnyExt = false; - continue; - } + // Compute the index adjacent to V2Index and in the same half by toggling + // the low bit. + int V2AdjIndex = V2Index ^ 1; - // Each of the base elements needs to be consecutive indices into the - // same input vector. - SDValue V = Mask[i] < NumElements ? V1 : V2; - if (!InputV) - InputV = V; - else if (InputV != V) - return SDValue(); // Flip-flopping inputs. + if (Mask[V2AdjIndex] == -1) { + // Handles all the cases where we have a single V2 element and an undef. + // This will only ever happen in the high lanes because we commute the + // vector otherwise. + if (V2Index < 2) + std::swap(LowV, HighV); + NewMask[V2Index] -= 4; + } else { + // Handle the case where the V2 element ends up adjacent to a V1 element. + // To make this work, blend them together as the first step. + int V1Index = V2AdjIndex; + int BlendMask[4] = {Mask[V2Index] - 4, 0, Mask[V1Index], 0}; + V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1, + getV4X86ShuffleImm8ForMask(BlendMask, DAG)); - if (Mask[i] % NumElements != i / Scale) - return SDValue(); // Non-consecutive strided elements. + // Now proceed to reconstruct the final blend as we have the necessary + // high or low half formed. + if (V2Index < 2) { + LowV = V2; + HighV = V1; + } else { + HighV = V2; + } + NewMask[V1Index] = 2; // We put the V1 element in V2[2]. + NewMask[V2Index] = 0; // We shifted the V2 element into V2[0]. } + } else if (NumV2Elements == 2) { + if (Mask[0] < 4 && Mask[1] < 4) { + // Handle the easy case where we have V1 in the low lanes and V2 in the + // high lanes. + NewMask[2] -= 4; + NewMask[3] -= 4; + } else if (Mask[2] < 4 && Mask[3] < 4) { + // We also handle the reversed case because this utility may get called + // when we detect a SHUFPS pattern but can't easily commute the shuffle to + // arrange things in the right direction. + NewMask[0] -= 4; + NewMask[1] -= 4; + HighV = V1; + LowV = V2; + } else { + // We have a mixture of V1 and V2 in both low and high lanes. Rather than + // trying to place elements directly, just blend them and set up the final + // shuffle to place them. - // If we fail to find an input, we have a zero-shuffle which should always - // have already been handled. - // FIXME: Maybe handle this here in case during blending we end up with one? - if (!InputV) - return SDValue(); - - return lowerVectorShuffleAsSpecificZeroOrAnyExtend( - DL, VT, Scale, AnyExt, InputV, Subtarget, DAG); - }; - - // The widest scale possible for extending is to a 64-bit integer. - assert(Bits % 64 == 0 && - "The number of bits in a vector must be divisible by 64 on x86!"); - int NumExtElements = Bits / 64; + // The first two blend mask elements are for V1, the second two are for + // V2. + int BlendMask[4] = {Mask[0] < 4 ? Mask[0] : Mask[1], + Mask[2] < 4 ? Mask[2] : Mask[3], + (Mask[0] >= 4 ? Mask[0] : Mask[1]) - 4, + (Mask[2] >= 4 ? Mask[2] : Mask[3]) - 4}; + V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2, + getV4X86ShuffleImm8ForMask(BlendMask, DAG)); - // Each iteration, try extending the elements half as much, but into twice as - // many elements. - for (; NumExtElements < NumElements; NumExtElements *= 2) { - assert(NumElements % NumExtElements == 0 && - "The input vector size must be divisible by the extended size."); - if (SDValue V = Lower(NumElements / NumExtElements)) - return V; + // Now we do a normal shuffle of V1 by giving V1 as both operands to + // a blend. + LowV = HighV = V1; + NewMask[0] = Mask[0] < 4 ? 0 : 2; + NewMask[1] = Mask[0] < 4 ? 2 : 0; + NewMask[2] = Mask[2] < 4 ? 1 : 3; + NewMask[3] = Mask[2] < 4 ? 3 : 1; + } } + return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV, + getV4X86ShuffleImm8ForMask(NewMask, DAG)); +} - // General extends failed, but 128-bit vectors may be able to use MOVQ. - if (Bits != 128) - return SDValue(); +/// \brief Lower 4-lane 32-bit floating point shuffles. +/// +/// Uses instructions exclusively from the floating point unit to minimize +/// domain crossing penalties, as these are sufficient to implement all v4f32 +/// shuffles. +static SDValue lowerV4F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); + assert(Op.getSimpleValueType() == MVT::v4f32 && "Bad shuffle type!"); + assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); + ShuffleVectorSDNode *SVOp = cast(Op); + ArrayRef Mask = SVOp->getMask(); + assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"); - // Returns one of the source operands if the shuffle can be reduced to a - // MOVQ, copying the lower 64-bits and zero-extending to the upper 64-bits. - auto CanZExtLowHalf = [&]() { - for (int i = NumElements / 2; i != NumElements; ++i) - if (!Zeroable[i]) - return SDValue(); - if (isSequentialOrUndefInRange(Mask, 0, NumElements / 2, 0)) - return V1; - if (isSequentialOrUndefInRange(Mask, 0, NumElements / 2, NumElements)) - return V2; - return SDValue(); - }; + int NumV2Elements = + std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; }); - if (SDValue V = CanZExtLowHalf()) { - V = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, V); - V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V); - return DAG.getNode(ISD::BITCAST, DL, VT, V); + if (NumV2Elements == 0) { + // Check for being able to broadcast a single element. + if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v4f32, DL, V1, + Mask, Subtarget, DAG)) + return Broadcast; + + // Use even/odd duplicate instructions for masks that match their pattern. + if (Subtarget->hasSSE3()) { + if (isShuffleEquivalent(V1, V2, Mask, 0, 0, 2, 2)) + return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1); + if (isShuffleEquivalent(V1, V2, Mask, 1, 1, 3, 3)) + return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1); + } + + if (Subtarget->hasAVX()) { + // If we have AVX, we can use VPERMILPS which will allow folding a load + // into the shuffle. + return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1, + getV4X86ShuffleImm8ForMask(Mask, DAG)); + } + + // Otherwise, use a straight shuffle of a single input vector. We pass the + // input vector to both operands to simulate this with a SHUFPS. + return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1, + getV4X86ShuffleImm8ForMask(Mask, DAG)); } - // No viable ext lowering found. - return SDValue(); -} + // There are special ways we can lower some single-element blends. However, we + // have custom ways we can lower more complex single-element blends below that + // we defer to if both this and BLENDPS fail to match, so restrict this to + // when the V2 input is targeting element 0 of the mask -- that is the fast + // case here. + if (NumV2Elements == 1 && Mask[0] >= 4) + if (SDValue V = lowerVectorShuffleAsElementInsertion(MVT::v4f32, DL, V1, V2, + Mask, Subtarget, DAG)) + return V; -/// \brief Try to get a scalar value for a specific element of a vector. -/// -/// Looks through BUILD_VECTOR and SCALAR_TO_VECTOR nodes to find a scalar. -static SDValue getScalarValueForVectorElement(SDValue V, int Idx, - SelectionDAG &DAG) { - MVT VT = V.getSimpleValueType(); - MVT EltVT = VT.getVectorElementType(); - while (V.getOpcode() == ISD::BITCAST) - V = V.getOperand(0); - // If the bitcasts shift the element size, we can't extract an equivalent - // element from it. - MVT NewVT = V.getSimpleValueType(); - if (!NewVT.isVector() || NewVT.getScalarSizeInBits() != VT.getScalarSizeInBits()) - return SDValue(); + if (Subtarget->hasSSE41()) { + if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask, + Subtarget, DAG)) + return Blend; - if (V.getOpcode() == ISD::BUILD_VECTOR || - (Idx == 0 && V.getOpcode() == ISD::SCALAR_TO_VECTOR)) - return DAG.getNode(ISD::BITCAST, SDLoc(V), EltVT, V.getOperand(Idx)); + // Use INSERTPS if we can complete the shuffle efficiently. + if (SDValue V = lowerVectorShuffleAsInsertPS(Op, V1, V2, Mask, DAG)) + return V; - return SDValue(); -} + if (!isSingleSHUFPSMask(Mask)) + if (SDValue BlendPerm = lowerVectorShuffleAsBlendAndPermute( + DL, MVT::v4f32, V1, V2, Mask, DAG)) + return BlendPerm; + } -/// \brief Helper to test for a load that can be folded with x86 shuffles. -/// -/// This is particularly important because the set of instructions varies -/// significantly based on whether the operand is a load or not. -static bool isShuffleFoldableLoad(SDValue V) { - while (V.getOpcode() == ISD::BITCAST) - V = V.getOperand(0); + // Use dedicated unpack instructions for masks that match their pattern. + if (isShuffleEquivalent(V1, V2, Mask, 0, 4, 1, 5)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f32, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, 2, 6, 3, 7)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f32, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, 4, 0, 5, 1)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f32, V2, V1); + if (isShuffleEquivalent(V1, V2, Mask, 6, 2, 7, 3)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f32, V2, V1); - return ISD::isNON_EXTLoad(V.getNode()); + // Otherwise fall back to a SHUFPS lowering strategy. + return lowerVectorShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG); } -/// \brief Try to lower insertion of a single element into a zero vector. +/// \brief Lower 4-lane i32 vector shuffles. /// -/// This is a common pattern that we have especially efficient patterns to lower -/// across all subtarget feature sets. -static SDValue lowerVectorShuffleAsElementInsertion( - MVT VT, SDLoc DL, SDValue V1, SDValue V2, ArrayRef Mask, - const X86Subtarget *Subtarget, SelectionDAG &DAG) { - SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2); - MVT ExtVT = VT; - MVT EltVT = VT.getVectorElementType(); +/// We try to handle these with integer-domain shuffles where we can, but for +/// blends we use the floating point domain blend instructions. +static SDValue lowerV4I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); + assert(Op.getSimpleValueType() == MVT::v4i32 && "Bad shuffle type!"); + assert(V1.getSimpleValueType() == MVT::v4i32 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!"); + ShuffleVectorSDNode *SVOp = cast(Op); + ArrayRef Mask = SVOp->getMask(); + assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"); - int V2Index = std::find_if(Mask.begin(), Mask.end(), - [&Mask](int M) { return M >= (int)Mask.size(); }) - - Mask.begin(); - bool IsV1Zeroable = true; - for (int i = 0, Size = Mask.size(); i < Size; ++i) - if (i != V2Index && !Zeroable[i]) { - IsV1Zeroable = false; - break; - } + // Whenever we can lower this as a zext, that instruction is strictly faster + // than any alternative. It also allows us to fold memory operands into the + // shuffle in many cases. + if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v4i32, V1, V2, + Mask, Subtarget, DAG)) + return ZExt; - // Check for a single input from a SCALAR_TO_VECTOR node. - // FIXME: All of this should be canonicalized into INSERT_VECTOR_ELT and - // all the smarts here sunk into that routine. However, the current - // lowering of BUILD_VECTOR makes that nearly impossible until the old - // vector shuffle lowering is dead. - if (SDValue V2S = getScalarValueForVectorElement( - V2, Mask[V2Index] - Mask.size(), DAG)) { - // We need to zext the scalar if it is smaller than an i32. - V2S = DAG.getNode(ISD::BITCAST, DL, EltVT, V2S); - if (EltVT == MVT::i8 || EltVT == MVT::i16) { - // Using zext to expand a narrow element won't work for non-zero - // insertions. - if (!IsV1Zeroable) - return SDValue(); + int NumV2Elements = + std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; }); - // Zero-extend directly to i32. - ExtVT = MVT::v4i32; - V2S = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, V2S); - } - V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S); - } else if (Mask[V2Index] != (int)Mask.size() || EltVT == MVT::i8 || - EltVT == MVT::i16) { - // Either not inserting from the low element of the input or the input - // element size is too small to use VZEXT_MOVL to clear the high bits. - return SDValue(); - } + if (NumV2Elements == 0) { + // Check for being able to broadcast a single element. + if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v4i32, DL, V1, + Mask, Subtarget, DAG)) + return Broadcast; - if (!IsV1Zeroable) { - // If V1 can't be treated as a zero vector we have fewer options to lower - // this. We can't support integer vectors or non-zero targets cheaply, and - // the V1 elements can't be permuted in any way. - assert(VT == ExtVT && "Cannot change extended type when non-zeroable!"); - if (!VT.isFloatingPoint() || V2Index != 0) - return SDValue(); - SmallVector V1Mask(Mask.begin(), Mask.end()); - V1Mask[V2Index] = -1; - if (!isNoopShuffleMask(V1Mask)) - return SDValue(); - // This is essentially a special case blend operation, but if we have - // general purpose blend operations, they are always faster. Bail and let - // the rest of the lowering handle these as blends. - if (Subtarget->hasSSE41()) - return SDValue(); + // Straight shuffle of a single input vector. For everything from SSE2 + // onward this has a single fast instruction with no scary immediates. + // We coerce the shuffle pattern to be compatible with UNPCK instructions + // but we aren't actually going to use the UNPCK instruction because doing + // so prevents folding a load into this instruction or making a copy. + const int UnpackLoMask[] = {0, 0, 1, 1}; + const int UnpackHiMask[] = {2, 2, 3, 3}; + if (isShuffleEquivalent(V1, V2, Mask, 0, 0, 1, 1)) + Mask = UnpackLoMask; + else if (isShuffleEquivalent(V1, V2, Mask, 2, 2, 3, 3)) + Mask = UnpackHiMask; - // Otherwise, use MOVSD or MOVSS. - assert((EltVT == MVT::f32 || EltVT == MVT::f64) && - "Only two types of floating point element types to handle!"); - return DAG.getNode(EltVT == MVT::f32 ? X86ISD::MOVSS : X86ISD::MOVSD, DL, - ExtVT, V1, V2); + return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1, + getV4X86ShuffleImm8ForMask(Mask, DAG)); } - // This lowering only works for the low element with floating point vectors. - if (VT.isFloatingPoint() && V2Index != 0) - return SDValue(); + // Try to use shift instructions. + if (SDValue Shift = + lowerVectorShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask, DAG)) + return Shift; - V2 = DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2); - if (ExtVT != VT) - V2 = DAG.getNode(ISD::BITCAST, DL, VT, V2); + // There are special ways we can lower some single-element blends. + if (NumV2Elements == 1) + if (SDValue V = lowerVectorShuffleAsElementInsertion(MVT::v4i32, DL, V1, V2, + Mask, Subtarget, DAG)) + return V; - if (V2Index != 0) { - // If we have 4 or fewer lanes we can cheaply shuffle the element into - // the desired position. Otherwise it is more efficient to do a vector - // shift left. We know that we can do a vector shift left because all - // the inputs are zero. - if (VT.isFloatingPoint() || VT.getVectorNumElements() <= 4) { - SmallVector V2Shuffle(Mask.size(), 1); - V2Shuffle[V2Index] = 0; - V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Shuffle); - } else { - V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, V2); - V2 = DAG.getNode( - X86ISD::VSHLDQ, DL, MVT::v2i64, V2, - DAG.getConstant( - V2Index * EltVT.getSizeInBits()/8, - DAG.getTargetLoweringInfo().getScalarShiftAmountTy(MVT::v2i64))); - V2 = DAG.getNode(ISD::BITCAST, DL, VT, V2); - } - } - return V2; -} + // We have different paths for blend lowering, but they all must use the + // *exact* same predicate. + bool IsBlendSupported = Subtarget->hasSSE41(); + if (IsBlendSupported) + if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask, + Subtarget, DAG)) + return Blend; -/// \brief Try to lower broadcast of a single element. -/// -/// For convenience, this code also bundles all of the subtarget feature set -/// filtering. While a little annoying to re-dispatch on type here, there isn't -/// a convenient way to factor it out. -static SDValue lowerVectorShuffleAsBroadcast(MVT VT, SDLoc DL, SDValue V, - ArrayRef Mask, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - if (!Subtarget->hasAVX()) - return SDValue(); - if (VT.isInteger() && !Subtarget->hasAVX2()) - return SDValue(); - - // Check that the mask is a broadcast. - int BroadcastIdx = -1; - for (int M : Mask) - if (M >= 0 && BroadcastIdx == -1) - BroadcastIdx = M; - else if (M >= 0 && M != BroadcastIdx) - return SDValue(); - - assert(BroadcastIdx < (int)Mask.size() && "We only expect to be called with " - "a sorted mask where the broadcast " - "comes from V1."); - - // Go up the chain of (vector) values to try and find a scalar load that - // we can combine with the broadcast. - for (;;) { - switch (V.getOpcode()) { - case ISD::CONCAT_VECTORS: { - int OperandSize = Mask.size() / V.getNumOperands(); - V = V.getOperand(BroadcastIdx / OperandSize); - BroadcastIdx %= OperandSize; - continue; - } + if (SDValue Masked = + lowerVectorShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask, DAG)) + return Masked; - case ISD::INSERT_SUBVECTOR: { - SDValue VOuter = V.getOperand(0), VInner = V.getOperand(1); - auto ConstantIdx = dyn_cast(V.getOperand(2)); - if (!ConstantIdx) - break; + // Use dedicated unpack instructions for masks that match their pattern. + if (isShuffleEquivalent(V1, V2, Mask, 0, 4, 1, 5)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i32, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, 2, 6, 3, 7)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i32, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, 4, 0, 5, 1)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i32, V2, V1); + if (isShuffleEquivalent(V1, V2, Mask, 6, 2, 7, 3)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i32, V2, V1); - int BeginIdx = (int)ConstantIdx->getZExtValue(); - int EndIdx = - BeginIdx + (int)VInner.getValueType().getVectorNumElements(); - if (BroadcastIdx >= BeginIdx && BroadcastIdx < EndIdx) { - BroadcastIdx -= BeginIdx; - V = VInner; - } else { - V = VOuter; - } - continue; - } - } - break; - } + // Try to use byte rotation instructions. + // Its more profitable for pre-SSSE3 to use shuffles/unpacks. + if (Subtarget->hasSSSE3()) + if (SDValue Rotate = lowerVectorShuffleAsByteRotate( + DL, MVT::v4i32, V1, V2, Mask, Subtarget, DAG)) + return Rotate; - // Check if this is a broadcast of a scalar. We special case lowering - // for scalars so that we can more effectively fold with loads. - if (V.getOpcode() == ISD::BUILD_VECTOR || - (V.getOpcode() == ISD::SCALAR_TO_VECTOR && BroadcastIdx == 0)) { - V = V.getOperand(BroadcastIdx); + // If we have direct support for blends, we should lower by decomposing into + // a permute. That will be faster than the domain cross. + if (IsBlendSupported) + return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i32, V1, V2, + Mask, DAG); - // If the scalar isn't a load we can't broadcast from it in AVX1, only with - // AVX2. - if (!Subtarget->hasAVX2() && !isShuffleFoldableLoad(V)) - return SDValue(); - } else if (BroadcastIdx != 0 || !Subtarget->hasAVX2()) { - // We can't broadcast from a vector register w/o AVX2, and we can only - // broadcast from the zero-element of a vector register. - return SDValue(); - } + // Try to lower by permuting the inputs into an unpack instruction. + if (SDValue Unpack = + lowerVectorShuffleAsUnpack(MVT::v4i32, DL, V1, V2, Mask, DAG)) + return Unpack; - return DAG.getNode(X86ISD::VBROADCAST, DL, VT, V); + // We implement this with SHUFPS because it can blend from two vectors. + // Because we're going to eventually use SHUFPS, we use SHUFPS even to build + // up the inputs, bypassing domain shift penalties that we would encur if we + // directly used PSHUFD on Nehalem and older. For newer chips, this isn't + // relevant. + return DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, + DAG.getVectorShuffle( + MVT::v4f32, DL, + DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V1), + DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V2), Mask)); } -// Check for whether we can use INSERTPS to perform the shuffle. We only use -// INSERTPS when the V1 elements are already in the correct locations -// because otherwise we can just always use two SHUFPS instructions which -// are much smaller to encode than a SHUFPS and an INSERTPS. We can also -// perform INSERTPS if a single V1 element is out of place and all V2 -// elements are zeroable. -static SDValue lowerVectorShuffleAsInsertPS(SDValue Op, SDValue V1, SDValue V2, - ArrayRef Mask, - SelectionDAG &DAG) { - assert(Op.getSimpleValueType() == MVT::v4f32 && "Bad shuffle type!"); - assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); - assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"); +/// \brief Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 +/// shuffle lowering, and the most complex part. +/// +/// The lowering strategy is to try to form pairs of input lanes which are +/// targeted at the same half of the final vector, and then use a dword shuffle +/// to place them onto the right half, and finally unpack the paired lanes into +/// their final position. +/// +/// The exact breakdown of how to form these dword pairs and align them on the +/// correct sides is really tricky. See the comments within the function for +/// more of the details. +static SDValue lowerV8I16SingleInputVectorShuffle( + SDLoc DL, SDValue V, MutableArrayRef Mask, + const X86Subtarget *Subtarget, SelectionDAG &DAG) { + assert(V.getSimpleValueType() == MVT::v8i16 && "Bad input type!"); + MutableArrayRef LoMask = Mask.slice(0, 4); + MutableArrayRef HiMask = Mask.slice(4, 4); - SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2); + SmallVector LoInputs; + std::copy_if(LoMask.begin(), LoMask.end(), std::back_inserter(LoInputs), + [](int M) { return M >= 0; }); + std::sort(LoInputs.begin(), LoInputs.end()); + LoInputs.erase(std::unique(LoInputs.begin(), LoInputs.end()), LoInputs.end()); + SmallVector HiInputs; + std::copy_if(HiMask.begin(), HiMask.end(), std::back_inserter(HiInputs), + [](int M) { return M >= 0; }); + std::sort(HiInputs.begin(), HiInputs.end()); + HiInputs.erase(std::unique(HiInputs.begin(), HiInputs.end()), HiInputs.end()); + int NumLToL = + std::lower_bound(LoInputs.begin(), LoInputs.end(), 4) - LoInputs.begin(); + int NumHToL = LoInputs.size() - NumLToL; + int NumLToH = + std::lower_bound(HiInputs.begin(), HiInputs.end(), 4) - HiInputs.begin(); + int NumHToH = HiInputs.size() - NumLToH; + MutableArrayRef LToLInputs(LoInputs.data(), NumLToL); + MutableArrayRef LToHInputs(HiInputs.data(), NumLToH); + MutableArrayRef HToLInputs(LoInputs.data() + NumLToL, NumHToL); + MutableArrayRef HToHInputs(HiInputs.data() + NumLToH, NumHToH); - unsigned ZMask = 0; - int V1DstIndex = -1; - int V2DstIndex = -1; - bool V1UsedInPlace = false; + // Check for being able to broadcast a single element. + if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v8i16, DL, V, + Mask, Subtarget, DAG)) + return Broadcast; - for (int i = 0; i < 4; ++i) { - // Synthesize a zero mask from the zeroable elements (includes undefs). - if (Zeroable[i]) { - ZMask |= 1 << i; - continue; - } + // Try to use shift instructions. + if (SDValue Shift = + lowerVectorShuffleAsShift(DL, MVT::v8i16, V, V, Mask, DAG)) + return Shift; - // Flag if we use any V1 inputs in place. - if (i == Mask[i]) { - V1UsedInPlace = true; - continue; - } + // Use dedicated unpack instructions for masks that match their pattern. + if (isShuffleEquivalent(V, V, Mask, 0, 0, 1, 1, 2, 2, 3, 3)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i16, V, V); + if (isShuffleEquivalent(V, V, Mask, 4, 4, 5, 5, 6, 6, 7, 7)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i16, V, V); - // We can only insert a single non-zeroable element. - if (V1DstIndex != -1 || V2DstIndex != -1) - return SDValue(); + // Try to use byte rotation instructions. + if (SDValue Rotate = lowerVectorShuffleAsByteRotate( + DL, MVT::v8i16, V, V, Mask, Subtarget, DAG)) + return Rotate; - if (Mask[i] < 4) { - // V1 input out of place for insertion. - V1DstIndex = i; - } else { - // V2 input for insertion. - V2DstIndex = i; - } - } + // Simplify the 1-into-3 and 3-into-1 cases with a single pshufd. For all + // such inputs we can swap two of the dwords across the half mark and end up + // with <=2 inputs to each half in each half. Once there, we can fall through + // to the generic code below. For example: + // + // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h] + // Mask: [0, 1, 2, 7, 4, 5, 6, 3] -----------------> [0, 1, 4, 7, 2, 3, 6, 5] + // + // However in some very rare cases we have a 1-into-3 or 3-into-1 on one half + // and an existing 2-into-2 on the other half. In this case we may have to + // pre-shuffle the 2-into-2 half to avoid turning it into a 3-into-1 or + // 1-into-3 which could cause us to cycle endlessly fixing each side in turn. + // Fortunately, we don't have to handle anything but a 2-into-2 pattern + // because any other situation (including a 3-into-1 or 1-into-3 in the other + // half than the one we target for fixing) will be fixed when we re-enter this + // path. We will also combine away any sequence of PSHUFD instructions that + // result into a single instruction. Here is an example of the tricky case: + // + // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h] + // Mask: [3, 7, 1, 0, 2, 7, 3, 5] -THIS-IS-BAD!!!!-> [5, 7, 1, 0, 4, 7, 5, 3] + // + // This now has a 1-into-3 in the high half! Instead, we do two shuffles: + // + // Input: [a, b, c, d, e, f, g, h] PSHUFHW[0,2,1,3]-> [a, b, c, d, e, g, f, h] + // Mask: [3, 7, 1, 0, 2, 7, 3, 5] -----------------> [3, 7, 1, 0, 2, 7, 3, 6] + // + // Input: [a, b, c, d, e, g, f, h] -PSHUFD[0,2,1,3]-> [a, b, e, g, c, d, f, h] + // Mask: [3, 7, 1, 0, 2, 7, 3, 6] -----------------> [5, 7, 1, 0, 4, 7, 5, 6] + // + // The result is fine to be handled by the generic logic. + auto balanceSides = [&](ArrayRef AToAInputs, ArrayRef BToAInputs, + ArrayRef BToBInputs, ArrayRef AToBInputs, + int AOffset, int BOffset) { + assert((AToAInputs.size() == 3 || AToAInputs.size() == 1) && + "Must call this with A having 3 or 1 inputs from the A half."); + assert((BToAInputs.size() == 1 || BToAInputs.size() == 3) && + "Must call this with B having 1 or 3 inputs from the B half."); + assert(AToAInputs.size() + BToAInputs.size() == 4 && + "Must call this with either 3:1 or 1:3 inputs (summing to 4)."); - // Don't bother if we have no (non-zeroable) element for insertion. - if (V1DstIndex == -1 && V2DstIndex == -1) - return SDValue(); + // Compute the index of dword with only one word among the three inputs in + // a half by taking the sum of the half with three inputs and subtracting + // the sum of the actual three inputs. The difference is the remaining + // slot. + int ADWord, BDWord; + int &TripleDWord = AToAInputs.size() == 3 ? ADWord : BDWord; + int &OneInputDWord = AToAInputs.size() == 3 ? BDWord : ADWord; + int TripleInputOffset = AToAInputs.size() == 3 ? AOffset : BOffset; + ArrayRef TripleInputs = AToAInputs.size() == 3 ? AToAInputs : BToAInputs; + int OneInput = AToAInputs.size() == 3 ? BToAInputs[0] : AToAInputs[0]; + int TripleInputSum = 0 + 1 + 2 + 3 + (4 * TripleInputOffset); + int TripleNonInputIdx = + TripleInputSum - std::accumulate(TripleInputs.begin(), TripleInputs.end(), 0); + TripleDWord = TripleNonInputIdx / 2; - // Determine element insertion src/dst indices. The src index is from the - // start of the inserted vector, not the start of the concatenated vector. - unsigned V2SrcIndex = 0; - if (V1DstIndex != -1) { - // If we have a V1 input out of place, we use V1 as the V2 element insertion - // and don't use the original V2 at all. - V2SrcIndex = Mask[V1DstIndex]; - V2DstIndex = V1DstIndex; - V2 = V1; - } else { - V2SrcIndex = Mask[V2DstIndex] - 4; - } + // We use xor with one to compute the adjacent DWord to whichever one the + // OneInput is in. + OneInputDWord = (OneInput / 2) ^ 1; - // If no V1 inputs are used in place, then the result is created only from - // the zero mask and the V2 insertion - so remove V1 dependency. - if (!V1UsedInPlace) - V1 = DAG.getUNDEF(MVT::v4f32); + // Check for one tricky case: We're fixing a 3<-1 or a 1<-3 shuffle for AToA + // and BToA inputs. If there is also such a problem with the BToB and AToB + // inputs, we don't try to fix it necessarily -- we'll recurse and see it in + // the next pass. However, if we have a 2<-2 in the BToB and AToB inputs, it + // is essential that we don't *create* a 3<-1 as then we might oscillate. + if (BToBInputs.size() == 2 && AToBInputs.size() == 2) { + // Compute how many inputs will be flipped by swapping these DWords. We + // need + // to balance this to ensure we don't form a 3-1 shuffle in the other + // half. + int NumFlippedAToBInputs = + std::count(AToBInputs.begin(), AToBInputs.end(), 2 * ADWord) + + std::count(AToBInputs.begin(), AToBInputs.end(), 2 * ADWord + 1); + int NumFlippedBToBInputs = + std::count(BToBInputs.begin(), BToBInputs.end(), 2 * BDWord) + + std::count(BToBInputs.begin(), BToBInputs.end(), 2 * BDWord + 1); + if ((NumFlippedAToBInputs == 1 && + (NumFlippedBToBInputs == 0 || NumFlippedBToBInputs == 2)) || + (NumFlippedBToBInputs == 1 && + (NumFlippedAToBInputs == 0 || NumFlippedAToBInputs == 2))) { + // We choose whether to fix the A half or B half based on whether that + // half has zero flipped inputs. At zero, we may not be able to fix it + // with that half. We also bias towards fixing the B half because that + // will more commonly be the high half, and we have to bias one way. + auto FixFlippedInputs = [&V, &DL, &Mask, &DAG](int PinnedIdx, int DWord, + ArrayRef Inputs) { + int FixIdx = PinnedIdx ^ 1; // The adjacent slot to the pinned slot. + bool IsFixIdxInput = std::find(Inputs.begin(), Inputs.end(), + PinnedIdx ^ 1) != Inputs.end(); + // Determine whether the free index is in the flipped dword or the + // unflipped dword based on where the pinned index is. We use this bit + // in an xor to conditionally select the adjacent dword. + int FixFreeIdx = 2 * (DWord ^ (PinnedIdx / 2 == DWord)); + bool IsFixFreeIdxInput = std::find(Inputs.begin(), Inputs.end(), + FixFreeIdx) != Inputs.end(); + if (IsFixIdxInput == IsFixFreeIdxInput) + FixFreeIdx += 1; + IsFixFreeIdxInput = std::find(Inputs.begin(), Inputs.end(), + FixFreeIdx) != Inputs.end(); + assert(IsFixIdxInput != IsFixFreeIdxInput && + "We need to be changing the number of flipped inputs!"); + int PSHUFHalfMask[] = {0, 1, 2, 3}; + std::swap(PSHUFHalfMask[FixFreeIdx % 4], PSHUFHalfMask[FixIdx % 4]); + V = DAG.getNode(FixIdx < 4 ? X86ISD::PSHUFLW : X86ISD::PSHUFHW, DL, + MVT::v8i16, V, + getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DAG)); - unsigned InsertPSMask = V2SrcIndex << 6 | V2DstIndex << 4 | ZMask; - assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!"); + for (int &M : Mask) + if (M != -1 && M == FixIdx) + M = FixFreeIdx; + else if (M != -1 && M == FixFreeIdx) + M = FixIdx; + }; + if (NumFlippedBToBInputs != 0) { + int BPinnedIdx = + BToAInputs.size() == 3 ? TripleNonInputIdx : OneInput; + FixFlippedInputs(BPinnedIdx, BDWord, BToBInputs); + } else { + assert(NumFlippedAToBInputs != 0 && "Impossible given predicates!"); + int APinnedIdx = + AToAInputs.size() == 3 ? TripleNonInputIdx : OneInput; + FixFlippedInputs(APinnedIdx, ADWord, AToBInputs); + } + } + } - // Insert the V2 element into the desired position. - SDLoc DL(Op); - return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2, - DAG.getConstant(InsertPSMask, MVT::i8)); -} + int PSHUFDMask[] = {0, 1, 2, 3}; + PSHUFDMask[ADWord] = BDWord; + PSHUFDMask[BDWord] = ADWord; + V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, + DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, + DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V), + getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG))); -/// \brief Try to lower a shuffle as a permute of the inputs followed by an -/// UNPCK instruction. -/// -/// This specifically targets cases where we end up with alternating between -/// the two inputs, and so can permute them into something that feeds a single -/// UNPCK instruction. Note that this routine only targets integer vectors -/// because for floating point vectors we have a generalized SHUFPS lowering -/// strategy that handles everything that doesn't *exactly* match an unpack, -/// making this clever lowering unnecessary. -static SDValue lowerVectorShuffleAsUnpack(MVT VT, SDLoc DL, SDValue V1, - SDValue V2, ArrayRef Mask, - SelectionDAG &DAG) { - assert(!VT.isFloatingPoint() && - "This routine only supports integer vectors."); - assert(!isSingleInputShuffleMask(Mask) && - "This routine should only be used when blending two inputs."); - assert(Mask.size() >= 2 && "Single element masks are invalid."); + // Adjust the mask to match the new locations of A and B. + for (int &M : Mask) + if (M != -1 && M/2 == ADWord) + M = 2 * BDWord + M % 2; + else if (M != -1 && M/2 == BDWord) + M = 2 * ADWord + M % 2; - int Size = Mask.size(); + // Recurse back into this routine to re-compute state now that this isn't + // a 3 and 1 problem. + return DAG.getVectorShuffle(MVT::v8i16, DL, V, DAG.getUNDEF(MVT::v8i16), + Mask); + }; + if ((NumLToL == 3 && NumHToL == 1) || (NumLToL == 1 && NumHToL == 3)) + return balanceSides(LToLInputs, HToLInputs, HToHInputs, LToHInputs, 0, 4); + else if ((NumHToH == 3 && NumLToH == 1) || (NumHToH == 1 && NumLToH == 3)) + return balanceSides(HToHInputs, LToHInputs, LToLInputs, HToLInputs, 4, 0); - int NumLoInputs = std::count_if(Mask.begin(), Mask.end(), [Size](int M) { - return M >= 0 && M % Size < Size / 2; - }); - int NumHiInputs = std::count_if( - Mask.begin(), Mask.end(), [Size](int M) { return M % Size >= Size / 2; }); + // At this point there are at most two inputs to the low and high halves from + // each half. That means the inputs can always be grouped into dwords and + // those dwords can then be moved to the correct half with a dword shuffle. + // We use at most one low and one high word shuffle to collect these paired + // inputs into dwords, and finally a dword shuffle to place them. + int PSHUFLMask[4] = {-1, -1, -1, -1}; + int PSHUFHMask[4] = {-1, -1, -1, -1}; + int PSHUFDMask[4] = {-1, -1, -1, -1}; - bool UnpackLo = NumLoInputs >= NumHiInputs; + // First fix the masks for all the inputs that are staying in their + // original halves. This will then dictate the targets of the cross-half + // shuffles. + auto fixInPlaceInputs = + [&PSHUFDMask](ArrayRef InPlaceInputs, ArrayRef IncomingInputs, + MutableArrayRef SourceHalfMask, + MutableArrayRef HalfMask, int HalfOffset) { + if (InPlaceInputs.empty()) + return; + if (InPlaceInputs.size() == 1) { + SourceHalfMask[InPlaceInputs[0] - HalfOffset] = + InPlaceInputs[0] - HalfOffset; + PSHUFDMask[InPlaceInputs[0] / 2] = InPlaceInputs[0] / 2; + return; + } + if (IncomingInputs.empty()) { + // Just fix all of the in place inputs. + for (int Input : InPlaceInputs) { + SourceHalfMask[Input - HalfOffset] = Input - HalfOffset; + PSHUFDMask[Input / 2] = Input / 2; + } + return; + } - auto TryUnpack = [&](MVT UnpackVT, int Scale) { - SmallVector V1Mask(Mask.size(), -1); - SmallVector V2Mask(Mask.size(), -1); + assert(InPlaceInputs.size() == 2 && "Cannot handle 3 or 4 inputs!"); + SourceHalfMask[InPlaceInputs[0] - HalfOffset] = + InPlaceInputs[0] - HalfOffset; + // Put the second input next to the first so that they are packed into + // a dword. We find the adjacent index by toggling the low bit. + int AdjIndex = InPlaceInputs[0] ^ 1; + SourceHalfMask[AdjIndex - HalfOffset] = InPlaceInputs[1] - HalfOffset; + std::replace(HalfMask.begin(), HalfMask.end(), InPlaceInputs[1], AdjIndex); + PSHUFDMask[AdjIndex / 2] = AdjIndex / 2; + }; + fixInPlaceInputs(LToLInputs, HToLInputs, PSHUFLMask, LoMask, 0); + fixInPlaceInputs(HToHInputs, LToHInputs, PSHUFHMask, HiMask, 4); - for (int i = 0; i < Size; ++i) { - if (Mask[i] < 0) - continue; + // Now gather the cross-half inputs and place them into a free dword of + // their target half. + // FIXME: This operation could almost certainly be simplified dramatically to + // look more like the 3-1 fixing operation. + auto moveInputsToRightHalf = [&PSHUFDMask]( + MutableArrayRef IncomingInputs, ArrayRef ExistingInputs, + MutableArrayRef SourceHalfMask, MutableArrayRef HalfMask, + MutableArrayRef FinalSourceHalfMask, int SourceOffset, + int DestOffset) { + auto isWordClobbered = [](ArrayRef SourceHalfMask, int Word) { + return SourceHalfMask[Word] != -1 && SourceHalfMask[Word] != Word; + }; + auto isDWordClobbered = [&isWordClobbered](ArrayRef SourceHalfMask, + int Word) { + int LowWord = Word & ~1; + int HighWord = Word | 1; + return isWordClobbered(SourceHalfMask, LowWord) || + isWordClobbered(SourceHalfMask, HighWord); + }; - // Each element of the unpack contains Scale elements from this mask. - int UnpackIdx = i / Scale; + if (IncomingInputs.empty()) + return; - // We only handle the case where V1 feeds the first slots of the unpack. - // We rely on canonicalization to ensure this is the case. - if ((UnpackIdx % 2 == 0) != (Mask[i] < Size)) - return SDValue(); + if (ExistingInputs.empty()) { + // Map any dwords with inputs from them into the right half. + for (int Input : IncomingInputs) { + // If the source half mask maps over the inputs, turn those into + // swaps and use the swapped lane. + if (isWordClobbered(SourceHalfMask, Input - SourceOffset)) { + if (SourceHalfMask[SourceHalfMask[Input - SourceOffset]] == -1) { + SourceHalfMask[SourceHalfMask[Input - SourceOffset]] = + Input - SourceOffset; + // We have to swap the uses in our half mask in one sweep. + for (int &M : HalfMask) + if (M == SourceHalfMask[Input - SourceOffset] + SourceOffset) + M = Input; + else if (M == Input) + M = SourceHalfMask[Input - SourceOffset] + SourceOffset; + } else { + assert(SourceHalfMask[SourceHalfMask[Input - SourceOffset]] == + Input - SourceOffset && + "Previous placement doesn't match!"); + } + // Note that this correctly re-maps both when we do a swap and when + // we observe the other side of the swap above. We rely on that to + // avoid swapping the members of the input list directly. + Input = SourceHalfMask[Input - SourceOffset] + SourceOffset; + } - // Setup the mask for this input. The indexing is tricky as we have to - // handle the unpack stride. - SmallVectorImpl &VMask = (UnpackIdx % 2 == 0) ? V1Mask : V2Mask; - VMask[(UnpackIdx / 2) * Scale + i % Scale + (UnpackLo ? 0 : Size / 2)] = - Mask[i] % Size; + // Map the input's dword into the correct half. + if (PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] == -1) + PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] = Input / 2; + else + assert(PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] == + Input / 2 && + "Previous placement doesn't match!"); + } + + // And just directly shift any other-half mask elements to be same-half + // as we will have mirrored the dword containing the element into the + // same position within that half. + for (int &M : HalfMask) + if (M >= SourceOffset && M < SourceOffset + 4) { + M = M - SourceOffset + DestOffset; + assert(M >= 0 && "This should never wrap below zero!"); + } + return; } - // If we will have to shuffle both inputs to use the unpack, check whether - // we can just unpack first and shuffle the result. If so, skip this unpack. - if ((NumLoInputs == 0 || NumHiInputs == 0) && !isNoopShuffleMask(V1Mask) && - !isNoopShuffleMask(V2Mask)) - return SDValue(); - - // Shuffle the inputs into place. - V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask); - V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask); - - // Cast the inputs to the type we will use to unpack them. - V1 = DAG.getNode(ISD::BITCAST, DL, UnpackVT, V1); - V2 = DAG.getNode(ISD::BITCAST, DL, UnpackVT, V2); - - // Unpack the inputs and cast the result back to the desired type. - return DAG.getNode(ISD::BITCAST, DL, VT, - DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, - DL, UnpackVT, V1, V2)); - }; - - // We try each unpack from the largest to the smallest to try and find one - // that fits this mask. - int OrigNumElements = VT.getVectorNumElements(); - int OrigScalarSize = VT.getScalarSizeInBits(); - for (int ScalarSize = 64; ScalarSize >= OrigScalarSize; ScalarSize /= 2) { - int Scale = ScalarSize / OrigScalarSize; - int NumElements = OrigNumElements / Scale; - MVT UnpackVT = MVT::getVectorVT(MVT::getIntegerVT(ScalarSize), NumElements); - if (SDValue Unpack = TryUnpack(UnpackVT, Scale)) - return Unpack; - } - - // If none of the unpack-rooted lowerings worked (or were profitable) try an - // initial unpack. - if (NumLoInputs == 0 || NumHiInputs == 0) { - assert((NumLoInputs > 0 || NumHiInputs > 0) && - "We have to have *some* inputs!"); - int HalfOffset = NumLoInputs == 0 ? Size / 2 : 0; - - // FIXME: We could consider the total complexity of the permute of each - // possible unpacking. Or at the least we should consider how many - // half-crossings are created. - // FIXME: We could consider commuting the unpacks. - - SmallVector PermMask; - PermMask.assign(Size, -1); - for (int i = 0; i < Size; ++i) { - if (Mask[i] < 0) - continue; - - assert(Mask[i] % Size >= HalfOffset && "Found input from wrong half!"); - - PermMask[i] = - 2 * ((Mask[i] % Size) - HalfOffset) + (Mask[i] < Size ? 0 : 1); - } - return DAG.getVectorShuffle( - VT, DL, DAG.getNode(NumLoInputs == 0 ? X86ISD::UNPCKH : X86ISD::UNPCKL, - DL, VT, V1, V2), - DAG.getUNDEF(VT), PermMask); - } - - return SDValue(); -} - -/// \brief Handle lowering of 2-lane 64-bit floating point shuffles. -/// -/// This is the basis function for the 2-lane 64-bit shuffles as we have full -/// support for floating point shuffles but not integer shuffles. These -/// instructions will incur a domain crossing penalty on some chips though so -/// it is better to avoid lowering through this for integer vectors where -/// possible. -static SDValue lowerV2F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - SDLoc DL(Op); - assert(Op.getSimpleValueType() == MVT::v2f64 && "Bad shuffle type!"); - assert(V1.getSimpleValueType() == MVT::v2f64 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!"); - - if (isSingleInputShuffleMask(Mask)) { - // Use low duplicate instructions for masks that match their pattern. - if (Subtarget->hasSSE3()) - if (isShuffleEquivalent(V1, V2, Mask, 0, 0)) - return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, V1); - - // Straight shuffle of a single input vector. Simulate this by using the - // single input as both of the "inputs" to this instruction.. - unsigned SHUFPDMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1); - - if (Subtarget->hasAVX()) { - // If we have AVX, we can use VPERMILPS which will allow folding a load - // into the shuffle. - return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1, - DAG.getConstant(SHUFPDMask, MVT::i8)); - } - - return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V1, - DAG.getConstant(SHUFPDMask, MVT::i8)); - } - assert(Mask[0] >= 0 && Mask[0] < 2 && "Non-canonicalized blend!"); - assert(Mask[1] >= 2 && "Non-canonicalized blend!"); - - // If we have a single input, insert that into V1 if we can do so cheaply. - if ((Mask[0] >= 2) + (Mask[1] >= 2) == 1) { - if (SDValue Insertion = lowerVectorShuffleAsElementInsertion( - MVT::v2f64, DL, V1, V2, Mask, Subtarget, DAG)) - return Insertion; - // Try inverting the insertion since for v2 masks it is easy to do and we - // can't reliably sort the mask one way or the other. - int InverseMask[2] = {Mask[0] < 0 ? -1 : (Mask[0] ^ 2), - Mask[1] < 0 ? -1 : (Mask[1] ^ 2)}; - if (SDValue Insertion = lowerVectorShuffleAsElementInsertion( - MVT::v2f64, DL, V2, V1, InverseMask, Subtarget, DAG)) - return Insertion; - } - - // Try to use one of the special instruction patterns to handle two common - // blend patterns if a zero-blend above didn't work. - if (isShuffleEquivalent(V1, V2, Mask, 0, 3) || isShuffleEquivalent(V1, V2, Mask, 1, 3)) - if (SDValue V1S = getScalarValueForVectorElement(V1, Mask[0], DAG)) - // We can either use a special instruction to load over the low double or - // to move just the low double. - return DAG.getNode( - isShuffleFoldableLoad(V1S) ? X86ISD::MOVLPD : X86ISD::MOVSD, - DL, MVT::v2f64, V2, - DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S)); - - if (Subtarget->hasSSE41()) - if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask, - Subtarget, DAG)) - return Blend; - - // Use dedicated unpack instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, Mask, 0, 2)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2f64, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, 1, 3)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v2f64, V1, V2); - - unsigned SHUFPDMask = (Mask[0] == 1) | (((Mask[1] - 2) == 1) << 1); - return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V2, - DAG.getConstant(SHUFPDMask, MVT::i8)); -} - -/// \brief Handle lowering of 2-lane 64-bit integer shuffles. -/// -/// Tries to lower a 2-lane 64-bit shuffle using shuffle operations provided by -/// the integer unit to minimize domain crossing penalties. However, for blends -/// it falls back to the floating point shuffle operation with appropriate bit -/// casting. -static SDValue lowerV2I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - SDLoc DL(Op); - assert(Op.getSimpleValueType() == MVT::v2i64 && "Bad shuffle type!"); - assert(V1.getSimpleValueType() == MVT::v2i64 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!"); - - if (isSingleInputShuffleMask(Mask)) { - // Check for being able to broadcast a single element. - if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v2i64, DL, V1, - Mask, Subtarget, DAG)) - return Broadcast; - - // Straight shuffle of a single input vector. For everything from SSE2 - // onward this has a single fast instruction with no scary immediates. - // We have to map the mask as it is actually a v4i32 shuffle instruction. - V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V1); - int WidenedMask[4] = { - std::max(Mask[0], 0) * 2, std::max(Mask[0], 0) * 2 + 1, - std::max(Mask[1], 0) * 2, std::max(Mask[1], 0) * 2 + 1}; - return DAG.getNode( - ISD::BITCAST, DL, MVT::v2i64, - DAG.getNode(X86ISD::PSHUFD, SDLoc(Op), MVT::v4i32, V1, - getV4X86ShuffleImm8ForMask(WidenedMask, DAG))); - } - assert(Mask[0] != -1 && "No undef lanes in multi-input v2 shuffles!"); - assert(Mask[1] != -1 && "No undef lanes in multi-input v2 shuffles!"); - assert(Mask[0] < 2 && "We sort V1 to be the first input."); - assert(Mask[1] >= 2 && "We sort V2 to be the second input."); - - // If we have a blend of two PACKUS operations an the blend aligns with the - // low and half halves, we can just merge the PACKUS operations. This is - // particularly important as it lets us merge shuffles that this routine itself - // creates. - auto GetPackNode = [](SDValue V) { - while (V.getOpcode() == ISD::BITCAST) - V = V.getOperand(0); - - return V.getOpcode() == X86ISD::PACKUS ? V : SDValue(); - }; - if (SDValue V1Pack = GetPackNode(V1)) - if (SDValue V2Pack = GetPackNode(V2)) - return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, - DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, - Mask[0] == 0 ? V1Pack.getOperand(0) - : V1Pack.getOperand(1), - Mask[1] == 2 ? V2Pack.getOperand(0) - : V2Pack.getOperand(1))); - - // Try to use shift instructions. - if (SDValue Shift = - lowerVectorShuffleAsShift(DL, MVT::v2i64, V1, V2, Mask, DAG)) - return Shift; - - // When loading a scalar and then shuffling it into a vector we can often do - // the insertion cheaply. - if (SDValue Insertion = lowerVectorShuffleAsElementInsertion( - MVT::v2i64, DL, V1, V2, Mask, Subtarget, DAG)) - return Insertion; - // Try inverting the insertion since for v2 masks it is easy to do and we - // can't reliably sort the mask one way or the other. - int InverseMask[2] = {Mask[0] ^ 2, Mask[1] ^ 2}; - if (SDValue Insertion = lowerVectorShuffleAsElementInsertion( - MVT::v2i64, DL, V2, V1, InverseMask, Subtarget, DAG)) - return Insertion; - - // We have different paths for blend lowering, but they all must use the - // *exact* same predicate. - bool IsBlendSupported = Subtarget->hasSSE41(); - if (IsBlendSupported) - if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask, - Subtarget, DAG)) - return Blend; - - // Use dedicated unpack instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, Mask, 0, 2)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, 1, 3)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v2i64, V1, V2); - - // Try to use byte rotation instructions. - // Its more profitable for pre-SSSE3 to use shuffles/unpacks. - if (Subtarget->hasSSSE3()) - if (SDValue Rotate = lowerVectorShuffleAsByteRotate( - DL, MVT::v2i64, V1, V2, Mask, Subtarget, DAG)) - return Rotate; - - // If we have direct support for blends, we should lower by decomposing into - // a permute. That will be faster than the domain cross. - if (IsBlendSupported) - return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v2i64, V1, V2, - Mask, DAG); - - // We implement this with SHUFPD which is pretty lame because it will likely - // incur 2 cycles of stall for integer vectors on Nehalem and older chips. - // However, all the alternatives are still more cycles and newer chips don't - // have this problem. It would be really nice if x86 had better shuffles here. - V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V1); - V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V2); - return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, - DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask)); -} - -/// \brief Test whether this can be lowered with a single SHUFPS instruction. -/// -/// This is used to disable more specialized lowerings when the shufps lowering -/// will happen to be efficient. -static bool isSingleSHUFPSMask(ArrayRef Mask) { - // This routine only handles 128-bit shufps. - assert(Mask.size() == 4 && "Unsupported mask size!"); - - // To lower with a single SHUFPS we need to have the low half and high half - // each requiring a single input. - if (Mask[0] != -1 && Mask[1] != -1 && (Mask[0] < 4) != (Mask[1] < 4)) - return false; - if (Mask[2] != -1 && Mask[3] != -1 && (Mask[2] < 4) != (Mask[3] < 4)) - return false; - - return true; -} - -/// \brief Lower a vector shuffle using the SHUFPS instruction. -/// -/// This is a helper routine dedicated to lowering vector shuffles using SHUFPS. -/// It makes no assumptions about whether this is the *best* lowering, it simply -/// uses it. -static SDValue lowerVectorShuffleWithSHUFPS(SDLoc DL, MVT VT, - ArrayRef Mask, SDValue V1, - SDValue V2, SelectionDAG &DAG) { - SDValue LowV = V1, HighV = V2; - int NewMask[4] = {Mask[0], Mask[1], Mask[2], Mask[3]}; - - int NumV2Elements = - std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; }); - - if (NumV2Elements == 1) { - int V2Index = - std::find_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; }) - - Mask.begin(); - - // Compute the index adjacent to V2Index and in the same half by toggling - // the low bit. - int V2AdjIndex = V2Index ^ 1; - - if (Mask[V2AdjIndex] == -1) { - // Handles all the cases where we have a single V2 element and an undef. - // This will only ever happen in the high lanes because we commute the - // vector otherwise. - if (V2Index < 2) - std::swap(LowV, HighV); - NewMask[V2Index] -= 4; - } else { - // Handle the case where the V2 element ends up adjacent to a V1 element. - // To make this work, blend them together as the first step. - int V1Index = V2AdjIndex; - int BlendMask[4] = {Mask[V2Index] - 4, 0, Mask[V1Index], 0}; - V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1, - getV4X86ShuffleImm8ForMask(BlendMask, DAG)); - - // Now proceed to reconstruct the final blend as we have the necessary - // high or low half formed. - if (V2Index < 2) { - LowV = V2; - HighV = V1; - } else { - HighV = V2; - } - NewMask[V1Index] = 2; // We put the V1 element in V2[2]. - NewMask[V2Index] = 0; // We shifted the V2 element into V2[0]. - } - } else if (NumV2Elements == 2) { - if (Mask[0] < 4 && Mask[1] < 4) { - // Handle the easy case where we have V1 in the low lanes and V2 in the - // high lanes. - NewMask[2] -= 4; - NewMask[3] -= 4; - } else if (Mask[2] < 4 && Mask[3] < 4) { - // We also handle the reversed case because this utility may get called - // when we detect a SHUFPS pattern but can't easily commute the shuffle to - // arrange things in the right direction. - NewMask[0] -= 4; - NewMask[1] -= 4; - HighV = V1; - LowV = V2; - } else { - // We have a mixture of V1 and V2 in both low and high lanes. Rather than - // trying to place elements directly, just blend them and set up the final - // shuffle to place them. - - // The first two blend mask elements are for V1, the second two are for - // V2. - int BlendMask[4] = {Mask[0] < 4 ? Mask[0] : Mask[1], - Mask[2] < 4 ? Mask[2] : Mask[3], - (Mask[0] >= 4 ? Mask[0] : Mask[1]) - 4, - (Mask[2] >= 4 ? Mask[2] : Mask[3]) - 4}; - V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2, - getV4X86ShuffleImm8ForMask(BlendMask, DAG)); - - // Now we do a normal shuffle of V1 by giving V1 as both operands to - // a blend. - LowV = HighV = V1; - NewMask[0] = Mask[0] < 4 ? 0 : 2; - NewMask[1] = Mask[0] < 4 ? 2 : 0; - NewMask[2] = Mask[2] < 4 ? 1 : 3; - NewMask[3] = Mask[2] < 4 ? 3 : 1; - } - } - return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV, - getV4X86ShuffleImm8ForMask(NewMask, DAG)); -} - -/// \brief Lower 4-lane 32-bit floating point shuffles. -/// -/// Uses instructions exclusively from the floating point unit to minimize -/// domain crossing penalties, as these are sufficient to implement all v4f32 -/// shuffles. -static SDValue lowerV4F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - SDLoc DL(Op); - assert(Op.getSimpleValueType() == MVT::v4f32 && "Bad shuffle type!"); - assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"); - - int NumV2Elements = - std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; }); - - if (NumV2Elements == 0) { - // Check for being able to broadcast a single element. - if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v4f32, DL, V1, - Mask, Subtarget, DAG)) - return Broadcast; - - // Use even/odd duplicate instructions for masks that match their pattern. - if (Subtarget->hasSSE3()) { - if (isShuffleEquivalent(V1, V2, Mask, 0, 0, 2, 2)) - return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1); - if (isShuffleEquivalent(V1, V2, Mask, 1, 1, 3, 3)) - return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1); - } - - if (Subtarget->hasAVX()) { - // If we have AVX, we can use VPERMILPS which will allow folding a load - // into the shuffle. - return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1, - getV4X86ShuffleImm8ForMask(Mask, DAG)); - } - - // Otherwise, use a straight shuffle of a single input vector. We pass the - // input vector to both operands to simulate this with a SHUFPS. - return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1, - getV4X86ShuffleImm8ForMask(Mask, DAG)); - } - - // There are special ways we can lower some single-element blends. However, we - // have custom ways we can lower more complex single-element blends below that - // we defer to if both this and BLENDPS fail to match, so restrict this to - // when the V2 input is targeting element 0 of the mask -- that is the fast - // case here. - if (NumV2Elements == 1 && Mask[0] >= 4) - if (SDValue V = lowerVectorShuffleAsElementInsertion(MVT::v4f32, DL, V1, V2, - Mask, Subtarget, DAG)) - return V; - - if (Subtarget->hasSSE41()) { - if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask, - Subtarget, DAG)) - return Blend; - - // Use INSERTPS if we can complete the shuffle efficiently. - if (SDValue V = lowerVectorShuffleAsInsertPS(Op, V1, V2, Mask, DAG)) - return V; - - if (!isSingleSHUFPSMask(Mask)) - if (SDValue BlendPerm = lowerVectorShuffleAsBlendAndPermute( - DL, MVT::v4f32, V1, V2, Mask, DAG)) - return BlendPerm; - } - - // Use dedicated unpack instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, Mask, 0, 4, 1, 5)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f32, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, 2, 6, 3, 7)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f32, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, 4, 0, 5, 1)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f32, V2, V1); - if (isShuffleEquivalent(V1, V2, Mask, 6, 2, 7, 3)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f32, V2, V1); - - // Otherwise fall back to a SHUFPS lowering strategy. - return lowerVectorShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG); -} - -/// \brief Lower 4-lane i32 vector shuffles. -/// -/// We try to handle these with integer-domain shuffles where we can, but for -/// blends we use the floating point domain blend instructions. -static SDValue lowerV4I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - SDLoc DL(Op); - assert(Op.getSimpleValueType() == MVT::v4i32 && "Bad shuffle type!"); - assert(V1.getSimpleValueType() == MVT::v4i32 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"); - - // Whenever we can lower this as a zext, that instruction is strictly faster - // than any alternative. It also allows us to fold memory operands into the - // shuffle in many cases. - if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v4i32, V1, V2, - Mask, Subtarget, DAG)) - return ZExt; - - int NumV2Elements = - std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; }); - - if (NumV2Elements == 0) { - // Check for being able to broadcast a single element. - if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v4i32, DL, V1, - Mask, Subtarget, DAG)) - return Broadcast; - - // Straight shuffle of a single input vector. For everything from SSE2 - // onward this has a single fast instruction with no scary immediates. - // We coerce the shuffle pattern to be compatible with UNPCK instructions - // but we aren't actually going to use the UNPCK instruction because doing - // so prevents folding a load into this instruction or making a copy. - const int UnpackLoMask[] = {0, 0, 1, 1}; - const int UnpackHiMask[] = {2, 2, 3, 3}; - if (isShuffleEquivalent(V1, V2, Mask, 0, 0, 1, 1)) - Mask = UnpackLoMask; - else if (isShuffleEquivalent(V1, V2, Mask, 2, 2, 3, 3)) - Mask = UnpackHiMask; - - return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1, - getV4X86ShuffleImm8ForMask(Mask, DAG)); - } - - // Try to use shift instructions. - if (SDValue Shift = - lowerVectorShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask, DAG)) - return Shift; - - // There are special ways we can lower some single-element blends. - if (NumV2Elements == 1) - if (SDValue V = lowerVectorShuffleAsElementInsertion(MVT::v4i32, DL, V1, V2, - Mask, Subtarget, DAG)) - return V; - - // We have different paths for blend lowering, but they all must use the - // *exact* same predicate. - bool IsBlendSupported = Subtarget->hasSSE41(); - if (IsBlendSupported) - if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask, - Subtarget, DAG)) - return Blend; - - if (SDValue Masked = - lowerVectorShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask, DAG)) - return Masked; - - // Use dedicated unpack instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, Mask, 0, 4, 1, 5)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i32, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, 2, 6, 3, 7)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i32, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, 4, 0, 5, 1)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i32, V2, V1); - if (isShuffleEquivalent(V1, V2, Mask, 6, 2, 7, 3)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i32, V2, V1); - - // Try to use byte rotation instructions. - // Its more profitable for pre-SSSE3 to use shuffles/unpacks. - if (Subtarget->hasSSSE3()) - if (SDValue Rotate = lowerVectorShuffleAsByteRotate( - DL, MVT::v4i32, V1, V2, Mask, Subtarget, DAG)) - return Rotate; - - // If we have direct support for blends, we should lower by decomposing into - // a permute. That will be faster than the domain cross. - if (IsBlendSupported) - return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i32, V1, V2, - Mask, DAG); - - // Try to lower by permuting the inputs into an unpack instruction. - if (SDValue Unpack = - lowerVectorShuffleAsUnpack(MVT::v4i32, DL, V1, V2, Mask, DAG)) - return Unpack; - - // We implement this with SHUFPS because it can blend from two vectors. - // Because we're going to eventually use SHUFPS, we use SHUFPS even to build - // up the inputs, bypassing domain shift penalties that we would encur if we - // directly used PSHUFD on Nehalem and older. For newer chips, this isn't - // relevant. - return DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, - DAG.getVectorShuffle( - MVT::v4f32, DL, - DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V1), - DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V2), Mask)); -} - -/// \brief Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 -/// shuffle lowering, and the most complex part. -/// -/// The lowering strategy is to try to form pairs of input lanes which are -/// targeted at the same half of the final vector, and then use a dword shuffle -/// to place them onto the right half, and finally unpack the paired lanes into -/// their final position. -/// -/// The exact breakdown of how to form these dword pairs and align them on the -/// correct sides is really tricky. See the comments within the function for -/// more of the details. -static SDValue lowerV8I16SingleInputVectorShuffle( - SDLoc DL, SDValue V, MutableArrayRef Mask, - const X86Subtarget *Subtarget, SelectionDAG &DAG) { - assert(V.getSimpleValueType() == MVT::v8i16 && "Bad input type!"); - MutableArrayRef LoMask = Mask.slice(0, 4); - MutableArrayRef HiMask = Mask.slice(4, 4); - - SmallVector LoInputs; - std::copy_if(LoMask.begin(), LoMask.end(), std::back_inserter(LoInputs), - [](int M) { return M >= 0; }); - std::sort(LoInputs.begin(), LoInputs.end()); - LoInputs.erase(std::unique(LoInputs.begin(), LoInputs.end()), LoInputs.end()); - SmallVector HiInputs; - std::copy_if(HiMask.begin(), HiMask.end(), std::back_inserter(HiInputs), - [](int M) { return M >= 0; }); - std::sort(HiInputs.begin(), HiInputs.end()); - HiInputs.erase(std::unique(HiInputs.begin(), HiInputs.end()), HiInputs.end()); - int NumLToL = - std::lower_bound(LoInputs.begin(), LoInputs.end(), 4) - LoInputs.begin(); - int NumHToL = LoInputs.size() - NumLToL; - int NumLToH = - std::lower_bound(HiInputs.begin(), HiInputs.end(), 4) - HiInputs.begin(); - int NumHToH = HiInputs.size() - NumLToH; - MutableArrayRef LToLInputs(LoInputs.data(), NumLToL); - MutableArrayRef LToHInputs(HiInputs.data(), NumLToH); - MutableArrayRef HToLInputs(LoInputs.data() + NumLToL, NumHToL); - MutableArrayRef HToHInputs(HiInputs.data() + NumLToH, NumHToH); - - // Check for being able to broadcast a single element. - if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v8i16, DL, V, - Mask, Subtarget, DAG)) - return Broadcast; - - // Try to use shift instructions. - if (SDValue Shift = - lowerVectorShuffleAsShift(DL, MVT::v8i16, V, V, Mask, DAG)) - return Shift; - - // Use dedicated unpack instructions for masks that match their pattern. - if (isShuffleEquivalent(V, V, Mask, 0, 0, 1, 1, 2, 2, 3, 3)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i16, V, V); - if (isShuffleEquivalent(V, V, Mask, 4, 4, 5, 5, 6, 6, 7, 7)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i16, V, V); - - // Try to use byte rotation instructions. - if (SDValue Rotate = lowerVectorShuffleAsByteRotate( - DL, MVT::v8i16, V, V, Mask, Subtarget, DAG)) - return Rotate; - - // Simplify the 1-into-3 and 3-into-1 cases with a single pshufd. For all - // such inputs we can swap two of the dwords across the half mark and end up - // with <=2 inputs to each half in each half. Once there, we can fall through - // to the generic code below. For example: - // - // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h] - // Mask: [0, 1, 2, 7, 4, 5, 6, 3] -----------------> [0, 1, 4, 7, 2, 3, 6, 5] - // - // However in some very rare cases we have a 1-into-3 or 3-into-1 on one half - // and an existing 2-into-2 on the other half. In this case we may have to - // pre-shuffle the 2-into-2 half to avoid turning it into a 3-into-1 or - // 1-into-3 which could cause us to cycle endlessly fixing each side in turn. - // Fortunately, we don't have to handle anything but a 2-into-2 pattern - // because any other situation (including a 3-into-1 or 1-into-3 in the other - // half than the one we target for fixing) will be fixed when we re-enter this - // path. We will also combine away any sequence of PSHUFD instructions that - // result into a single instruction. Here is an example of the tricky case: - // - // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h] - // Mask: [3, 7, 1, 0, 2, 7, 3, 5] -THIS-IS-BAD!!!!-> [5, 7, 1, 0, 4, 7, 5, 3] - // - // This now has a 1-into-3 in the high half! Instead, we do two shuffles: - // - // Input: [a, b, c, d, e, f, g, h] PSHUFHW[0,2,1,3]-> [a, b, c, d, e, g, f, h] - // Mask: [3, 7, 1, 0, 2, 7, 3, 5] -----------------> [3, 7, 1, 0, 2, 7, 3, 6] - // - // Input: [a, b, c, d, e, g, f, h] -PSHUFD[0,2,1,3]-> [a, b, e, g, c, d, f, h] - // Mask: [3, 7, 1, 0, 2, 7, 3, 6] -----------------> [5, 7, 1, 0, 4, 7, 5, 6] - // - // The result is fine to be handled by the generic logic. - auto balanceSides = [&](ArrayRef AToAInputs, ArrayRef BToAInputs, - ArrayRef BToBInputs, ArrayRef AToBInputs, - int AOffset, int BOffset) { - assert((AToAInputs.size() == 3 || AToAInputs.size() == 1) && - "Must call this with A having 3 or 1 inputs from the A half."); - assert((BToAInputs.size() == 1 || BToAInputs.size() == 3) && - "Must call this with B having 1 or 3 inputs from the B half."); - assert(AToAInputs.size() + BToAInputs.size() == 4 && - "Must call this with either 3:1 or 1:3 inputs (summing to 4)."); - - // Compute the index of dword with only one word among the three inputs in - // a half by taking the sum of the half with three inputs and subtracting - // the sum of the actual three inputs. The difference is the remaining - // slot. - int ADWord, BDWord; - int &TripleDWord = AToAInputs.size() == 3 ? ADWord : BDWord; - int &OneInputDWord = AToAInputs.size() == 3 ? BDWord : ADWord; - int TripleInputOffset = AToAInputs.size() == 3 ? AOffset : BOffset; - ArrayRef TripleInputs = AToAInputs.size() == 3 ? AToAInputs : BToAInputs; - int OneInput = AToAInputs.size() == 3 ? BToAInputs[0] : AToAInputs[0]; - int TripleInputSum = 0 + 1 + 2 + 3 + (4 * TripleInputOffset); - int TripleNonInputIdx = - TripleInputSum - std::accumulate(TripleInputs.begin(), TripleInputs.end(), 0); - TripleDWord = TripleNonInputIdx / 2; - - // We use xor with one to compute the adjacent DWord to whichever one the - // OneInput is in. - OneInputDWord = (OneInput / 2) ^ 1; - - // Check for one tricky case: We're fixing a 3<-1 or a 1<-3 shuffle for AToA - // and BToA inputs. If there is also such a problem with the BToB and AToB - // inputs, we don't try to fix it necessarily -- we'll recurse and see it in - // the next pass. However, if we have a 2<-2 in the BToB and AToB inputs, it - // is essential that we don't *create* a 3<-1 as then we might oscillate. - if (BToBInputs.size() == 2 && AToBInputs.size() == 2) { - // Compute how many inputs will be flipped by swapping these DWords. We - // need - // to balance this to ensure we don't form a 3-1 shuffle in the other - // half. - int NumFlippedAToBInputs = - std::count(AToBInputs.begin(), AToBInputs.end(), 2 * ADWord) + - std::count(AToBInputs.begin(), AToBInputs.end(), 2 * ADWord + 1); - int NumFlippedBToBInputs = - std::count(BToBInputs.begin(), BToBInputs.end(), 2 * BDWord) + - std::count(BToBInputs.begin(), BToBInputs.end(), 2 * BDWord + 1); - if ((NumFlippedAToBInputs == 1 && - (NumFlippedBToBInputs == 0 || NumFlippedBToBInputs == 2)) || - (NumFlippedBToBInputs == 1 && - (NumFlippedAToBInputs == 0 || NumFlippedAToBInputs == 2))) { - // We choose whether to fix the A half or B half based on whether that - // half has zero flipped inputs. At zero, we may not be able to fix it - // with that half. We also bias towards fixing the B half because that - // will more commonly be the high half, and we have to bias one way. - auto FixFlippedInputs = [&V, &DL, &Mask, &DAG](int PinnedIdx, int DWord, - ArrayRef Inputs) { - int FixIdx = PinnedIdx ^ 1; // The adjacent slot to the pinned slot. - bool IsFixIdxInput = std::find(Inputs.begin(), Inputs.end(), - PinnedIdx ^ 1) != Inputs.end(); - // Determine whether the free index is in the flipped dword or the - // unflipped dword based on where the pinned index is. We use this bit - // in an xor to conditionally select the adjacent dword. - int FixFreeIdx = 2 * (DWord ^ (PinnedIdx / 2 == DWord)); - bool IsFixFreeIdxInput = std::find(Inputs.begin(), Inputs.end(), - FixFreeIdx) != Inputs.end(); - if (IsFixIdxInput == IsFixFreeIdxInput) - FixFreeIdx += 1; - IsFixFreeIdxInput = std::find(Inputs.begin(), Inputs.end(), - FixFreeIdx) != Inputs.end(); - assert(IsFixIdxInput != IsFixFreeIdxInput && - "We need to be changing the number of flipped inputs!"); - int PSHUFHalfMask[] = {0, 1, 2, 3}; - std::swap(PSHUFHalfMask[FixFreeIdx % 4], PSHUFHalfMask[FixIdx % 4]); - V = DAG.getNode(FixIdx < 4 ? X86ISD::PSHUFLW : X86ISD::PSHUFHW, DL, - MVT::v8i16, V, - getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DAG)); - - for (int &M : Mask) - if (M != -1 && M == FixIdx) - M = FixFreeIdx; - else if (M != -1 && M == FixFreeIdx) - M = FixIdx; - }; - if (NumFlippedBToBInputs != 0) { - int BPinnedIdx = - BToAInputs.size() == 3 ? TripleNonInputIdx : OneInput; - FixFlippedInputs(BPinnedIdx, BDWord, BToBInputs); - } else { - assert(NumFlippedAToBInputs != 0 && "Impossible given predicates!"); - int APinnedIdx = - AToAInputs.size() == 3 ? TripleNonInputIdx : OneInput; - FixFlippedInputs(APinnedIdx, ADWord, AToBInputs); - } - } - } - - int PSHUFDMask[] = {0, 1, 2, 3}; - PSHUFDMask[ADWord] = BDWord; - PSHUFDMask[BDWord] = ADWord; - V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, - DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, - DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V), - getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG))); - - // Adjust the mask to match the new locations of A and B. - for (int &M : Mask) - if (M != -1 && M/2 == ADWord) - M = 2 * BDWord + M % 2; - else if (M != -1 && M/2 == BDWord) - M = 2 * ADWord + M % 2; - - // Recurse back into this routine to re-compute state now that this isn't - // a 3 and 1 problem. - return DAG.getVectorShuffle(MVT::v8i16, DL, V, DAG.getUNDEF(MVT::v8i16), - Mask); - }; - if ((NumLToL == 3 && NumHToL == 1) || (NumLToL == 1 && NumHToL == 3)) - return balanceSides(LToLInputs, HToLInputs, HToHInputs, LToHInputs, 0, 4); - else if ((NumHToH == 3 && NumLToH == 1) || (NumHToH == 1 && NumLToH == 3)) - return balanceSides(HToHInputs, LToHInputs, LToLInputs, HToLInputs, 4, 0); - - // At this point there are at most two inputs to the low and high halves from - // each half. That means the inputs can always be grouped into dwords and - // those dwords can then be moved to the correct half with a dword shuffle. - // We use at most one low and one high word shuffle to collect these paired - // inputs into dwords, and finally a dword shuffle to place them. - int PSHUFLMask[4] = {-1, -1, -1, -1}; - int PSHUFHMask[4] = {-1, -1, -1, -1}; - int PSHUFDMask[4] = {-1, -1, -1, -1}; - - // First fix the masks for all the inputs that are staying in their - // original halves. This will then dictate the targets of the cross-half - // shuffles. - auto fixInPlaceInputs = - [&PSHUFDMask](ArrayRef InPlaceInputs, ArrayRef IncomingInputs, - MutableArrayRef SourceHalfMask, - MutableArrayRef HalfMask, int HalfOffset) { - if (InPlaceInputs.empty()) - return; - if (InPlaceInputs.size() == 1) { - SourceHalfMask[InPlaceInputs[0] - HalfOffset] = - InPlaceInputs[0] - HalfOffset; - PSHUFDMask[InPlaceInputs[0] / 2] = InPlaceInputs[0] / 2; - return; - } - if (IncomingInputs.empty()) { - // Just fix all of the in place inputs. - for (int Input : InPlaceInputs) { - SourceHalfMask[Input - HalfOffset] = Input - HalfOffset; - PSHUFDMask[Input / 2] = Input / 2; - } - return; - } - - assert(InPlaceInputs.size() == 2 && "Cannot handle 3 or 4 inputs!"); - SourceHalfMask[InPlaceInputs[0] - HalfOffset] = - InPlaceInputs[0] - HalfOffset; - // Put the second input next to the first so that they are packed into - // a dword. We find the adjacent index by toggling the low bit. - int AdjIndex = InPlaceInputs[0] ^ 1; - SourceHalfMask[AdjIndex - HalfOffset] = InPlaceInputs[1] - HalfOffset; - std::replace(HalfMask.begin(), HalfMask.end(), InPlaceInputs[1], AdjIndex); - PSHUFDMask[AdjIndex / 2] = AdjIndex / 2; - }; - fixInPlaceInputs(LToLInputs, HToLInputs, PSHUFLMask, LoMask, 0); - fixInPlaceInputs(HToHInputs, LToHInputs, PSHUFHMask, HiMask, 4); - - // Now gather the cross-half inputs and place them into a free dword of - // their target half. - // FIXME: This operation could almost certainly be simplified dramatically to - // look more like the 3-1 fixing operation. - auto moveInputsToRightHalf = [&PSHUFDMask]( - MutableArrayRef IncomingInputs, ArrayRef ExistingInputs, - MutableArrayRef SourceHalfMask, MutableArrayRef HalfMask, - MutableArrayRef FinalSourceHalfMask, int SourceOffset, - int DestOffset) { - auto isWordClobbered = [](ArrayRef SourceHalfMask, int Word) { - return SourceHalfMask[Word] != -1 && SourceHalfMask[Word] != Word; - }; - auto isDWordClobbered = [&isWordClobbered](ArrayRef SourceHalfMask, - int Word) { - int LowWord = Word & ~1; - int HighWord = Word | 1; - return isWordClobbered(SourceHalfMask, LowWord) || - isWordClobbered(SourceHalfMask, HighWord); - }; - - if (IncomingInputs.empty()) - return; - - if (ExistingInputs.empty()) { - // Map any dwords with inputs from them into the right half. - for (int Input : IncomingInputs) { - // If the source half mask maps over the inputs, turn those into - // swaps and use the swapped lane. - if (isWordClobbered(SourceHalfMask, Input - SourceOffset)) { - if (SourceHalfMask[SourceHalfMask[Input - SourceOffset]] == -1) { - SourceHalfMask[SourceHalfMask[Input - SourceOffset]] = - Input - SourceOffset; - // We have to swap the uses in our half mask in one sweep. - for (int &M : HalfMask) - if (M == SourceHalfMask[Input - SourceOffset] + SourceOffset) - M = Input; - else if (M == Input) - M = SourceHalfMask[Input - SourceOffset] + SourceOffset; - } else { - assert(SourceHalfMask[SourceHalfMask[Input - SourceOffset]] == - Input - SourceOffset && - "Previous placement doesn't match!"); - } - // Note that this correctly re-maps both when we do a swap and when - // we observe the other side of the swap above. We rely on that to - // avoid swapping the members of the input list directly. - Input = SourceHalfMask[Input - SourceOffset] + SourceOffset; - } - - // Map the input's dword into the correct half. - if (PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] == -1) - PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] = Input / 2; - else - assert(PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] == - Input / 2 && - "Previous placement doesn't match!"); - } - - // And just directly shift any other-half mask elements to be same-half - // as we will have mirrored the dword containing the element into the - // same position within that half. - for (int &M : HalfMask) - if (M >= SourceOffset && M < SourceOffset + 4) { - M = M - SourceOffset + DestOffset; - assert(M >= 0 && "This should never wrap below zero!"); - } - return; - } - - // Ensure we have the input in a viable dword of its current half. This - // is particularly tricky because the original position may be clobbered - // by inputs being moved and *staying* in that half. - if (IncomingInputs.size() == 1) { - if (isWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) { - int InputFixed = std::find(std::begin(SourceHalfMask), - std::end(SourceHalfMask), -1) - - std::begin(SourceHalfMask) + SourceOffset; - SourceHalfMask[InputFixed - SourceOffset] = - IncomingInputs[0] - SourceOffset; - std::replace(HalfMask.begin(), HalfMask.end(), IncomingInputs[0], - InputFixed); - IncomingInputs[0] = InputFixed; - } - } else if (IncomingInputs.size() == 2) { - if (IncomingInputs[0] / 2 != IncomingInputs[1] / 2 || - isDWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) { - // We have two non-adjacent or clobbered inputs we need to extract from - // the source half. To do this, we need to map them into some adjacent - // dword slot in the source mask. - int InputsFixed[2] = {IncomingInputs[0] - SourceOffset, - IncomingInputs[1] - SourceOffset}; - - // If there is a free slot in the source half mask adjacent to one of - // the inputs, place the other input in it. We use (Index XOR 1) to - // compute an adjacent index. - if (!isWordClobbered(SourceHalfMask, InputsFixed[0]) && - SourceHalfMask[InputsFixed[0] ^ 1] == -1) { - SourceHalfMask[InputsFixed[0]] = InputsFixed[0]; - SourceHalfMask[InputsFixed[0] ^ 1] = InputsFixed[1]; - InputsFixed[1] = InputsFixed[0] ^ 1; - } else if (!isWordClobbered(SourceHalfMask, InputsFixed[1]) && - SourceHalfMask[InputsFixed[1] ^ 1] == -1) { - SourceHalfMask[InputsFixed[1]] = InputsFixed[1]; - SourceHalfMask[InputsFixed[1] ^ 1] = InputsFixed[0]; - InputsFixed[0] = InputsFixed[1] ^ 1; - } else if (SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1)] == -1 && - SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1) + 1] == -1) { - // The two inputs are in the same DWord but it is clobbered and the - // adjacent DWord isn't used at all. Move both inputs to the free - // slot. - SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1)] = InputsFixed[0]; - SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1) + 1] = InputsFixed[1]; - InputsFixed[0] = 2 * ((InputsFixed[0] / 2) ^ 1); - InputsFixed[1] = 2 * ((InputsFixed[0] / 2) ^ 1) + 1; - } else { - // The only way we hit this point is if there is no clobbering - // (because there are no off-half inputs to this half) and there is no - // free slot adjacent to one of the inputs. In this case, we have to - // swap an input with a non-input. - for (int i = 0; i < 4; ++i) - assert((SourceHalfMask[i] == -1 || SourceHalfMask[i] == i) && - "We can't handle any clobbers here!"); - assert(InputsFixed[1] != (InputsFixed[0] ^ 1) && - "Cannot have adjacent inputs here!"); - - SourceHalfMask[InputsFixed[0] ^ 1] = InputsFixed[1]; - SourceHalfMask[InputsFixed[1]] = InputsFixed[0] ^ 1; - - // We also have to update the final source mask in this case because - // it may need to undo the above swap. - for (int &M : FinalSourceHalfMask) - if (M == (InputsFixed[0] ^ 1) + SourceOffset) - M = InputsFixed[1] + SourceOffset; - else if (M == InputsFixed[1] + SourceOffset) - M = (InputsFixed[0] ^ 1) + SourceOffset; - - InputsFixed[1] = InputsFixed[0] ^ 1; - } - - // Point everything at the fixed inputs. - for (int &M : HalfMask) - if (M == IncomingInputs[0]) - M = InputsFixed[0] + SourceOffset; - else if (M == IncomingInputs[1]) - M = InputsFixed[1] + SourceOffset; - - IncomingInputs[0] = InputsFixed[0] + SourceOffset; - IncomingInputs[1] = InputsFixed[1] + SourceOffset; - } - } else { - llvm_unreachable("Unhandled input size!"); - } - - // Now hoist the DWord down to the right half. - int FreeDWord = (PSHUFDMask[DestOffset / 2] == -1 ? 0 : 1) + DestOffset / 2; - assert(PSHUFDMask[FreeDWord] == -1 && "DWord not free"); - PSHUFDMask[FreeDWord] = IncomingInputs[0] / 2; - for (int &M : HalfMask) - for (int Input : IncomingInputs) - if (M == Input) - M = FreeDWord * 2 + Input % 2; - }; - moveInputsToRightHalf(HToLInputs, LToLInputs, PSHUFHMask, LoMask, HiMask, - /*SourceOffset*/ 4, /*DestOffset*/ 0); - moveInputsToRightHalf(LToHInputs, HToHInputs, PSHUFLMask, HiMask, LoMask, - /*SourceOffset*/ 0, /*DestOffset*/ 4); - - // Now enact all the shuffles we've computed to move the inputs into their - // target half. - if (!isNoopShuffleMask(PSHUFLMask)) - V = DAG.getNode(X86ISD::PSHUFLW, DL, MVT::v8i16, V, - getV4X86ShuffleImm8ForMask(PSHUFLMask, DAG)); - if (!isNoopShuffleMask(PSHUFHMask)) - V = DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16, V, - getV4X86ShuffleImm8ForMask(PSHUFHMask, DAG)); - if (!isNoopShuffleMask(PSHUFDMask)) - V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, - DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, - DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V), - getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG))); - - // At this point, each half should contain all its inputs, and we can then - // just shuffle them into their final position. - assert(std::count_if(LoMask.begin(), LoMask.end(), - [](int M) { return M >= 4; }) == 0 && - "Failed to lift all the high half inputs to the low mask!"); - assert(std::count_if(HiMask.begin(), HiMask.end(), - [](int M) { return M >= 0 && M < 4; }) == 0 && - "Failed to lift all the low half inputs to the high mask!"); - - // Do a half shuffle for the low mask. - if (!isNoopShuffleMask(LoMask)) - V = DAG.getNode(X86ISD::PSHUFLW, DL, MVT::v8i16, V, - getV4X86ShuffleImm8ForMask(LoMask, DAG)); - - // Do a half shuffle with the high mask after shifting its values down. - for (int &M : HiMask) - if (M >= 0) - M -= 4; - if (!isNoopShuffleMask(HiMask)) - V = DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16, V, - getV4X86ShuffleImm8ForMask(HiMask, DAG)); - - return V; -} - -/// \brief Helper to form a PSHUFB-based shuffle+blend. -static SDValue lowerVectorShuffleAsPSHUFB(SDLoc DL, MVT VT, SDValue V1, - SDValue V2, ArrayRef Mask, - SelectionDAG &DAG, bool &V1InUse, - bool &V2InUse) { - SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2); - SDValue V1Mask[16]; - SDValue V2Mask[16]; - V1InUse = false; - V2InUse = false; - - int Size = Mask.size(); - int Scale = 16 / Size; - for (int i = 0; i < 16; ++i) { - if (Mask[i / Scale] == -1) { - V1Mask[i] = V2Mask[i] = DAG.getUNDEF(MVT::i8); - } else { - const int ZeroMask = 0x80; - int V1Idx = Mask[i / Scale] < Size ? Mask[i / Scale] * Scale + i % Scale - : ZeroMask; - int V2Idx = Mask[i / Scale] < Size - ? ZeroMask - : (Mask[i / Scale] - Size) * Scale + i % Scale; - if (Zeroable[i / Scale]) - V1Idx = V2Idx = ZeroMask; - V1Mask[i] = DAG.getConstant(V1Idx, MVT::i8); - V2Mask[i] = DAG.getConstant(V2Idx, MVT::i8); - V1InUse |= (ZeroMask != V1Idx); - V2InUse |= (ZeroMask != V2Idx); - } - } - - if (V1InUse) - V1 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, - DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, V1), - DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, V1Mask)); - if (V2InUse) - V2 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, - DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, V2), - DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, V2Mask)); - - // If we need shuffled inputs from both, blend the two. - SDValue V; - if (V1InUse && V2InUse) - V = DAG.getNode(ISD::OR, DL, MVT::v16i8, V1, V2); - else - V = V1InUse ? V1 : V2; - - // Cast the result back to the correct type. - return DAG.getNode(ISD::BITCAST, DL, VT, V); -} - -/// \brief Generic lowering of 8-lane i16 shuffles. -/// -/// This handles both single-input shuffles and combined shuffle/blends with -/// two inputs. The single input shuffles are immediately delegated to -/// a dedicated lowering routine. -/// -/// The blends are lowered in one of three fundamental ways. If there are few -/// enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle -/// of the input is significantly cheaper when lowered as an interleaving of -/// the two inputs, try to interleave them. Otherwise, blend the low and high -/// halves of the inputs separately (making them have relatively few inputs) -/// and then concatenate them. -static SDValue lowerV8I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - SDLoc DL(Op); - assert(Op.getSimpleValueType() == MVT::v8i16 && "Bad shuffle type!"); - assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef OrigMask = SVOp->getMask(); - int MaskStorage[8] = {OrigMask[0], OrigMask[1], OrigMask[2], OrigMask[3], - OrigMask[4], OrigMask[5], OrigMask[6], OrigMask[7]}; - MutableArrayRef Mask(MaskStorage); - - assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"); - - // Whenever we can lower this as a zext, that instruction is strictly faster - // than any alternative. - if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend( - DL, MVT::v8i16, V1, V2, OrigMask, Subtarget, DAG)) - return ZExt; - - auto isV1 = [](int M) { return M >= 0 && M < 8; }; - auto isV2 = [](int M) { return M >= 8; }; - - int NumV1Inputs = std::count_if(Mask.begin(), Mask.end(), isV1); - int NumV2Inputs = std::count_if(Mask.begin(), Mask.end(), isV2); - - if (NumV2Inputs == 0) - return lowerV8I16SingleInputVectorShuffle(DL, V1, Mask, Subtarget, DAG); - - assert(NumV1Inputs > 0 && "All single-input shuffles should be canonicalized " - "to be V1-input shuffles."); - - // Try to use shift instructions. - if (SDValue Shift = - lowerVectorShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask, DAG)) - return Shift; - - // There are special ways we can lower some single-element blends. - if (NumV2Inputs == 1) - if (SDValue V = lowerVectorShuffleAsElementInsertion(MVT::v8i16, DL, V1, V2, - Mask, Subtarget, DAG)) - return V; - - // We have different paths for blend lowering, but they all must use the - // *exact* same predicate. - bool IsBlendSupported = Subtarget->hasSSE41(); - if (IsBlendSupported) - if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask, - Subtarget, DAG)) - return Blend; - - if (SDValue Masked = - lowerVectorShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask, DAG)) - return Masked; - - // Use dedicated unpack instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 1, 9, 2, 10, 3, 11)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i16, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, 4, 12, 5, 13, 6, 14, 7, 15)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i16, V1, V2); - - // Try to use byte rotation instructions. - if (SDValue Rotate = lowerVectorShuffleAsByteRotate( - DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG)) - return Rotate; - - if (SDValue BitBlend = - lowerVectorShuffleAsBitBlend(DL, MVT::v8i16, V1, V2, Mask, DAG)) - return BitBlend; - - if (SDValue Unpack = - lowerVectorShuffleAsUnpack(MVT::v8i16, DL, V1, V2, Mask, DAG)) - return Unpack; - - // If we can't directly blend but can use PSHUFB, that will be better as it - // can both shuffle and set up the inefficient blend. - if (!IsBlendSupported && Subtarget->hasSSSE3()) { - bool V1InUse, V2InUse; - return lowerVectorShuffleAsPSHUFB(DL, MVT::v8i16, V1, V2, Mask, DAG, - V1InUse, V2InUse); - } - - // We can always bit-blend if we have to so the fallback strategy is to - // decompose into single-input permutes and blends. - return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i16, V1, V2, - Mask, DAG); -} - -/// \brief Check whether a compaction lowering can be done by dropping even -/// elements and compute how many times even elements must be dropped. -/// -/// This handles shuffles which take every Nth element where N is a power of -/// two. Example shuffle masks: -/// -/// N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14 -/// N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 -/// N = 2: 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12 -/// N = 2: 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28 -/// N = 3: 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8 -/// N = 3: 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24 -/// -/// Any of these lanes can of course be undef. -/// -/// This routine only supports N <= 3. -/// FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here -/// for larger N. -/// -/// \returns N above, or the number of times even elements must be dropped if -/// there is such a number. Otherwise returns zero. -static int canLowerByDroppingEvenElements(ArrayRef Mask) { - // Figure out whether we're looping over two inputs or just one. - bool IsSingleInput = isSingleInputShuffleMask(Mask); - - // The modulus for the shuffle vector entries is based on whether this is - // a single input or not. - int ShuffleModulus = Mask.size() * (IsSingleInput ? 1 : 2); - assert(isPowerOf2_32((uint32_t)ShuffleModulus) && - "We should only be called with masks with a power-of-2 size!"); - - uint64_t ModMask = (uint64_t)ShuffleModulus - 1; - - // We track whether the input is viable for all power-of-2 strides 2^1, 2^2, - // and 2^3 simultaneously. This is because we may have ambiguity with - // partially undef inputs. - bool ViableForN[3] = {true, true, true}; - - for (int i = 0, e = Mask.size(); i < e; ++i) { - // Ignore undef lanes, we'll optimistically collapse them to the pattern we - // want. - if (Mask[i] == -1) - continue; - - bool IsAnyViable = false; - for (unsigned j = 0; j != array_lengthof(ViableForN); ++j) - if (ViableForN[j]) { - uint64_t N = j + 1; - - // The shuffle mask must be equal to (i * 2^N) % M. - if ((uint64_t)Mask[i] == (((uint64_t)i << N) & ModMask)) - IsAnyViable = true; - else - ViableForN[j] = false; - } - // Early exit if we exhaust the possible powers of two. - if (!IsAnyViable) - break; - } - - for (unsigned j = 0; j != array_lengthof(ViableForN); ++j) - if (ViableForN[j]) - return j + 1; - - // Return 0 as there is no viable power of two. - return 0; -} - -/// \brief Generic lowering of v16i8 shuffles. -/// -/// This is a hybrid strategy to lower v16i8 vectors. It first attempts to -/// detect any complexity reducing interleaving. If that doesn't help, it uses -/// UNPCK to spread the i8 elements across two i16-element vectors, and uses -/// the existing lowering for v8i16 blends on each half, finally PACK-ing them -/// back together. -static SDValue lowerV16I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - SDLoc DL(Op); - assert(Op.getSimpleValueType() == MVT::v16i8 && "Bad shuffle type!"); - assert(V1.getSimpleValueType() == MVT::v16i8 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!"); - - // Try to use shift instructions. - if (SDValue Shift = - lowerVectorShuffleAsShift(DL, MVT::v16i8, V1, V2, Mask, DAG)) - return Shift; - - // Try to use byte rotation instructions. - if (SDValue Rotate = lowerVectorShuffleAsByteRotate( - DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG)) - return Rotate; - - // Try to use a zext lowering. - if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend( - DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG)) - return ZExt; - - int NumV2Elements = - std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 16; }); - - // For single-input shuffles, there are some nicer lowering tricks we can use. - if (NumV2Elements == 0) { - // Check for being able to broadcast a single element. - if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v16i8, DL, V1, - Mask, Subtarget, DAG)) - return Broadcast; - - // Check whether we can widen this to an i16 shuffle by duplicating bytes. - // Notably, this handles splat and partial-splat shuffles more efficiently. - // However, it only makes sense if the pre-duplication shuffle simplifies - // things significantly. Currently, this means we need to be able to - // express the pre-duplication shuffle as an i16 shuffle. - // - // FIXME: We should check for other patterns which can be widened into an - // i16 shuffle as well. - auto canWidenViaDuplication = [](ArrayRef Mask) { - for (int i = 0; i < 16; i += 2) - if (Mask[i] != -1 && Mask[i + 1] != -1 && Mask[i] != Mask[i + 1]) - return false; - - return true; - }; - auto tryToWidenViaDuplication = [&]() -> SDValue { - if (!canWidenViaDuplication(Mask)) - return SDValue(); - SmallVector LoInputs; - std::copy_if(Mask.begin(), Mask.end(), std::back_inserter(LoInputs), - [](int M) { return M >= 0 && M < 8; }); - std::sort(LoInputs.begin(), LoInputs.end()); - LoInputs.erase(std::unique(LoInputs.begin(), LoInputs.end()), - LoInputs.end()); - SmallVector HiInputs; - std::copy_if(Mask.begin(), Mask.end(), std::back_inserter(HiInputs), - [](int M) { return M >= 8; }); - std::sort(HiInputs.begin(), HiInputs.end()); - HiInputs.erase(std::unique(HiInputs.begin(), HiInputs.end()), - HiInputs.end()); - - bool TargetLo = LoInputs.size() >= HiInputs.size(); - ArrayRef InPlaceInputs = TargetLo ? LoInputs : HiInputs; - ArrayRef MovingInputs = TargetLo ? HiInputs : LoInputs; - - int PreDupI16Shuffle[] = {-1, -1, -1, -1, -1, -1, -1, -1}; - SmallDenseMap LaneMap; - for (int I : InPlaceInputs) { - PreDupI16Shuffle[I/2] = I/2; - LaneMap[I] = I; - } - int j = TargetLo ? 0 : 4, je = j + 4; - for (int i = 0, ie = MovingInputs.size(); i < ie; ++i) { - // Check if j is already a shuffle of this input. This happens when - // there are two adjacent bytes after we move the low one. - if (PreDupI16Shuffle[j] != MovingInputs[i] / 2) { - // If we haven't yet mapped the input, search for a slot into which - // we can map it. - while (j < je && PreDupI16Shuffle[j] != -1) - ++j; - - if (j == je) - // We can't place the inputs into a single half with a simple i16 shuffle, so bail. - return SDValue(); - - // Map this input with the i16 shuffle. - PreDupI16Shuffle[j] = MovingInputs[i] / 2; - } - - // Update the lane map based on the mapping we ended up with. - LaneMap[MovingInputs[i]] = 2 * j + MovingInputs[i] % 2; - } - V1 = DAG.getNode( - ISD::BITCAST, DL, MVT::v16i8, - DAG.getVectorShuffle(MVT::v8i16, DL, - DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1), - DAG.getUNDEF(MVT::v8i16), PreDupI16Shuffle)); - - // Unpack the bytes to form the i16s that will be shuffled into place. - V1 = DAG.getNode(TargetLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL, - MVT::v16i8, V1, V1); - - int PostDupI16Shuffle[8] = {-1, -1, -1, -1, -1, -1, -1, -1}; - for (int i = 0; i < 16; ++i) - if (Mask[i] != -1) { - int MappedMask = LaneMap[Mask[i]] - (TargetLo ? 0 : 8); - assert(MappedMask < 8 && "Invalid v8 shuffle mask!"); - if (PostDupI16Shuffle[i / 2] == -1) - PostDupI16Shuffle[i / 2] = MappedMask; - else - assert(PostDupI16Shuffle[i / 2] == MappedMask && - "Conflicting entrties in the original shuffle!"); - } - return DAG.getNode( - ISD::BITCAST, DL, MVT::v16i8, - DAG.getVectorShuffle(MVT::v8i16, DL, - DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1), - DAG.getUNDEF(MVT::v8i16), PostDupI16Shuffle)); - }; - if (SDValue V = tryToWidenViaDuplication()) - return V; - } - - // Use dedicated unpack instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, Mask, - 0, 16, 1, 17, 2, 18, 3, 19, - 4, 20, 5, 21, 6, 22, 7, 23)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, - 8, 24, 9, 25, 10, 26, 11, 27, - 12, 28, 13, 29, 14, 30, 15, 31)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V1, V2); - - // Check for SSSE3 which lets us lower all v16i8 shuffles much more directly - // with PSHUFB. It is important to do this before we attempt to generate any - // blends but after all of the single-input lowerings. If the single input - // lowerings can find an instruction sequence that is faster than a PSHUFB, we - // want to preserve that and we can DAG combine any longer sequences into - // a PSHUFB in the end. But once we start blending from multiple inputs, - // the complexity of DAG combining bad patterns back into PSHUFB is too high, - // and there are *very* few patterns that would actually be faster than the - // PSHUFB approach because of its ability to zero lanes. - // - // FIXME: The only exceptions to the above are blends which are exact - // interleavings with direct instructions supporting them. We currently don't - // handle those well here. - if (Subtarget->hasSSSE3()) { - bool V1InUse = false; - bool V2InUse = false; - - SDValue PSHUFB = lowerVectorShuffleAsPSHUFB(DL, MVT::v16i8, V1, V2, Mask, - DAG, V1InUse, V2InUse); - - // If both V1 and V2 are in use and we can use a direct blend or an unpack, - // do so. This avoids using them to handle blends-with-zero which is - // important as a single pshufb is significantly faster for that. - if (V1InUse && V2InUse) { - if (Subtarget->hasSSE41()) - if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i8, V1, V2, - Mask, Subtarget, DAG)) - return Blend; - - // We can use an unpack to do the blending rather than an or in some - // cases. Even though the or may be (very minorly) more efficient, we - // preference this lowering because there are common cases where part of - // the complexity of the shuffles goes away when we do the final blend as - // an unpack. - // FIXME: It might be worth trying to detect if the unpack-feeding - // shuffles will both be pshufb, in which case we shouldn't bother with - // this. - if (SDValue Unpack = - lowerVectorShuffleAsUnpack(MVT::v16i8, DL, V1, V2, Mask, DAG)) - return Unpack; - } - - return PSHUFB; - } - - // There are special ways we can lower some single-element blends. - if (NumV2Elements == 1) - if (SDValue V = lowerVectorShuffleAsElementInsertion(MVT::v16i8, DL, V1, V2, - Mask, Subtarget, DAG)) - return V; - - if (SDValue BitBlend = - lowerVectorShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG)) - return BitBlend; - - // Check whether a compaction lowering can be done. This handles shuffles - // which take every Nth element for some even N. See the helper function for - // details. - // - // We special case these as they can be particularly efficiently handled with - // the PACKUSB instruction on x86 and they show up in common patterns of - // rearranging bytes to truncate wide elements. - if (int NumEvenDrops = canLowerByDroppingEvenElements(Mask)) { - // NumEvenDrops is the power of two stride of the elements. Another way of - // thinking about it is that we need to drop the even elements this many - // times to get the original input. - bool IsSingleInput = isSingleInputShuffleMask(Mask); - - // First we need to zero all the dropped bytes. - assert(NumEvenDrops <= 3 && - "No support for dropping even elements more than 3 times."); - // We use the mask type to pick which bytes are preserved based on how many - // elements are dropped. - MVT MaskVTs[] = { MVT::v8i16, MVT::v4i32, MVT::v2i64 }; - SDValue ByteClearMask = - DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, - DAG.getConstant(0xFF, MaskVTs[NumEvenDrops - 1])); - V1 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V1, ByteClearMask); - if (!IsSingleInput) - V2 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V2, ByteClearMask); - - // Now pack things back together. - V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1); - V2 = IsSingleInput ? V1 : DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V2); - SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, V2); - for (int i = 1; i < NumEvenDrops; ++i) { - Result = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, Result); - Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result); - } - - return Result; - } - - // Handle multi-input cases by blending single-input shuffles. - if (NumV2Elements > 0) - return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v16i8, V1, V2, - Mask, DAG); - - // The fallback path for single-input shuffles widens this into two v8i16 - // vectors with unpacks, shuffles those, and then pulls them back together - // with a pack. - SDValue V = V1; - - int LoBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1}; - int HiBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1}; - for (int i = 0; i < 16; ++i) - if (Mask[i] >= 0) - (i < 8 ? LoBlendMask[i] : HiBlendMask[i % 8]) = Mask[i]; - - SDValue Zero = getZeroVector(MVT::v8i16, Subtarget, DAG, DL); - - SDValue VLoHalf, VHiHalf; - // Check if any of the odd lanes in the v16i8 are used. If not, we can mask - // them out and avoid using UNPCK{L,H} to extract the elements of V as - // i16s. - if (std::none_of(std::begin(LoBlendMask), std::end(LoBlendMask), - [](int M) { return M >= 0 && M % 2 == 1; }) && - std::none_of(std::begin(HiBlendMask), std::end(HiBlendMask), - [](int M) { return M >= 0 && M % 2 == 1; })) { - // Use a mask to drop the high bytes. - VLoHalf = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V); - VLoHalf = DAG.getNode(ISD::AND, DL, MVT::v8i16, VLoHalf, - DAG.getConstant(0x00FF, MVT::v8i16)); - - // This will be a single vector shuffle instead of a blend so nuke VHiHalf. - VHiHalf = DAG.getUNDEF(MVT::v8i16); - - // Squash the masks to point directly into VLoHalf. - for (int &M : LoBlendMask) - if (M >= 0) - M /= 2; - for (int &M : HiBlendMask) - if (M >= 0) - M /= 2; - } else { - // Otherwise just unpack the low half of V into VLoHalf and the high half into - // VHiHalf so that we can blend them as i16s. - VLoHalf = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, - DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero)); - VHiHalf = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, - DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero)); - } - - SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask); - SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask); - - return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV); -} - -/// \brief Dispatching routine to lower various 128-bit x86 vector shuffles. -/// -/// This routine breaks down the specific type of 128-bit shuffle and -/// dispatches to the lowering routines accordingly. -static SDValue lower128BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2, - MVT VT, const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - switch (VT.SimpleTy) { - case MVT::v2i64: - return lowerV2I64VectorShuffle(Op, V1, V2, Subtarget, DAG); - case MVT::v2f64: - return lowerV2F64VectorShuffle(Op, V1, V2, Subtarget, DAG); - case MVT::v4i32: - return lowerV4I32VectorShuffle(Op, V1, V2, Subtarget, DAG); - case MVT::v4f32: - return lowerV4F32VectorShuffle(Op, V1, V2, Subtarget, DAG); - case MVT::v8i16: - return lowerV8I16VectorShuffle(Op, V1, V2, Subtarget, DAG); - case MVT::v16i8: - return lowerV16I8VectorShuffle(Op, V1, V2, Subtarget, DAG); - - default: - llvm_unreachable("Unimplemented!"); - } -} - -/// \brief Helper function to test whether a shuffle mask could be -/// simplified by widening the elements being shuffled. -/// -/// Appends the mask for wider elements in WidenedMask if valid. Otherwise -/// leaves it in an unspecified state. -/// -/// NOTE: This must handle normal vector shuffle masks and *target* vector -/// shuffle masks. The latter have the special property of a '-2' representing -/// a zero-ed lane of a vector. -static bool canWidenShuffleElements(ArrayRef Mask, - SmallVectorImpl &WidenedMask) { - for (int i = 0, Size = Mask.size(); i < Size; i += 2) { - // If both elements are undef, its trivial. - if (Mask[i] == SM_SentinelUndef && Mask[i + 1] == SM_SentinelUndef) { - WidenedMask.push_back(SM_SentinelUndef); - continue; - } - - // Check for an undef mask and a mask value properly aligned to fit with - // a pair of values. If we find such a case, use the non-undef mask's value. - if (Mask[i] == SM_SentinelUndef && Mask[i + 1] >= 0 && Mask[i + 1] % 2 == 1) { - WidenedMask.push_back(Mask[i + 1] / 2); - continue; - } - if (Mask[i + 1] == SM_SentinelUndef && Mask[i] >= 0 && Mask[i] % 2 == 0) { - WidenedMask.push_back(Mask[i] / 2); - continue; - } - - // When zeroing, we need to spread the zeroing across both lanes to widen. - if (Mask[i] == SM_SentinelZero || Mask[i + 1] == SM_SentinelZero) { - if ((Mask[i] == SM_SentinelZero || Mask[i] == SM_SentinelUndef) && - (Mask[i + 1] == SM_SentinelZero || Mask[i + 1] == SM_SentinelUndef)) { - WidenedMask.push_back(SM_SentinelZero); - continue; - } - return false; - } - - // Finally check if the two mask values are adjacent and aligned with - // a pair. - if (Mask[i] != SM_SentinelUndef && Mask[i] % 2 == 0 && Mask[i] + 1 == Mask[i + 1]) { - WidenedMask.push_back(Mask[i] / 2); - continue; - } - - // Otherwise we can't safely widen the elements used in this shuffle. - return false; - } - assert(WidenedMask.size() == Mask.size() / 2 && - "Incorrect size of mask after widening the elements!"); - - return true; -} - -/// \brief Generic routine to split vector shuffle into half-sized shuffles. -/// -/// This routine just extracts two subvectors, shuffles them independently, and -/// then concatenates them back together. This should work effectively with all -/// AVX vector shuffle types. -static SDValue splitAndLowerVectorShuffle(SDLoc DL, MVT VT, SDValue V1, - SDValue V2, ArrayRef Mask, - SelectionDAG &DAG) { - assert(VT.getSizeInBits() >= 256 && - "Only for 256-bit or wider vector shuffles!"); - assert(V1.getSimpleValueType() == VT && "Bad operand type!"); - assert(V2.getSimpleValueType() == VT && "Bad operand type!"); - - ArrayRef LoMask = Mask.slice(0, Mask.size() / 2); - ArrayRef HiMask = Mask.slice(Mask.size() / 2); - - int NumElements = VT.getVectorNumElements(); - int SplitNumElements = NumElements / 2; - MVT ScalarVT = VT.getScalarType(); - MVT SplitVT = MVT::getVectorVT(ScalarVT, NumElements / 2); - - // Rather than splitting build-vectors, just build two narrower build - // vectors. This helps shuffling with splats and zeros. - auto SplitVector = [&](SDValue V) { - while (V.getOpcode() == ISD::BITCAST) - V = V->getOperand(0); - - MVT OrigVT = V.getSimpleValueType(); - int OrigNumElements = OrigVT.getVectorNumElements(); - int OrigSplitNumElements = OrigNumElements / 2; - MVT OrigScalarVT = OrigVT.getScalarType(); - MVT OrigSplitVT = MVT::getVectorVT(OrigScalarVT, OrigNumElements / 2); - - SDValue LoV, HiV; - - auto *BV = dyn_cast(V); - if (!BV) { - LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigSplitVT, V, - DAG.getIntPtrConstant(0)); - HiV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigSplitVT, V, - DAG.getIntPtrConstant(OrigSplitNumElements)); - } else { - - SmallVector LoOps, HiOps; - for (int i = 0; i < OrigSplitNumElements; ++i) { - LoOps.push_back(BV->getOperand(i)); - HiOps.push_back(BV->getOperand(i + OrigSplitNumElements)); - } - LoV = DAG.getNode(ISD::BUILD_VECTOR, DL, OrigSplitVT, LoOps); - HiV = DAG.getNode(ISD::BUILD_VECTOR, DL, OrigSplitVT, HiOps); - } - return std::make_pair(DAG.getNode(ISD::BITCAST, DL, SplitVT, LoV), - DAG.getNode(ISD::BITCAST, DL, SplitVT, HiV)); - }; - - SDValue LoV1, HiV1, LoV2, HiV2; - std::tie(LoV1, HiV1) = SplitVector(V1); - std::tie(LoV2, HiV2) = SplitVector(V2); - - // Now create two 4-way blends of these half-width vectors. - auto HalfBlend = [&](ArrayRef HalfMask) { - bool UseLoV1 = false, UseHiV1 = false, UseLoV2 = false, UseHiV2 = false; - SmallVector V1BlendMask, V2BlendMask, BlendMask; - for (int i = 0; i < SplitNumElements; ++i) { - int M = HalfMask[i]; - if (M >= NumElements) { - if (M >= NumElements + SplitNumElements) - UseHiV2 = true; - else - UseLoV2 = true; - V2BlendMask.push_back(M - NumElements); - V1BlendMask.push_back(-1); - BlendMask.push_back(SplitNumElements + i); - } else if (M >= 0) { - if (M >= SplitNumElements) - UseHiV1 = true; - else - UseLoV1 = true; - V2BlendMask.push_back(-1); - V1BlendMask.push_back(M); - BlendMask.push_back(i); - } else { - V2BlendMask.push_back(-1); - V1BlendMask.push_back(-1); - BlendMask.push_back(-1); - } - } - - // Because the lowering happens after all combining takes place, we need to - // manually combine these blend masks as much as possible so that we create - // a minimal number of high-level vector shuffle nodes. - - // First try just blending the halves of V1 or V2. - if (!UseLoV1 && !UseHiV1 && !UseLoV2 && !UseHiV2) - return DAG.getUNDEF(SplitVT); - if (!UseLoV2 && !UseHiV2) - return DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask); - if (!UseLoV1 && !UseHiV1) - return DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask); - - SDValue V1Blend, V2Blend; - if (UseLoV1 && UseHiV1) { - V1Blend = - DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask); - } else { - // We only use half of V1 so map the usage down into the final blend mask. - V1Blend = UseLoV1 ? LoV1 : HiV1; - for (int i = 0; i < SplitNumElements; ++i) - if (BlendMask[i] >= 0 && BlendMask[i] < SplitNumElements) - BlendMask[i] = V1BlendMask[i] - (UseLoV1 ? 0 : SplitNumElements); - } - if (UseLoV2 && UseHiV2) { - V2Blend = - DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask); - } else { - // We only use half of V2 so map the usage down into the final blend mask. - V2Blend = UseLoV2 ? LoV2 : HiV2; - for (int i = 0; i < SplitNumElements; ++i) - if (BlendMask[i] >= SplitNumElements) - BlendMask[i] = V2BlendMask[i] + (UseLoV2 ? SplitNumElements : 0); - } - return DAG.getVectorShuffle(SplitVT, DL, V1Blend, V2Blend, BlendMask); - }; - SDValue Lo = HalfBlend(LoMask); - SDValue Hi = HalfBlend(HiMask); - return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); -} - -/// \brief Either split a vector in halves or decompose the shuffles and the -/// blend. -/// -/// This is provided as a good fallback for many lowerings of non-single-input -/// shuffles with more than one 128-bit lane. In those cases, we want to select -/// between splitting the shuffle into 128-bit components and stitching those -/// back together vs. extracting the single-input shuffles and blending those -/// results. -static SDValue lowerVectorShuffleAsSplitOrBlend(SDLoc DL, MVT VT, SDValue V1, - SDValue V2, ArrayRef Mask, - SelectionDAG &DAG) { - assert(!isSingleInputShuffleMask(Mask) && "This routine must not be used to " - "lower single-input shuffles as it " - "could then recurse on itself."); - int Size = Mask.size(); - - // If this can be modeled as a broadcast of two elements followed by a blend, - // prefer that lowering. This is especially important because broadcasts can - // often fold with memory operands. - auto DoBothBroadcast = [&] { - int V1BroadcastIdx = -1, V2BroadcastIdx = -1; - for (int M : Mask) - if (M >= Size) { - if (V2BroadcastIdx == -1) - V2BroadcastIdx = M - Size; - else if (M - Size != V2BroadcastIdx) - return false; - } else if (M >= 0) { - if (V1BroadcastIdx == -1) - V1BroadcastIdx = M; - else if (M != V1BroadcastIdx) - return false; - } - return true; - }; - if (DoBothBroadcast()) - return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask, - DAG); - - // If the inputs all stem from a single 128-bit lane of each input, then we - // split them rather than blending because the split will decompose to - // unusually few instructions. - int LaneCount = VT.getSizeInBits() / 128; - int LaneSize = Size / LaneCount; - SmallBitVector LaneInputs[2]; - LaneInputs[0].resize(LaneCount, false); - LaneInputs[1].resize(LaneCount, false); - for (int i = 0; i < Size; ++i) - if (Mask[i] >= 0) - LaneInputs[Mask[i] / Size][(Mask[i] % Size) / LaneSize] = true; - if (LaneInputs[0].count() <= 1 && LaneInputs[1].count() <= 1) - return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG); - - // Otherwise, just fall back to decomposed shuffles and a blend. This requires - // that the decomposed single-input shuffles don't end up here. - return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask, DAG); -} - -/// \brief Lower a vector shuffle crossing multiple 128-bit lanes as -/// a permutation and blend of those lanes. -/// -/// This essentially blends the out-of-lane inputs to each lane into the lane -/// from a permuted copy of the vector. This lowering strategy results in four -/// instructions in the worst case for a single-input cross lane shuffle which -/// is lower than any other fully general cross-lane shuffle strategy I'm aware -/// of. Special cases for each particular shuffle pattern should be handled -/// prior to trying this lowering. -static SDValue lowerVectorShuffleAsLanePermuteAndBlend(SDLoc DL, MVT VT, - SDValue V1, SDValue V2, - ArrayRef Mask, - SelectionDAG &DAG) { - // FIXME: This should probably be generalized for 512-bit vectors as well. - assert(VT.getSizeInBits() == 256 && "Only for 256-bit vector shuffles!"); - int LaneSize = Mask.size() / 2; - - // If there are only inputs from one 128-bit lane, splitting will in fact be - // less expensive. The flags track wether the given lane contains an element - // that crosses to another lane. - bool LaneCrossing[2] = {false, false}; - for (int i = 0, Size = Mask.size(); i < Size; ++i) - if (Mask[i] >= 0 && (Mask[i] % Size) / LaneSize != i / LaneSize) - LaneCrossing[(Mask[i] % Size) / LaneSize] = true; - if (!LaneCrossing[0] || !LaneCrossing[1]) - return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG); - - if (isSingleInputShuffleMask(Mask)) { - SmallVector FlippedBlendMask; - for (int i = 0, Size = Mask.size(); i < Size; ++i) - FlippedBlendMask.push_back( - Mask[i] < 0 ? -1 : (((Mask[i] % Size) / LaneSize == i / LaneSize) - ? Mask[i] - : Mask[i] % LaneSize + - (i / LaneSize) * LaneSize + Size)); - - // Flip the vector, and blend the results which should now be in-lane. The - // VPERM2X128 mask uses the low 2 bits for the low source and bits 4 and - // 5 for the high source. The value 3 selects the high half of source 2 and - // the value 2 selects the low half of source 2. We only use source 2 to - // allow folding it into a memory operand. - unsigned PERMMask = 3 | 2 << 4; - SDValue Flipped = DAG.getNode(X86ISD::VPERM2X128, DL, VT, DAG.getUNDEF(VT), - V1, DAG.getConstant(PERMMask, MVT::i8)); - return DAG.getVectorShuffle(VT, DL, V1, Flipped, FlippedBlendMask); - } - - // This now reduces to two single-input shuffles of V1 and V2 which at worst - // will be handled by the above logic and a blend of the results, much like - // other patterns in AVX. - return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask, DAG); -} - -/// \brief Handle lowering 2-lane 128-bit shuffles. -static SDValue lowerV2X128VectorShuffle(SDLoc DL, MVT VT, SDValue V1, - SDValue V2, ArrayRef Mask, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - // Blends are faster and handle all the non-lane-crossing cases. - if (SDValue Blend = lowerVectorShuffleAsBlend(DL, VT, V1, V2, Mask, - Subtarget, DAG)) - return Blend; - - MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), - VT.getVectorNumElements() / 2); - // Check for patterns which can be matched with a single insert of a 128-bit - // subvector. - if (isShuffleEquivalent(V1, V2, Mask, 0, 1, 0, 1) || - isShuffleEquivalent(V1, V2, Mask, 0, 1, 4, 5)) { - SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1, - DAG.getIntPtrConstant(0)); - SDValue HiV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, - Mask[2] < 4 ? V1 : V2, DAG.getIntPtrConstant(0)); - return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LoV, HiV); - } - if (isShuffleEquivalent(V1, V2, Mask, 0, 1, 6, 7)) { - SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1, - DAG.getIntPtrConstant(0)); - SDValue HiV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V2, - DAG.getIntPtrConstant(2)); - return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LoV, HiV); - } - - // Otherwise form a 128-bit permutation. - // FIXME: Detect zero-vector inputs and use the VPERM2X128 to zero that half. - unsigned PermMask = Mask[0] / 2 | (Mask[2] / 2) << 4; - return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2, - DAG.getConstant(PermMask, MVT::i8)); -} - -/// \brief Lower a vector shuffle by first fixing the 128-bit lanes and then -/// shuffling each lane. -/// -/// This will only succeed when the result of fixing the 128-bit lanes results -/// in a single-input non-lane-crossing shuffle with a repeating shuffle mask in -/// each 128-bit lanes. This handles many cases where we can quickly blend away -/// the lane crosses early and then use simpler shuffles within each lane. -/// -/// FIXME: It might be worthwhile at some point to support this without -/// requiring the 128-bit lane-relative shuffles to be repeating, but currently -/// in x86 only floating point has interesting non-repeating shuffles, and even -/// those are still *marginally* more expensive. -static SDValue lowerVectorShuffleByMerging128BitLanes( - SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef Mask, - const X86Subtarget *Subtarget, SelectionDAG &DAG) { - assert(!isSingleInputShuffleMask(Mask) && - "This is only useful with multiple inputs."); - - int Size = Mask.size(); - int LaneSize = 128 / VT.getScalarSizeInBits(); - int NumLanes = Size / LaneSize; - assert(NumLanes > 1 && "Only handles 256-bit and wider shuffles."); - - // See if we can build a hypothetical 128-bit lane-fixing shuffle mask. Also - // check whether the in-128-bit lane shuffles share a repeating pattern. - SmallVector Lanes; - Lanes.resize(NumLanes, -1); - SmallVector InLaneMask; - InLaneMask.resize(LaneSize, -1); - for (int i = 0; i < Size; ++i) { - if (Mask[i] < 0) - continue; - - int j = i / LaneSize; - - if (Lanes[j] < 0) { - // First entry we've seen for this lane. - Lanes[j] = Mask[i] / LaneSize; - } else if (Lanes[j] != Mask[i] / LaneSize) { - // This doesn't match the lane selected previously! - return SDValue(); - } - - // Check that within each lane we have a consistent shuffle mask. - int k = i % LaneSize; - if (InLaneMask[k] < 0) { - InLaneMask[k] = Mask[i] % LaneSize; - } else if (InLaneMask[k] != Mask[i] % LaneSize) { - // This doesn't fit a repeating in-lane mask. - return SDValue(); - } - } - - // First shuffle the lanes into place. - MVT LaneVT = MVT::getVectorVT(VT.isFloatingPoint() ? MVT::f64 : MVT::i64, - VT.getSizeInBits() / 64); - SmallVector LaneMask; - LaneMask.resize(NumLanes * 2, -1); - for (int i = 0; i < NumLanes; ++i) - if (Lanes[i] >= 0) { - LaneMask[2 * i + 0] = 2*Lanes[i] + 0; - LaneMask[2 * i + 1] = 2*Lanes[i] + 1; - } - - V1 = DAG.getNode(ISD::BITCAST, DL, LaneVT, V1); - V2 = DAG.getNode(ISD::BITCAST, DL, LaneVT, V2); - SDValue LaneShuffle = DAG.getVectorShuffle(LaneVT, DL, V1, V2, LaneMask); - - // Cast it back to the type we actually want. - LaneShuffle = DAG.getNode(ISD::BITCAST, DL, VT, LaneShuffle); - - // Now do a simple shuffle that isn't lane crossing. - SmallVector NewMask; - NewMask.resize(Size, -1); - for (int i = 0; i < Size; ++i) - if (Mask[i] >= 0) - NewMask[i] = (i / LaneSize) * LaneSize + Mask[i] % LaneSize; - assert(!is128BitLaneCrossingShuffleMask(VT, NewMask) && - "Must not introduce lane crosses at this point!"); - - return DAG.getVectorShuffle(VT, DL, LaneShuffle, DAG.getUNDEF(VT), NewMask); -} - -/// \brief Test whether the specified input (0 or 1) is in-place blended by the -/// given mask. -/// -/// This returns true if the elements from a particular input are already in the -/// slot required by the given mask and require no permutation. -static bool isShuffleMaskInputInPlace(int Input, ArrayRef Mask) { - assert((Input == 0 || Input == 1) && "Only two inputs to shuffles."); - int Size = Mask.size(); - for (int i = 0; i < Size; ++i) - if (Mask[i] >= 0 && Mask[i] / Size == Input && Mask[i] % Size != i) - return false; - - return true; -} - -/// \brief Handle lowering of 4-lane 64-bit floating point shuffles. -/// -/// Also ends up handling lowering of 4-lane 64-bit integer shuffles when AVX2 -/// isn't available. -static SDValue lowerV4F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - SDLoc DL(Op); - assert(V1.getSimpleValueType() == MVT::v4f64 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v4f64 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"); - - SmallVector WidenedMask; - if (canWidenShuffleElements(Mask, WidenedMask)) - return lowerV2X128VectorShuffle(DL, MVT::v4f64, V1, V2, Mask, Subtarget, - DAG); - - if (isSingleInputShuffleMask(Mask)) { - // Check for being able to broadcast a single element. - if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v4f64, DL, V1, - Mask, Subtarget, DAG)) - return Broadcast; - - // Use low duplicate instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, Mask, 0, 0, 2, 2)) - return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1); - - if (!is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask)) { - // Non-half-crossing single input shuffles can be lowerid with an - // interleaved permutation. - unsigned VPERMILPMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1) | - ((Mask[2] == 3) << 2) | ((Mask[3] == 3) << 3); - return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1, - DAG.getConstant(VPERMILPMask, MVT::i8)); - } - - // With AVX2 we have direct support for this permutation. - if (Subtarget->hasAVX2()) - return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1, - getV4X86ShuffleImm8ForMask(Mask, DAG)); - - // Otherwise, fall back. - return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v4f64, V1, V2, Mask, - DAG); - } - - // X86 has dedicated unpack instructions that can handle specific blend - // operations: UNPCKH and UNPCKL. - if (isShuffleEquivalent(V1, V2, Mask, 0, 4, 2, 6)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f64, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, 1, 5, 3, 7)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f64, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, 4, 0, 6, 2)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f64, V2, V1); - if (isShuffleEquivalent(V1, V2, Mask, 5, 1, 7, 3)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f64, V2, V1); - - // If we have a single input to the zero element, insert that into V1 if we - // can do so cheaply. - int NumV2Elements = - std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; }); - if (NumV2Elements == 1 && Mask[0] >= 4) - if (SDValue Insertion = lowerVectorShuffleAsElementInsertion( - MVT::v4f64, DL, V1, V2, Mask, Subtarget, DAG)) - return Insertion; - - if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask, - Subtarget, DAG)) - return Blend; - - // Check if the blend happens to exactly fit that of SHUFPD. - if ((Mask[0] == -1 || Mask[0] < 2) && - (Mask[1] == -1 || (Mask[1] >= 4 && Mask[1] < 6)) && - (Mask[2] == -1 || (Mask[2] >= 2 && Mask[2] < 4)) && - (Mask[3] == -1 || Mask[3] >= 6)) { - unsigned SHUFPDMask = (Mask[0] == 1) | ((Mask[1] == 5) << 1) | - ((Mask[2] == 3) << 2) | ((Mask[3] == 7) << 3); - return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f64, V1, V2, - DAG.getConstant(SHUFPDMask, MVT::i8)); - } - if ((Mask[0] == -1 || (Mask[0] >= 4 && Mask[0] < 6)) && - (Mask[1] == -1 || Mask[1] < 2) && - (Mask[2] == -1 || Mask[2] >= 6) && - (Mask[3] == -1 || (Mask[3] >= 2 && Mask[3] < 4))) { - unsigned SHUFPDMask = (Mask[0] == 5) | ((Mask[1] == 1) << 1) | - ((Mask[2] == 7) << 2) | ((Mask[3] == 3) << 3); - return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f64, V2, V1, - DAG.getConstant(SHUFPDMask, MVT::i8)); - } - - // Try to simplify this by merging 128-bit lanes to enable a lane-based - // shuffle. However, if we have AVX2 and either inputs are already in place, - // we will be able to shuffle even across lanes the other input in a single - // instruction so skip this pattern. - if (!(Subtarget->hasAVX2() && (isShuffleMaskInputInPlace(0, Mask) || - isShuffleMaskInputInPlace(1, Mask)))) - if (SDValue Result = lowerVectorShuffleByMerging128BitLanes( - DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG)) - return Result; - - // If we have AVX2 then we always want to lower with a blend because an v4 we - // can fully permute the elements. - if (Subtarget->hasAVX2()) - return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4f64, V1, V2, - Mask, DAG); - - // Otherwise fall back on generic lowering. - return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask, DAG); -} - -/// \brief Handle lowering of 4-lane 64-bit integer shuffles. -/// -/// This routine is only called when we have AVX2 and thus a reasonable -/// instruction set for v4i64 shuffling.. -static SDValue lowerV4I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - SDLoc DL(Op); - assert(V1.getSimpleValueType() == MVT::v4i64 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v4i64 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"); - assert(Subtarget->hasAVX2() && "We can only lower v4i64 with AVX2!"); - - SmallVector WidenedMask; - if (canWidenShuffleElements(Mask, WidenedMask)) - return lowerV2X128VectorShuffle(DL, MVT::v4i64, V1, V2, Mask, Subtarget, - DAG); - - if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask, - Subtarget, DAG)) - return Blend; - - // Check for being able to broadcast a single element. - if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v4i64, DL, V1, - Mask, Subtarget, DAG)) - return Broadcast; - - // When the shuffle is mirrored between the 128-bit lanes of the unit, we can - // use lower latency instructions that will operate on both 128-bit lanes. - SmallVector RepeatedMask; - if (is128BitLaneRepeatedShuffleMask(MVT::v4i64, Mask, RepeatedMask)) { - if (isSingleInputShuffleMask(Mask)) { - int PSHUFDMask[] = {-1, -1, -1, -1}; - for (int i = 0; i < 2; ++i) - if (RepeatedMask[i] >= 0) { - PSHUFDMask[2 * i] = 2 * RepeatedMask[i]; - PSHUFDMask[2 * i + 1] = 2 * RepeatedMask[i] + 1; - } - return DAG.getNode( - ISD::BITCAST, DL, MVT::v4i64, - DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, - DAG.getNode(ISD::BITCAST, DL, MVT::v8i32, V1), - getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG))); - } - } - - // AVX2 provides a direct instruction for permuting a single input across - // lanes. - if (isSingleInputShuffleMask(Mask)) - return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1, - getV4X86ShuffleImm8ForMask(Mask, DAG)); - - // Try to use shift instructions. - if (SDValue Shift = - lowerVectorShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask, DAG)) - return Shift; - - // Use dedicated unpack instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, Mask, 0, 4, 2, 6)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i64, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, 1, 5, 3, 7)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i64, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, 4, 0, 6, 2)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i64, V2, V1); - if (isShuffleEquivalent(V1, V2, Mask, 5, 1, 7, 3)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i64, V2, V1); - - // Try to simplify this by merging 128-bit lanes to enable a lane-based - // shuffle. However, if we have AVX2 and either inputs are already in place, - // we will be able to shuffle even across lanes the other input in a single - // instruction so skip this pattern. - if (!(Subtarget->hasAVX2() && (isShuffleMaskInputInPlace(0, Mask) || - isShuffleMaskInputInPlace(1, Mask)))) - if (SDValue Result = lowerVectorShuffleByMerging128BitLanes( - DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG)) - return Result; - - // Otherwise fall back on generic blend lowering. - return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i64, V1, V2, - Mask, DAG); -} - -/// \brief Handle lowering of 8-lane 32-bit floating point shuffles. -/// -/// Also ends up handling lowering of 8-lane 32-bit integer shuffles when AVX2 -/// isn't available. -static SDValue lowerV8F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - SDLoc DL(Op); - assert(V1.getSimpleValueType() == MVT::v8f32 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v8f32 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"); - - if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask, - Subtarget, DAG)) - return Blend; - - // Check for being able to broadcast a single element. - if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v8f32, DL, V1, - Mask, Subtarget, DAG)) - return Broadcast; - - // If the shuffle mask is repeated in each 128-bit lane, we have many more - // options to efficiently lower the shuffle. - SmallVector RepeatedMask; - if (is128BitLaneRepeatedShuffleMask(MVT::v8f32, Mask, RepeatedMask)) { - assert(RepeatedMask.size() == 4 && - "Repeated masks must be half the mask width!"); - - // Use even/odd duplicate instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, Mask, 0, 0, 2, 2, 4, 4, 6, 6)) - return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1); - if (isShuffleEquivalent(V1, V2, Mask, 1, 1, 3, 3, 5, 5, 7, 7)) - return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1); - - if (isSingleInputShuffleMask(Mask)) - return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1, - getV4X86ShuffleImm8ForMask(RepeatedMask, DAG)); - - // Use dedicated unpack instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 1, 9, 4, 12, 5, 13)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8f32, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, 2, 10, 3, 11, 6, 14, 7, 15)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8f32, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, 8, 0, 9, 1, 12, 4, 13, 5)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8f32, V2, V1); - if (isShuffleEquivalent(V1, V2, Mask, 10, 2, 11, 3, 14, 6, 15, 7)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8f32, V2, V1); - - // Otherwise, fall back to a SHUFPS sequence. Here it is important that we - // have already handled any direct blends. We also need to squash the - // repeated mask into a simulated v4f32 mask. - for (int i = 0; i < 4; ++i) - if (RepeatedMask[i] >= 8) - RepeatedMask[i] -= 4; - return lowerVectorShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG); - } - - // If we have a single input shuffle with different shuffle patterns in the - // two 128-bit lanes use the variable mask to VPERMILPS. - if (isSingleInputShuffleMask(Mask)) { - SDValue VPermMask[8]; - for (int i = 0; i < 8; ++i) - VPermMask[i] = Mask[i] < 0 ? DAG.getUNDEF(MVT::i32) - : DAG.getConstant(Mask[i], MVT::i32); - if (!is128BitLaneCrossingShuffleMask(MVT::v8f32, Mask)) - return DAG.getNode( - X86ISD::VPERMILPV, DL, MVT::v8f32, V1, - DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i32, VPermMask)); + // Ensure we have the input in a viable dword of its current half. This + // is particularly tricky because the original position may be clobbered + // by inputs being moved and *staying* in that half. + if (IncomingInputs.size() == 1) { + if (isWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) { + int InputFixed = std::find(std::begin(SourceHalfMask), + std::end(SourceHalfMask), -1) - + std::begin(SourceHalfMask) + SourceOffset; + SourceHalfMask[InputFixed - SourceOffset] = + IncomingInputs[0] - SourceOffset; + std::replace(HalfMask.begin(), HalfMask.end(), IncomingInputs[0], + InputFixed); + IncomingInputs[0] = InputFixed; + } + } else if (IncomingInputs.size() == 2) { + if (IncomingInputs[0] / 2 != IncomingInputs[1] / 2 || + isDWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) { + // We have two non-adjacent or clobbered inputs we need to extract from + // the source half. To do this, we need to map them into some adjacent + // dword slot in the source mask. + int InputsFixed[2] = {IncomingInputs[0] - SourceOffset, + IncomingInputs[1] - SourceOffset}; - if (Subtarget->hasAVX2()) - return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32, - DAG.getNode(ISD::BITCAST, DL, MVT::v8f32, - DAG.getNode(ISD::BUILD_VECTOR, DL, - MVT::v8i32, VPermMask)), - V1); + // If there is a free slot in the source half mask adjacent to one of + // the inputs, place the other input in it. We use (Index XOR 1) to + // compute an adjacent index. + if (!isWordClobbered(SourceHalfMask, InputsFixed[0]) && + SourceHalfMask[InputsFixed[0] ^ 1] == -1) { + SourceHalfMask[InputsFixed[0]] = InputsFixed[0]; + SourceHalfMask[InputsFixed[0] ^ 1] = InputsFixed[1]; + InputsFixed[1] = InputsFixed[0] ^ 1; + } else if (!isWordClobbered(SourceHalfMask, InputsFixed[1]) && + SourceHalfMask[InputsFixed[1] ^ 1] == -1) { + SourceHalfMask[InputsFixed[1]] = InputsFixed[1]; + SourceHalfMask[InputsFixed[1] ^ 1] = InputsFixed[0]; + InputsFixed[0] = InputsFixed[1] ^ 1; + } else if (SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1)] == -1 && + SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1) + 1] == -1) { + // The two inputs are in the same DWord but it is clobbered and the + // adjacent DWord isn't used at all. Move both inputs to the free + // slot. + SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1)] = InputsFixed[0]; + SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1) + 1] = InputsFixed[1]; + InputsFixed[0] = 2 * ((InputsFixed[0] / 2) ^ 1); + InputsFixed[1] = 2 * ((InputsFixed[0] / 2) ^ 1) + 1; + } else { + // The only way we hit this point is if there is no clobbering + // (because there are no off-half inputs to this half) and there is no + // free slot adjacent to one of the inputs. In this case, we have to + // swap an input with a non-input. + for (int i = 0; i < 4; ++i) + assert((SourceHalfMask[i] == -1 || SourceHalfMask[i] == i) && + "We can't handle any clobbers here!"); + assert(InputsFixed[1] != (InputsFixed[0] ^ 1) && + "Cannot have adjacent inputs here!"); - // Otherwise, fall back. - return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v8f32, V1, V2, Mask, - DAG); - } + SourceHalfMask[InputsFixed[0] ^ 1] = InputsFixed[1]; + SourceHalfMask[InputsFixed[1]] = InputsFixed[0] ^ 1; - // Try to simplify this by merging 128-bit lanes to enable a lane-based - // shuffle. - if (SDValue Result = lowerVectorShuffleByMerging128BitLanes( - DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG)) - return Result; + // We also have to update the final source mask in this case because + // it may need to undo the above swap. + for (int &M : FinalSourceHalfMask) + if (M == (InputsFixed[0] ^ 1) + SourceOffset) + M = InputsFixed[1] + SourceOffset; + else if (M == InputsFixed[1] + SourceOffset) + M = (InputsFixed[0] ^ 1) + SourceOffset; - // If we have AVX2 then we always want to lower with a blend because at v8 we - // can fully permute the elements. - if (Subtarget->hasAVX2()) - return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8f32, V1, V2, - Mask, DAG); + InputsFixed[1] = InputsFixed[0] ^ 1; + } - // Otherwise fall back on generic lowering. - return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, DAG); -} + // Point everything at the fixed inputs. + for (int &M : HalfMask) + if (M == IncomingInputs[0]) + M = InputsFixed[0] + SourceOffset; + else if (M == IncomingInputs[1]) + M = InputsFixed[1] + SourceOffset; -/// \brief Handle lowering of 8-lane 32-bit integer shuffles. -/// -/// This routine is only called when we have AVX2 and thus a reasonable -/// instruction set for v8i32 shuffling.. -static SDValue lowerV8I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - SDLoc DL(Op); - assert(V1.getSimpleValueType() == MVT::v8i32 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v8i32 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"); - assert(Subtarget->hasAVX2() && "We can only lower v8i32 with AVX2!"); + IncomingInputs[0] = InputsFixed[0] + SourceOffset; + IncomingInputs[1] = InputsFixed[1] + SourceOffset; + } + } else { + llvm_unreachable("Unhandled input size!"); + } - // Whenever we can lower this as a zext, that instruction is strictly faster - // than any alternative. It also allows us to fold memory operands into the - // shuffle in many cases. - if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, - Mask, Subtarget, DAG)) - return ZExt; + // Now hoist the DWord down to the right half. + int FreeDWord = (PSHUFDMask[DestOffset / 2] == -1 ? 0 : 1) + DestOffset / 2; + assert(PSHUFDMask[FreeDWord] == -1 && "DWord not free"); + PSHUFDMask[FreeDWord] = IncomingInputs[0] / 2; + for (int &M : HalfMask) + for (int Input : IncomingInputs) + if (M == Input) + M = FreeDWord * 2 + Input % 2; + }; + moveInputsToRightHalf(HToLInputs, LToLInputs, PSHUFHMask, LoMask, HiMask, + /*SourceOffset*/ 4, /*DestOffset*/ 0); + moveInputsToRightHalf(LToHInputs, HToHInputs, PSHUFLMask, HiMask, LoMask, + /*SourceOffset*/ 0, /*DestOffset*/ 4); - if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask, - Subtarget, DAG)) - return Blend; + // Now enact all the shuffles we've computed to move the inputs into their + // target half. + if (!isNoopShuffleMask(PSHUFLMask)) + V = DAG.getNode(X86ISD::PSHUFLW, DL, MVT::v8i16, V, + getV4X86ShuffleImm8ForMask(PSHUFLMask, DAG)); + if (!isNoopShuffleMask(PSHUFHMask)) + V = DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16, V, + getV4X86ShuffleImm8ForMask(PSHUFHMask, DAG)); + if (!isNoopShuffleMask(PSHUFDMask)) + V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, + DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, + DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V), + getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG))); - // Check for being able to broadcast a single element. - if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v8i32, DL, V1, - Mask, Subtarget, DAG)) - return Broadcast; + // At this point, each half should contain all its inputs, and we can then + // just shuffle them into their final position. + assert(std::count_if(LoMask.begin(), LoMask.end(), + [](int M) { return M >= 4; }) == 0 && + "Failed to lift all the high half inputs to the low mask!"); + assert(std::count_if(HiMask.begin(), HiMask.end(), + [](int M) { return M >= 0 && M < 4; }) == 0 && + "Failed to lift all the low half inputs to the high mask!"); - // If the shuffle mask is repeated in each 128-bit lane we can use more - // efficient instructions that mirror the shuffles across the two 128-bit - // lanes. - SmallVector RepeatedMask; - if (is128BitLaneRepeatedShuffleMask(MVT::v8i32, Mask, RepeatedMask)) { - assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!"); - if (isSingleInputShuffleMask(Mask)) - return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1, - getV4X86ShuffleImm8ForMask(RepeatedMask, DAG)); + // Do a half shuffle for the low mask. + if (!isNoopShuffleMask(LoMask)) + V = DAG.getNode(X86ISD::PSHUFLW, DL, MVT::v8i16, V, + getV4X86ShuffleImm8ForMask(LoMask, DAG)); - // Use dedicated unpack instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 1, 9, 4, 12, 5, 13)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i32, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, 2, 10, 3, 11, 6, 14, 7, 15)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i32, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, 8, 0, 9, 1, 12, 4, 13, 5)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i32, V2, V1); - if (isShuffleEquivalent(V1, V2, Mask, 10, 2, 11, 3, 14, 6, 15, 7)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i32, V2, V1); - } + // Do a half shuffle with the high mask after shifting its values down. + for (int &M : HiMask) + if (M >= 0) + M -= 4; + if (!isNoopShuffleMask(HiMask)) + V = DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16, V, + getV4X86ShuffleImm8ForMask(HiMask, DAG)); - // Try to use shift instructions. - if (SDValue Shift = - lowerVectorShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask, DAG)) - return Shift; + return V; +} - if (SDValue Rotate = lowerVectorShuffleAsByteRotate( - DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG)) - return Rotate; +/// \brief Helper to form a PSHUFB-based shuffle+blend. +static SDValue lowerVectorShuffleAsPSHUFB(SDLoc DL, MVT VT, SDValue V1, + SDValue V2, ArrayRef Mask, + SelectionDAG &DAG, bool &V1InUse, + bool &V2InUse) { + SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2); + SDValue V1Mask[16]; + SDValue V2Mask[16]; + V1InUse = false; + V2InUse = false; - // If the shuffle patterns aren't repeated but it is a single input, directly - // generate a cross-lane VPERMD instruction. - if (isSingleInputShuffleMask(Mask)) { - SDValue VPermMask[8]; - for (int i = 0; i < 8; ++i) - VPermMask[i] = Mask[i] < 0 ? DAG.getUNDEF(MVT::i32) - : DAG.getConstant(Mask[i], MVT::i32); - return DAG.getNode( - X86ISD::VPERMV, DL, MVT::v8i32, - DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i32, VPermMask), V1); + int Size = Mask.size(); + int Scale = 16 / Size; + for (int i = 0; i < 16; ++i) { + if (Mask[i / Scale] == -1) { + V1Mask[i] = V2Mask[i] = DAG.getUNDEF(MVT::i8); + } else { + const int ZeroMask = 0x80; + int V1Idx = Mask[i / Scale] < Size ? Mask[i / Scale] * Scale + i % Scale + : ZeroMask; + int V2Idx = Mask[i / Scale] < Size + ? ZeroMask + : (Mask[i / Scale] - Size) * Scale + i % Scale; + if (Zeroable[i / Scale]) + V1Idx = V2Idx = ZeroMask; + V1Mask[i] = DAG.getConstant(V1Idx, MVT::i8); + V2Mask[i] = DAG.getConstant(V2Idx, MVT::i8); + V1InUse |= (ZeroMask != V1Idx); + V2InUse |= (ZeroMask != V2Idx); + } } - // Try to simplify this by merging 128-bit lanes to enable a lane-based - // shuffle. - if (SDValue Result = lowerVectorShuffleByMerging128BitLanes( - DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG)) - return Result; + if (V1InUse) + V1 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, + DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, V1), + DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, V1Mask)); + if (V2InUse) + V2 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, + DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, V2), + DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, V2Mask)); - // Otherwise fall back on generic blend lowering. - return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i32, V1, V2, - Mask, DAG); + // If we need shuffled inputs from both, blend the two. + SDValue V; + if (V1InUse && V2InUse) + V = DAG.getNode(ISD::OR, DL, MVT::v16i8, V1, V2); + else + V = V1InUse ? V1 : V2; + + // Cast the result back to the correct type. + return DAG.getNode(ISD::BITCAST, DL, VT, V); } -/// \brief Handle lowering of 16-lane 16-bit integer shuffles. +/// \brief Generic lowering of 8-lane i16 shuffles. /// -/// This routine is only called when we have AVX2 and thus a reasonable -/// instruction set for v16i16 shuffling.. -static SDValue lowerV16I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { +/// This handles both single-input shuffles and combined shuffle/blends with +/// two inputs. The single input shuffles are immediately delegated to +/// a dedicated lowering routine. +/// +/// The blends are lowered in one of three fundamental ways. If there are few +/// enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle +/// of the input is significantly cheaper when lowered as an interleaving of +/// the two inputs, try to interleave them. Otherwise, blend the low and high +/// halves of the inputs separately (making them have relatively few inputs) +/// and then concatenate them. +static SDValue lowerV8I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { SDLoc DL(Op); - assert(V1.getSimpleValueType() == MVT::v16i16 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v16i16 && "Bad operand type!"); + assert(Op.getSimpleValueType() == MVT::v8i16 && "Bad shuffle type!"); + assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!"); ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!"); - assert(Subtarget->hasAVX2() && "We can only lower v16i16 with AVX2!"); + ArrayRef OrigMask = SVOp->getMask(); + int MaskStorage[8] = {OrigMask[0], OrigMask[1], OrigMask[2], OrigMask[3], + OrigMask[4], OrigMask[5], OrigMask[6], OrigMask[7]}; + MutableArrayRef Mask(MaskStorage); + + assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"); // Whenever we can lower this as a zext, that instruction is strictly faster - // than any alternative. It also allows us to fold memory operands into the - // shuffle in many cases. - if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v16i16, V1, V2, - Mask, Subtarget, DAG)) + // than any alternative. + if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend( + DL, MVT::v8i16, V1, V2, OrigMask, Subtarget, DAG)) return ZExt; - // Check for being able to broadcast a single element. - if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v16i16, DL, V1, - Mask, Subtarget, DAG)) - return Broadcast; + auto isV1 = [](int M) { return M >= 0 && M < 8; }; + auto isV2 = [](int M) { return M >= 8; }; - if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask, - Subtarget, DAG)) - return Blend; + int NumV1Inputs = std::count_if(Mask.begin(), Mask.end(), isV1); + int NumV2Inputs = std::count_if(Mask.begin(), Mask.end(), isV2); - // Use dedicated unpack instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, Mask, - // First 128-bit lane: - 0, 16, 1, 17, 2, 18, 3, 19, - // Second 128-bit lane: - 8, 24, 9, 25, 10, 26, 11, 27)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i16, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, - // First 128-bit lane: - 4, 20, 5, 21, 6, 22, 7, 23, - // Second 128-bit lane: - 12, 28, 13, 29, 14, 30, 15, 31)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i16, V1, V2); + if (NumV2Inputs == 0) + return lowerV8I16SingleInputVectorShuffle(DL, V1, Mask, Subtarget, DAG); + + assert(NumV1Inputs > 0 && "All single-input shuffles should be canonicalized " + "to be V1-input shuffles."); // Try to use shift instructions. if (SDValue Shift = - lowerVectorShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask, DAG)) + lowerVectorShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask, DAG)) return Shift; + // There are special ways we can lower some single-element blends. + if (NumV2Inputs == 1) + if (SDValue V = lowerVectorShuffleAsElementInsertion(MVT::v8i16, DL, V1, V2, + Mask, Subtarget, DAG)) + return V; + + // We have different paths for blend lowering, but they all must use the + // *exact* same predicate. + bool IsBlendSupported = Subtarget->hasSSE41(); + if (IsBlendSupported) + if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask, + Subtarget, DAG)) + return Blend; + + if (SDValue Masked = + lowerVectorShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask, DAG)) + return Masked; + + // Use dedicated unpack instructions for masks that match their pattern. + if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 1, 9, 2, 10, 3, 11)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i16, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, 4, 12, 5, 13, 6, 14, 7, 15)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i16, V1, V2); + // Try to use byte rotation instructions. if (SDValue Rotate = lowerVectorShuffleAsByteRotate( - DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG)) + DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG)) return Rotate; - if (isSingleInputShuffleMask(Mask)) { - // There are no generalized cross-lane shuffle operations available on i16 - // element types. - if (is128BitLaneCrossingShuffleMask(MVT::v16i16, Mask)) - return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v16i16, V1, V2, - Mask, DAG); + if (SDValue BitBlend = + lowerVectorShuffleAsBitBlend(DL, MVT::v8i16, V1, V2, Mask, DAG)) + return BitBlend; - SDValue PSHUFBMask[32]; - for (int i = 0; i < 16; ++i) { - if (Mask[i] == -1) { - PSHUFBMask[2 * i] = PSHUFBMask[2 * i + 1] = DAG.getUNDEF(MVT::i8); - continue; - } + if (SDValue Unpack = + lowerVectorShuffleAsUnpack(MVT::v8i16, DL, V1, V2, Mask, DAG)) + return Unpack; - int M = i < 8 ? Mask[i] : Mask[i] - 8; - assert(M >= 0 && M < 8 && "Invalid single-input mask!"); - PSHUFBMask[2 * i] = DAG.getConstant(2 * M, MVT::i8); - PSHUFBMask[2 * i + 1] = DAG.getConstant(2 * M + 1, MVT::i8); - } - return DAG.getNode( - ISD::BITCAST, DL, MVT::v16i16, - DAG.getNode( - X86ISD::PSHUFB, DL, MVT::v32i8, - DAG.getNode(ISD::BITCAST, DL, MVT::v32i8, V1), - DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, PSHUFBMask))); + // If we can't directly blend but can use PSHUFB, that will be better as it + // can both shuffle and set up the inefficient blend. + if (!IsBlendSupported && Subtarget->hasSSSE3()) { + bool V1InUse, V2InUse; + return lowerVectorShuffleAsPSHUFB(DL, MVT::v8i16, V1, V2, Mask, DAG, + V1InUse, V2InUse); } - // Try to simplify this by merging 128-bit lanes to enable a lane-based - // shuffle. - if (SDValue Result = lowerVectorShuffleByMerging128BitLanes( - DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG)) - return Result; + // We can always bit-blend if we have to so the fallback strategy is to + // decompose into single-input permutes and blends. + return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i16, V1, V2, + Mask, DAG); +} - // Otherwise fall back on generic lowering. - return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask, DAG); +/// \brief Check whether a compaction lowering can be done by dropping even +/// elements and compute how many times even elements must be dropped. +/// +/// This handles shuffles which take every Nth element where N is a power of +/// two. Example shuffle masks: +/// +/// N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14 +/// N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 +/// N = 2: 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12 +/// N = 2: 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28 +/// N = 3: 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8 +/// N = 3: 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24 +/// +/// Any of these lanes can of course be undef. +/// +/// This routine only supports N <= 3. +/// FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here +/// for larger N. +/// +/// \returns N above, or the number of times even elements must be dropped if +/// there is such a number. Otherwise returns zero. +static int canLowerByDroppingEvenElements(ArrayRef Mask) { + // Figure out whether we're looping over two inputs or just one. + bool IsSingleInput = isSingleInputShuffleMask(Mask); + + // The modulus for the shuffle vector entries is based on whether this is + // a single input or not. + int ShuffleModulus = Mask.size() * (IsSingleInput ? 1 : 2); + assert(isPowerOf2_32((uint32_t)ShuffleModulus) && + "We should only be called with masks with a power-of-2 size!"); + + uint64_t ModMask = (uint64_t)ShuffleModulus - 1; + + // We track whether the input is viable for all power-of-2 strides 2^1, 2^2, + // and 2^3 simultaneously. This is because we may have ambiguity with + // partially undef inputs. + bool ViableForN[3] = {true, true, true}; + + for (int i = 0, e = Mask.size(); i < e; ++i) { + // Ignore undef lanes, we'll optimistically collapse them to the pattern we + // want. + if (Mask[i] == -1) + continue; + + bool IsAnyViable = false; + for (unsigned j = 0; j != array_lengthof(ViableForN); ++j) + if (ViableForN[j]) { + uint64_t N = j + 1; + + // The shuffle mask must be equal to (i * 2^N) % M. + if ((uint64_t)Mask[i] == (((uint64_t)i << N) & ModMask)) + IsAnyViable = true; + else + ViableForN[j] = false; + } + // Early exit if we exhaust the possible powers of two. + if (!IsAnyViable) + break; + } + + for (unsigned j = 0; j != array_lengthof(ViableForN); ++j) + if (ViableForN[j]) + return j + 1; + + // Return 0 as there is no viable power of two. + return 0; } -/// \brief Handle lowering of 32-lane 8-bit integer shuffles. +/// \brief Generic lowering of v16i8 shuffles. /// -/// This routine is only called when we have AVX2 and thus a reasonable -/// instruction set for v32i8 shuffling.. -static SDValue lowerV32I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2, +/// This is a hybrid strategy to lower v16i8 vectors. It first attempts to +/// detect any complexity reducing interleaving. If that doesn't help, it uses +/// UNPCK to spread the i8 elements across two i16-element vectors, and uses +/// the existing lowering for v8i16 blends on each half, finally PACK-ing them +/// back together. +static SDValue lowerV16I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) { SDLoc DL(Op); - assert(V1.getSimpleValueType() == MVT::v32i8 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v32i8 && "Bad operand type!"); + assert(Op.getSimpleValueType() == MVT::v16i8 && "Bad shuffle type!"); + assert(V1.getSimpleValueType() == MVT::v16i8 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!"); ShuffleVectorSDNode *SVOp = cast(Op); ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!"); - assert(Subtarget->hasAVX2() && "We can only lower v32i8 with AVX2!"); + assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!"); - // Whenever we can lower this as a zext, that instruction is strictly faster - // than any alternative. It also allows us to fold memory operands into the - // shuffle in many cases. - if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v32i8, V1, V2, - Mask, Subtarget, DAG)) + // Try to use shift instructions. + if (SDValue Shift = + lowerVectorShuffleAsShift(DL, MVT::v16i8, V1, V2, Mask, DAG)) + return Shift; + + // Try to use byte rotation instructions. + if (SDValue Rotate = lowerVectorShuffleAsByteRotate( + DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG)) + return Rotate; + + // Try to use a zext lowering. + if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend( + DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG)) return ZExt; - // Check for being able to broadcast a single element. - if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v32i8, DL, V1, - Mask, Subtarget, DAG)) - return Broadcast; + int NumV2Elements = + std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 16; }); - if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask, - Subtarget, DAG)) - return Blend; + // For single-input shuffles, there are some nicer lowering tricks we can use. + if (NumV2Elements == 0) { + // Check for being able to broadcast a single element. + if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v16i8, DL, V1, + Mask, Subtarget, DAG)) + return Broadcast; - // Use dedicated unpack instructions for masks that match their pattern. - // Note that these are repeated 128-bit lane unpacks, not unpacks across all - // 256-bit lanes. - if (isShuffleEquivalent( - V1, V2, Mask, - // First 128-bit lane: - 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, - // Second 128-bit lane: - 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v32i8, V1, V2); - if (isShuffleEquivalent( - V1, V2, Mask, - // First 128-bit lane: - 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47, - // Second 128-bit lane: - 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v32i8, V1, V2); + // Check whether we can widen this to an i16 shuffle by duplicating bytes. + // Notably, this handles splat and partial-splat shuffles more efficiently. + // However, it only makes sense if the pre-duplication shuffle simplifies + // things significantly. Currently, this means we need to be able to + // express the pre-duplication shuffle as an i16 shuffle. + // + // FIXME: We should check for other patterns which can be widened into an + // i16 shuffle as well. + auto canWidenViaDuplication = [](ArrayRef Mask) { + for (int i = 0; i < 16; i += 2) + if (Mask[i] != -1 && Mask[i + 1] != -1 && Mask[i] != Mask[i + 1]) + return false; + + return true; + }; + auto tryToWidenViaDuplication = [&]() -> SDValue { + if (!canWidenViaDuplication(Mask)) + return SDValue(); + SmallVector LoInputs; + std::copy_if(Mask.begin(), Mask.end(), std::back_inserter(LoInputs), + [](int M) { return M >= 0 && M < 8; }); + std::sort(LoInputs.begin(), LoInputs.end()); + LoInputs.erase(std::unique(LoInputs.begin(), LoInputs.end()), + LoInputs.end()); + SmallVector HiInputs; + std::copy_if(Mask.begin(), Mask.end(), std::back_inserter(HiInputs), + [](int M) { return M >= 8; }); + std::sort(HiInputs.begin(), HiInputs.end()); + HiInputs.erase(std::unique(HiInputs.begin(), HiInputs.end()), + HiInputs.end()); - // Try to use shift instructions. - if (SDValue Shift = - lowerVectorShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask, DAG)) - return Shift; + bool TargetLo = LoInputs.size() >= HiInputs.size(); + ArrayRef InPlaceInputs = TargetLo ? LoInputs : HiInputs; + ArrayRef MovingInputs = TargetLo ? HiInputs : LoInputs; - // Try to use byte rotation instructions. - if (SDValue Rotate = lowerVectorShuffleAsByteRotate( - DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG)) - return Rotate; + int PreDupI16Shuffle[] = {-1, -1, -1, -1, -1, -1, -1, -1}; + SmallDenseMap LaneMap; + for (int I : InPlaceInputs) { + PreDupI16Shuffle[I/2] = I/2; + LaneMap[I] = I; + } + int j = TargetLo ? 0 : 4, je = j + 4; + for (int i = 0, ie = MovingInputs.size(); i < ie; ++i) { + // Check if j is already a shuffle of this input. This happens when + // there are two adjacent bytes after we move the low one. + if (PreDupI16Shuffle[j] != MovingInputs[i] / 2) { + // If we haven't yet mapped the input, search for a slot into which + // we can map it. + while (j < je && PreDupI16Shuffle[j] != -1) + ++j; - if (isSingleInputShuffleMask(Mask)) { - // There are no generalized cross-lane shuffle operations available on i8 - // element types. - if (is128BitLaneCrossingShuffleMask(MVT::v32i8, Mask)) - return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v32i8, V1, V2, - Mask, DAG); + if (j == je) + // We can't place the inputs into a single half with a simple i16 shuffle, so bail. + return SDValue(); - SDValue PSHUFBMask[32]; - for (int i = 0; i < 32; ++i) - PSHUFBMask[i] = - Mask[i] < 0 - ? DAG.getUNDEF(MVT::i8) - : DAG.getConstant(Mask[i] < 16 ? Mask[i] : Mask[i] - 16, MVT::i8); + // Map this input with the i16 shuffle. + PreDupI16Shuffle[j] = MovingInputs[i] / 2; + } - return DAG.getNode( - X86ISD::PSHUFB, DL, MVT::v32i8, V1, - DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, PSHUFBMask)); - } + // Update the lane map based on the mapping we ended up with. + LaneMap[MovingInputs[i]] = 2 * j + MovingInputs[i] % 2; + } + V1 = DAG.getNode( + ISD::BITCAST, DL, MVT::v16i8, + DAG.getVectorShuffle(MVT::v8i16, DL, + DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1), + DAG.getUNDEF(MVT::v8i16), PreDupI16Shuffle)); - // Try to simplify this by merging 128-bit lanes to enable a lane-based - // shuffle. - if (SDValue Result = lowerVectorShuffleByMerging128BitLanes( - DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG)) - return Result; + // Unpack the bytes to form the i16s that will be shuffled into place. + V1 = DAG.getNode(TargetLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL, + MVT::v16i8, V1, V1); - // Otherwise fall back on generic lowering. - return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask, DAG); -} + int PostDupI16Shuffle[8] = {-1, -1, -1, -1, -1, -1, -1, -1}; + for (int i = 0; i < 16; ++i) + if (Mask[i] != -1) { + int MappedMask = LaneMap[Mask[i]] - (TargetLo ? 0 : 8); + assert(MappedMask < 8 && "Invalid v8 shuffle mask!"); + if (PostDupI16Shuffle[i / 2] == -1) + PostDupI16Shuffle[i / 2] = MappedMask; + else + assert(PostDupI16Shuffle[i / 2] == MappedMask && + "Conflicting entrties in the original shuffle!"); + } + return DAG.getNode( + ISD::BITCAST, DL, MVT::v16i8, + DAG.getVectorShuffle(MVT::v8i16, DL, + DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1), + DAG.getUNDEF(MVT::v8i16), PostDupI16Shuffle)); + }; + if (SDValue V = tryToWidenViaDuplication()) + return V; + } -/// \brief High-level routine to lower various 256-bit x86 vector shuffles. -/// -/// This routine either breaks down the specific type of a 256-bit x86 vector -/// shuffle or splits it into two 128-bit shuffles and fuses the results back -/// together based on the available instructions. -static SDValue lower256BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2, - MVT VT, const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - SDLoc DL(Op); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); + // Use dedicated unpack instructions for masks that match their pattern. + if (isShuffleEquivalent(V1, V2, Mask, + 0, 16, 1, 17, 2, 18, 3, 19, + 4, 20, 5, 21, 6, 22, 7, 23)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, + 8, 24, 9, 25, 10, 26, 11, 27, + 12, 28, 13, 29, 14, 30, 15, 31)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V1, V2); - // There is a really nice hard cut-over between AVX1 and AVX2 that means we can - // check for those subtargets here and avoid much of the subtarget querying in - // the per-vector-type lowering routines. With AVX1 we have essentially *zero* - // ability to manipulate a 256-bit vector with integer types. Since we'll use - // floating point types there eventually, just immediately cast everything to - // a float and operate entirely in that domain. - if (VT.isInteger() && !Subtarget->hasAVX2()) { - int ElementBits = VT.getScalarSizeInBits(); - if (ElementBits < 32) - // No floating point type available, decompose into 128-bit vectors. - return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG); + // Check for SSSE3 which lets us lower all v16i8 shuffles much more directly + // with PSHUFB. It is important to do this before we attempt to generate any + // blends but after all of the single-input lowerings. If the single input + // lowerings can find an instruction sequence that is faster than a PSHUFB, we + // want to preserve that and we can DAG combine any longer sequences into + // a PSHUFB in the end. But once we start blending from multiple inputs, + // the complexity of DAG combining bad patterns back into PSHUFB is too high, + // and there are *very* few patterns that would actually be faster than the + // PSHUFB approach because of its ability to zero lanes. + // + // FIXME: The only exceptions to the above are blends which are exact + // interleavings with direct instructions supporting them. We currently don't + // handle those well here. + if (Subtarget->hasSSSE3()) { + bool V1InUse = false; + bool V2InUse = false; - MVT FpVT = MVT::getVectorVT(MVT::getFloatingPointVT(ElementBits), - VT.getVectorNumElements()); - V1 = DAG.getNode(ISD::BITCAST, DL, FpVT, V1); - V2 = DAG.getNode(ISD::BITCAST, DL, FpVT, V2); - return DAG.getNode(ISD::BITCAST, DL, VT, - DAG.getVectorShuffle(FpVT, DL, V1, V2, Mask)); - } + SDValue PSHUFB = lowerVectorShuffleAsPSHUFB(DL, MVT::v16i8, V1, V2, Mask, + DAG, V1InUse, V2InUse); - switch (VT.SimpleTy) { - case MVT::v4f64: - return lowerV4F64VectorShuffle(Op, V1, V2, Subtarget, DAG); - case MVT::v4i64: - return lowerV4I64VectorShuffle(Op, V1, V2, Subtarget, DAG); - case MVT::v8f32: - return lowerV8F32VectorShuffle(Op, V1, V2, Subtarget, DAG); - case MVT::v8i32: - return lowerV8I32VectorShuffle(Op, V1, V2, Subtarget, DAG); - case MVT::v16i16: - return lowerV16I16VectorShuffle(Op, V1, V2, Subtarget, DAG); - case MVT::v32i8: - return lowerV32I8VectorShuffle(Op, V1, V2, Subtarget, DAG); + // If both V1 and V2 are in use and we can use a direct blend or an unpack, + // do so. This avoids using them to handle blends-with-zero which is + // important as a single pshufb is significantly faster for that. + if (V1InUse && V2InUse) { + if (Subtarget->hasSSE41()) + if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i8, V1, V2, + Mask, Subtarget, DAG)) + return Blend; - default: - llvm_unreachable("Not a valid 256-bit x86 vector type!"); - } -} + // We can use an unpack to do the blending rather than an or in some + // cases. Even though the or may be (very minorly) more efficient, we + // preference this lowering because there are common cases where part of + // the complexity of the shuffles goes away when we do the final blend as + // an unpack. + // FIXME: It might be worth trying to detect if the unpack-feeding + // shuffles will both be pshufb, in which case we shouldn't bother with + // this. + if (SDValue Unpack = + lowerVectorShuffleAsUnpack(MVT::v16i8, DL, V1, V2, Mask, DAG)) + return Unpack; + } -/// \brief Handle lowering of 8-lane 64-bit floating point shuffles. -static SDValue lowerV8F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - SDLoc DL(Op); - assert(V1.getSimpleValueType() == MVT::v8f64 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v8f64 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"); + return PSHUFB; + } - // X86 has dedicated unpack instructions that can handle specific blend - // operations: UNPCKH and UNPCKL. - if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 2, 10, 4, 12, 6, 14)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8f64, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, 1, 9, 3, 11, 5, 13, 7, 15)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8f64, V1, V2); + // There are special ways we can lower some single-element blends. + if (NumV2Elements == 1) + if (SDValue V = lowerVectorShuffleAsElementInsertion(MVT::v16i8, DL, V1, V2, + Mask, Subtarget, DAG)) + return V; - // FIXME: Implement direct support for this type! - return splitAndLowerVectorShuffle(DL, MVT::v8f64, V1, V2, Mask, DAG); -} + if (SDValue BitBlend = + lowerVectorShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG)) + return BitBlend; -/// \brief Handle lowering of 16-lane 32-bit floating point shuffles. -static SDValue lowerV16F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - SDLoc DL(Op); - assert(V1.getSimpleValueType() == MVT::v16f32 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v16f32 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!"); + // Check whether a compaction lowering can be done. This handles shuffles + // which take every Nth element for some even N. See the helper function for + // details. + // + // We special case these as they can be particularly efficiently handled with + // the PACKUSB instruction on x86 and they show up in common patterns of + // rearranging bytes to truncate wide elements. + if (int NumEvenDrops = canLowerByDroppingEvenElements(Mask)) { + // NumEvenDrops is the power of two stride of the elements. Another way of + // thinking about it is that we need to drop the even elements this many + // times to get the original input. + bool IsSingleInput = isSingleInputShuffleMask(Mask); - // Use dedicated unpack instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, Mask, - 0, 16, 1, 17, 4, 20, 5, 21, - 8, 24, 9, 25, 12, 28, 13, 29)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16f32, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, - 2, 18, 3, 19, 6, 22, 7, 23, - 10, 26, 11, 27, 14, 30, 15, 31)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16f32, V1, V2); + // First we need to zero all the dropped bytes. + assert(NumEvenDrops <= 3 && + "No support for dropping even elements more than 3 times."); + // We use the mask type to pick which bytes are preserved based on how many + // elements are dropped. + MVT MaskVTs[] = { MVT::v8i16, MVT::v4i32, MVT::v2i64 }; + SDValue ByteClearMask = + DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, + DAG.getConstant(0xFF, MaskVTs[NumEvenDrops - 1])); + V1 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V1, ByteClearMask); + if (!IsSingleInput) + V2 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V2, ByteClearMask); - // FIXME: Implement direct support for this type! - return splitAndLowerVectorShuffle(DL, MVT::v16f32, V1, V2, Mask, DAG); -} + // Now pack things back together. + V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1); + V2 = IsSingleInput ? V1 : DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V2); + SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, V2); + for (int i = 1; i < NumEvenDrops; ++i) { + Result = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, Result); + Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result); + } -/// \brief Handle lowering of 8-lane 64-bit integer shuffles. -static SDValue lowerV8I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - SDLoc DL(Op); - assert(V1.getSimpleValueType() == MVT::v8i64 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v8i64 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"); + return Result; + } - // X86 has dedicated unpack instructions that can handle specific blend - // operations: UNPCKH and UNPCKL. - if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 2, 10, 4, 12, 6, 14)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i64, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, 1, 9, 3, 11, 5, 13, 7, 15)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i64, V1, V2); + // Handle multi-input cases by blending single-input shuffles. + if (NumV2Elements > 0) + return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v16i8, V1, V2, + Mask, DAG); - // FIXME: Implement direct support for this type! - return splitAndLowerVectorShuffle(DL, MVT::v8i64, V1, V2, Mask, DAG); -} + // The fallback path for single-input shuffles widens this into two v8i16 + // vectors with unpacks, shuffles those, and then pulls them back together + // with a pack. + SDValue V = V1; -/// \brief Handle lowering of 16-lane 32-bit integer shuffles. -static SDValue lowerV16I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - SDLoc DL(Op); - assert(V1.getSimpleValueType() == MVT::v16i32 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v16i32 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!"); + int LoBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1}; + int HiBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1}; + for (int i = 0; i < 16; ++i) + if (Mask[i] >= 0) + (i < 8 ? LoBlendMask[i] : HiBlendMask[i % 8]) = Mask[i]; - // Use dedicated unpack instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, Mask, - 0, 16, 1, 17, 4, 20, 5, 21, - 8, 24, 9, 25, 12, 28, 13, 29)) - return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i32, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, - 2, 18, 3, 19, 6, 22, 7, 23, - 10, 26, 11, 27, 14, 30, 15, 31)) - return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i32, V1, V2); + SDValue Zero = getZeroVector(MVT::v8i16, Subtarget, DAG, DL); - // FIXME: Implement direct support for this type! - return splitAndLowerVectorShuffle(DL, MVT::v16i32, V1, V2, Mask, DAG); -} + SDValue VLoHalf, VHiHalf; + // Check if any of the odd lanes in the v16i8 are used. If not, we can mask + // them out and avoid using UNPCK{L,H} to extract the elements of V as + // i16s. + if (std::none_of(std::begin(LoBlendMask), std::end(LoBlendMask), + [](int M) { return M >= 0 && M % 2 == 1; }) && + std::none_of(std::begin(HiBlendMask), std::end(HiBlendMask), + [](int M) { return M >= 0 && M % 2 == 1; })) { + // Use a mask to drop the high bytes. + VLoHalf = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V); + VLoHalf = DAG.getNode(ISD::AND, DL, MVT::v8i16, VLoHalf, + DAG.getConstant(0x00FF, MVT::v8i16)); -/// \brief Handle lowering of 32-lane 16-bit integer shuffles. -static SDValue lowerV32I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - SDLoc DL(Op); - assert(V1.getSimpleValueType() == MVT::v32i16 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v32i16 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!"); - assert(Subtarget->hasBWI() && "We can only lower v32i16 with AVX-512-BWI!"); + // This will be a single vector shuffle instead of a blend so nuke VHiHalf. + VHiHalf = DAG.getUNDEF(MVT::v8i16); - // FIXME: Implement direct support for this type! - return splitAndLowerVectorShuffle(DL, MVT::v32i16, V1, V2, Mask, DAG); -} + // Squash the masks to point directly into VLoHalf. + for (int &M : LoBlendMask) + if (M >= 0) + M /= 2; + for (int &M : HiBlendMask) + if (M >= 0) + M /= 2; + } else { + // Otherwise just unpack the low half of V into VLoHalf and the high half into + // VHiHalf so that we can blend them as i16s. + VLoHalf = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, + DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero)); + VHiHalf = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, + DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero)); + } -/// \brief Handle lowering of 64-lane 8-bit integer shuffles. -static SDValue lowerV64I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - SDLoc DL(Op); - assert(V1.getSimpleValueType() == MVT::v64i8 && "Bad operand type!"); - assert(V2.getSimpleValueType() == MVT::v64i8 && "Bad operand type!"); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Mask.size() == 64 && "Unexpected mask size for v64 shuffle!"); - assert(Subtarget->hasBWI() && "We can only lower v64i8 with AVX-512-BWI!"); + SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask); + SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask); - // FIXME: Implement direct support for this type! - return splitAndLowerVectorShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG); + return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV); } -/// \brief High-level routine to lower various 512-bit x86 vector shuffles. +/// \brief Dispatching routine to lower various 128-bit x86 vector shuffles. /// -/// This routine either breaks down the specific type of a 512-bit x86 vector -/// shuffle or splits it into two 256-bit shuffles and fuses the results back -/// together based on the available instructions. -static SDValue lower512BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2, +/// This routine breaks down the specific type of 128-bit shuffle and +/// dispatches to the lowering routines accordingly. +static SDValue lower128BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2, MVT VT, const X86Subtarget *Subtarget, SelectionDAG &DAG) { - SDLoc DL(Op); - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - assert(Subtarget->hasAVX512() && - "Cannot lower 512-bit vectors w/ basic ISA!"); - - // Check for being able to broadcast a single element. - if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(VT.SimpleTy, DL, V1, - Mask, Subtarget, DAG)) - return Broadcast; - - // Dispatch to each element type for lowering. If we don't have supprot for - // specific element type shuffles at 512 bits, immediately split them and - // lower them. Each lowering routine of a given type is allowed to assume that - // the requisite ISA extensions for that element type are available. switch (VT.SimpleTy) { - case MVT::v8f64: - return lowerV8F64VectorShuffle(Op, V1, V2, Subtarget, DAG); - case MVT::v16f32: - return lowerV16F32VectorShuffle(Op, V1, V2, Subtarget, DAG); - case MVT::v8i64: - return lowerV8I64VectorShuffle(Op, V1, V2, Subtarget, DAG); - case MVT::v16i32: - return lowerV16I32VectorShuffle(Op, V1, V2, Subtarget, DAG); - case MVT::v32i16: - if (Subtarget->hasBWI()) - return lowerV32I16VectorShuffle(Op, V1, V2, Subtarget, DAG); - break; - case MVT::v64i8: - if (Subtarget->hasBWI()) - return lowerV64I8VectorShuffle(Op, V1, V2, Subtarget, DAG); - break; + case MVT::v2i64: + return lowerV2I64VectorShuffle(Op, V1, V2, Subtarget, DAG); + case MVT::v2f64: + return lowerV2F64VectorShuffle(Op, V1, V2, Subtarget, DAG); + case MVT::v4i32: + return lowerV4I32VectorShuffle(Op, V1, V2, Subtarget, DAG); + case MVT::v4f32: + return lowerV4F32VectorShuffle(Op, V1, V2, Subtarget, DAG); + case MVT::v8i16: + return lowerV8I16VectorShuffle(Op, V1, V2, Subtarget, DAG); + case MVT::v16i8: + return lowerV16I8VectorShuffle(Op, V1, V2, Subtarget, DAG); default: - llvm_unreachable("Not a valid 512-bit x86 vector type!"); + llvm_unreachable("Unimplemented!"); } - - // Otherwise fall back on splitting. - return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG); } -/// \brief Top-level lowering for x86 vector shuffles. +/// \brief Helper function to test whether a shuffle mask could be +/// simplified by widening the elements being shuffled. /// -/// This handles decomposition, canonicalization, and lowering of all x86 -/// vector shuffles. Most of the specific lowering strategies are encapsulated -/// above in helper routines. The canonicalization attempts to widen shuffles -/// to involve fewer lanes of wider elements, consolidate symmetric patterns -/// s.t. only one of the two inputs needs to be tested, etc. -static SDValue lowerVectorShuffle(SDValue Op, const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - ShuffleVectorSDNode *SVOp = cast(Op); - ArrayRef Mask = SVOp->getMask(); - SDValue V1 = Op.getOperand(0); - SDValue V2 = Op.getOperand(1); - MVT VT = Op.getSimpleValueType(); - int NumElements = VT.getVectorNumElements(); - SDLoc dl(Op); +/// Appends the mask for wider elements in WidenedMask if valid. Otherwise +/// leaves it in an unspecified state. +/// +/// NOTE: This must handle normal vector shuffle masks and *target* vector +/// shuffle masks. The latter have the special property of a '-2' representing +/// a zero-ed lane of a vector. +static bool canWidenShuffleElements(ArrayRef Mask, + SmallVectorImpl &WidenedMask) { + for (int i = 0, Size = Mask.size(); i < Size; i += 2) { + // If both elements are undef, its trivial. + if (Mask[i] == SM_SentinelUndef && Mask[i + 1] == SM_SentinelUndef) { + WidenedMask.push_back(SM_SentinelUndef); + continue; + } - assert(VT.getSizeInBits() != 64 && "Can't lower MMX shuffles"); + // Check for an undef mask and a mask value properly aligned to fit with + // a pair of values. If we find such a case, use the non-undef mask's value. + if (Mask[i] == SM_SentinelUndef && Mask[i + 1] >= 0 && Mask[i + 1] % 2 == 1) { + WidenedMask.push_back(Mask[i + 1] / 2); + continue; + } + if (Mask[i + 1] == SM_SentinelUndef && Mask[i] >= 0 && Mask[i] % 2 == 0) { + WidenedMask.push_back(Mask[i] / 2); + continue; + } + + // When zeroing, we need to spread the zeroing across both lanes to widen. + if (Mask[i] == SM_SentinelZero || Mask[i + 1] == SM_SentinelZero) { + if ((Mask[i] == SM_SentinelZero || Mask[i] == SM_SentinelUndef) && + (Mask[i + 1] == SM_SentinelZero || Mask[i + 1] == SM_SentinelUndef)) { + WidenedMask.push_back(SM_SentinelZero); + continue; + } + return false; + } + + // Finally check if the two mask values are adjacent and aligned with + // a pair. + if (Mask[i] != SM_SentinelUndef && Mask[i] % 2 == 0 && Mask[i] + 1 == Mask[i + 1]) { + WidenedMask.push_back(Mask[i] / 2); + continue; + } + + // Otherwise we can't safely widen the elements used in this shuffle. + return false; + } + assert(WidenedMask.size() == Mask.size() / 2 && + "Incorrect size of mask after widening the elements!"); + + return true; +} - bool V1IsUndef = V1.getOpcode() == ISD::UNDEF; - bool V2IsUndef = V2.getOpcode() == ISD::UNDEF; - if (V1IsUndef && V2IsUndef) - return DAG.getUNDEF(VT); +/// \brief Generic routine to split vector shuffle into half-sized shuffles. +/// +/// This routine just extracts two subvectors, shuffles them independently, and +/// then concatenates them back together. This should work effectively with all +/// AVX vector shuffle types. +static SDValue splitAndLowerVectorShuffle(SDLoc DL, MVT VT, SDValue V1, + SDValue V2, ArrayRef Mask, + SelectionDAG &DAG) { + assert(VT.getSizeInBits() >= 256 && + "Only for 256-bit or wider vector shuffles!"); + assert(V1.getSimpleValueType() == VT && "Bad operand type!"); + assert(V2.getSimpleValueType() == VT && "Bad operand type!"); - // When we create a shuffle node we put the UNDEF node to second operand, - // but in some cases the first operand may be transformed to UNDEF. - // In this case we should just commute the node. - if (V1IsUndef) - return DAG.getCommutedVectorShuffle(*SVOp); + ArrayRef LoMask = Mask.slice(0, Mask.size() / 2); + ArrayRef HiMask = Mask.slice(Mask.size() / 2); - // Check for non-undef masks pointing at an undef vector and make the masks - // undef as well. This makes it easier to match the shuffle based solely on - // the mask. - if (V2IsUndef) - for (int M : Mask) - if (M >= NumElements) { - SmallVector NewMask(Mask.begin(), Mask.end()); - for (int &M : NewMask) - if (M >= NumElements) - M = -1; - return DAG.getVectorShuffle(VT, dl, V1, V2, NewMask); - } + int NumElements = VT.getVectorNumElements(); + int SplitNumElements = NumElements / 2; + MVT ScalarVT = VT.getScalarType(); + MVT SplitVT = MVT::getVectorVT(ScalarVT, NumElements / 2); - // We actually see shuffles that are entirely re-arrangements of a set of - // zero inputs. This mostly happens while decomposing complex shuffles into - // simple ones. Directly lower these as a buildvector of zeros. - SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2); - if (Zeroable.all()) - return getZeroVector(VT, Subtarget, DAG, dl); + // Rather than splitting build-vectors, just build two narrower build + // vectors. This helps shuffling with splats and zeros. + auto SplitVector = [&](SDValue V) { + while (V.getOpcode() == ISD::BITCAST) + V = V->getOperand(0); - // Try to collapse shuffles into using a vector type with fewer elements but - // wider element types. We cap this to not form integers or floating point - // elements wider than 64 bits, but it might be interesting to form i128 - // integers to handle flipping the low and high halves of AVX 256-bit vectors. - SmallVector WidenedMask; - if (VT.getScalarSizeInBits() < 64 && - canWidenShuffleElements(Mask, WidenedMask)) { - MVT NewEltVT = VT.isFloatingPoint() - ? MVT::getFloatingPointVT(VT.getScalarSizeInBits() * 2) - : MVT::getIntegerVT(VT.getScalarSizeInBits() * 2); - MVT NewVT = MVT::getVectorVT(NewEltVT, VT.getVectorNumElements() / 2); - // Make sure that the new vector type is legal. For example, v2f64 isn't - // legal on SSE1. - if (DAG.getTargetLoweringInfo().isTypeLegal(NewVT)) { - V1 = DAG.getNode(ISD::BITCAST, dl, NewVT, V1); - V2 = DAG.getNode(ISD::BITCAST, dl, NewVT, V2); - return DAG.getNode(ISD::BITCAST, dl, VT, - DAG.getVectorShuffle(NewVT, dl, V1, V2, WidenedMask)); - } - } + MVT OrigVT = V.getSimpleValueType(); + int OrigNumElements = OrigVT.getVectorNumElements(); + int OrigSplitNumElements = OrigNumElements / 2; + MVT OrigScalarVT = OrigVT.getScalarType(); + MVT OrigSplitVT = MVT::getVectorVT(OrigScalarVT, OrigNumElements / 2); - int NumV1Elements = 0, NumUndefElements = 0, NumV2Elements = 0; - for (int M : SVOp->getMask()) - if (M < 0) - ++NumUndefElements; - else if (M < NumElements) - ++NumV1Elements; - else - ++NumV2Elements; + SDValue LoV, HiV; - // Commute the shuffle as needed such that more elements come from V1 than - // V2. This allows us to match the shuffle pattern strictly on how many - // elements come from V1 without handling the symmetric cases. - if (NumV2Elements > NumV1Elements) - return DAG.getCommutedVectorShuffle(*SVOp); + auto *BV = dyn_cast(V); + if (!BV) { + LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigSplitVT, V, + DAG.getIntPtrConstant(0)); + HiV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigSplitVT, V, + DAG.getIntPtrConstant(OrigSplitNumElements)); + } else { - // When the number of V1 and V2 elements are the same, try to minimize the - // number of uses of V2 in the low half of the vector. When that is tied, - // ensure that the sum of indices for V1 is equal to or lower than the sum - // indices for V2. When those are equal, try to ensure that the number of odd - // indices for V1 is lower than the number of odd indices for V2. - if (NumV1Elements == NumV2Elements) { - int LowV1Elements = 0, LowV2Elements = 0; - for (int M : SVOp->getMask().slice(0, NumElements / 2)) - if (M >= NumElements) - ++LowV2Elements; - else if (M >= 0) - ++LowV1Elements; - if (LowV2Elements > LowV1Elements) { - return DAG.getCommutedVectorShuffle(*SVOp); - } else if (LowV2Elements == LowV1Elements) { - int SumV1Indices = 0, SumV2Indices = 0; - for (int i = 0, Size = SVOp->getMask().size(); i < Size; ++i) - if (SVOp->getMask()[i] >= NumElements) - SumV2Indices += i; - else if (SVOp->getMask()[i] >= 0) - SumV1Indices += i; - if (SumV2Indices < SumV1Indices) { - return DAG.getCommutedVectorShuffle(*SVOp); - } else if (SumV2Indices == SumV1Indices) { - int NumV1OddIndices = 0, NumV2OddIndices = 0; - for (int i = 0, Size = SVOp->getMask().size(); i < Size; ++i) - if (SVOp->getMask()[i] >= NumElements) - NumV2OddIndices += i % 2; - else if (SVOp->getMask()[i] >= 0) - NumV1OddIndices += i % 2; - if (NumV2OddIndices < NumV1OddIndices) - return DAG.getCommutedVectorShuffle(*SVOp); + SmallVector LoOps, HiOps; + for (int i = 0; i < OrigSplitNumElements; ++i) { + LoOps.push_back(BV->getOperand(i)); + HiOps.push_back(BV->getOperand(i + OrigSplitNumElements)); } + LoV = DAG.getNode(ISD::BUILD_VECTOR, DL, OrigSplitVT, LoOps); + HiV = DAG.getNode(ISD::BUILD_VECTOR, DL, OrigSplitVT, HiOps); } - } - - // For each vector width, delegate to a specialized lowering routine. - if (VT.getSizeInBits() == 128) - return lower128BitVectorShuffle(Op, V1, V2, VT, Subtarget, DAG); + return std::make_pair(DAG.getNode(ISD::BITCAST, DL, SplitVT, LoV), + DAG.getNode(ISD::BITCAST, DL, SplitVT, HiV)); + }; - if (VT.getSizeInBits() == 256) - return lower256BitVectorShuffle(Op, V1, V2, VT, Subtarget, DAG); + SDValue LoV1, HiV1, LoV2, HiV2; + std::tie(LoV1, HiV1) = SplitVector(V1); + std::tie(LoV2, HiV2) = SplitVector(V2); - // Force AVX-512 vectors to be scalarized for now. - // FIXME: Implement AVX-512 support! - if (VT.getSizeInBits() == 512) - return lower512BitVectorShuffle(Op, V1, V2, VT, Subtarget, DAG); + // Now create two 4-way blends of these half-width vectors. + auto HalfBlend = [&](ArrayRef HalfMask) { + bool UseLoV1 = false, UseHiV1 = false, UseLoV2 = false, UseHiV2 = false; + SmallVector V1BlendMask, V2BlendMask, BlendMask; + for (int i = 0; i < SplitNumElements; ++i) { + int M = HalfMask[i]; + if (M >= NumElements) { + if (M >= NumElements + SplitNumElements) + UseHiV2 = true; + else + UseLoV2 = true; + V2BlendMask.push_back(M - NumElements); + V1BlendMask.push_back(-1); + BlendMask.push_back(SplitNumElements + i); + } else if (M >= 0) { + if (M >= SplitNumElements) + UseHiV1 = true; + else + UseLoV1 = true; + V2BlendMask.push_back(-1); + V1BlendMask.push_back(M); + BlendMask.push_back(i); + } else { + V2BlendMask.push_back(-1); + V1BlendMask.push_back(-1); + BlendMask.push_back(-1); + } + } - llvm_unreachable("Unimplemented!"); -} + // Because the lowering happens after all combining takes place, we need to + // manually combine these blend masks as much as possible so that we create + // a minimal number of high-level vector shuffle nodes. + // First try just blending the halves of V1 or V2. + if (!UseLoV1 && !UseHiV1 && !UseLoV2 && !UseHiV2) + return DAG.getUNDEF(SplitVT); + if (!UseLoV2 && !UseHiV2) + return DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask); + if (!UseLoV1 && !UseHiV1) + return DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask); -//===----------------------------------------------------------------------===// -// Legacy vector shuffle lowering -// -// This code is the legacy code handling vector shuffles until the above -// replaces its functionality and performance. -//===----------------------------------------------------------------------===// + SDValue V1Blend, V2Blend; + if (UseLoV1 && UseHiV1) { + V1Blend = + DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask); + } else { + // We only use half of V1 so map the usage down into the final blend mask. + V1Blend = UseLoV1 ? LoV1 : HiV1; + for (int i = 0; i < SplitNumElements; ++i) + if (BlendMask[i] >= 0 && BlendMask[i] < SplitNumElements) + BlendMask[i] = V1BlendMask[i] - (UseLoV1 ? 0 : SplitNumElements); + } + if (UseLoV2 && UseHiV2) { + V2Blend = + DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask); + } else { + // We only use half of V2 so map the usage down into the final blend mask. + V2Blend = UseLoV2 ? LoV2 : HiV2; + for (int i = 0; i < SplitNumElements; ++i) + if (BlendMask[i] >= SplitNumElements) + BlendMask[i] = V2BlendMask[i] + (UseLoV2 ? SplitNumElements : 0); + } + return DAG.getVectorShuffle(SplitVT, DL, V1Blend, V2Blend, BlendMask); + }; + SDValue Lo = HalfBlend(LoMask); + SDValue Hi = HalfBlend(HiMask); + return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); +} -static bool isBlendMask(ArrayRef MaskVals, MVT VT, bool hasSSE41, - bool hasInt256, unsigned *MaskOut = nullptr) { - MVT EltVT = VT.getVectorElementType(); +/// \brief Either split a vector in halves or decompose the shuffles and the +/// blend. +/// +/// This is provided as a good fallback for many lowerings of non-single-input +/// shuffles with more than one 128-bit lane. In those cases, we want to select +/// between splitting the shuffle into 128-bit components and stitching those +/// back together vs. extracting the single-input shuffles and blending those +/// results. +static SDValue lowerVectorShuffleAsSplitOrBlend(SDLoc DL, MVT VT, SDValue V1, + SDValue V2, ArrayRef Mask, + SelectionDAG &DAG) { + assert(!isSingleInputShuffleMask(Mask) && "This routine must not be used to " + "lower single-input shuffles as it " + "could then recurse on itself."); + int Size = Mask.size(); - // There is no blend with immediate in AVX-512. - if (VT.is512BitVector()) - return false; + // If this can be modeled as a broadcast of two elements followed by a blend, + // prefer that lowering. This is especially important because broadcasts can + // often fold with memory operands. + auto DoBothBroadcast = [&] { + int V1BroadcastIdx = -1, V2BroadcastIdx = -1; + for (int M : Mask) + if (M >= Size) { + if (V2BroadcastIdx == -1) + V2BroadcastIdx = M - Size; + else if (M - Size != V2BroadcastIdx) + return false; + } else if (M >= 0) { + if (V1BroadcastIdx == -1) + V1BroadcastIdx = M; + else if (M != V1BroadcastIdx) + return false; + } + return true; + }; + if (DoBothBroadcast()) + return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask, + DAG); - if (!hasSSE41 || EltVT == MVT::i8) - return false; - if (!hasInt256 && VT == MVT::v16i16) - return false; + // If the inputs all stem from a single 128-bit lane of each input, then we + // split them rather than blending because the split will decompose to + // unusually few instructions. + int LaneCount = VT.getSizeInBits() / 128; + int LaneSize = Size / LaneCount; + SmallBitVector LaneInputs[2]; + LaneInputs[0].resize(LaneCount, false); + LaneInputs[1].resize(LaneCount, false); + for (int i = 0; i < Size; ++i) + if (Mask[i] >= 0) + LaneInputs[Mask[i] / Size][(Mask[i] % Size) / LaneSize] = true; + if (LaneInputs[0].count() <= 1 && LaneInputs[1].count() <= 1) + return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG); - unsigned MaskValue = 0; - unsigned NumElems = VT.getVectorNumElements(); - // There are 2 lanes if (NumElems > 8), and 1 lane otherwise. - unsigned NumLanes = (NumElems - 1) / 8 + 1; - unsigned NumElemsInLane = NumElems / NumLanes; + // Otherwise, just fall back to decomposed shuffles and a blend. This requires + // that the decomposed single-input shuffles don't end up here. + return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask, DAG); +} - // Blend for v16i16 should be symmetric for both lanes. - for (unsigned i = 0; i < NumElemsInLane; ++i) { +/// \brief Lower a vector shuffle crossing multiple 128-bit lanes as +/// a permutation and blend of those lanes. +/// +/// This essentially blends the out-of-lane inputs to each lane into the lane +/// from a permuted copy of the vector. This lowering strategy results in four +/// instructions in the worst case for a single-input cross lane shuffle which +/// is lower than any other fully general cross-lane shuffle strategy I'm aware +/// of. Special cases for each particular shuffle pattern should be handled +/// prior to trying this lowering. +static SDValue lowerVectorShuffleAsLanePermuteAndBlend(SDLoc DL, MVT VT, + SDValue V1, SDValue V2, + ArrayRef Mask, + SelectionDAG &DAG) { + // FIXME: This should probably be generalized for 512-bit vectors as well. + assert(VT.getSizeInBits() == 256 && "Only for 256-bit vector shuffles!"); + int LaneSize = Mask.size() / 2; - int SndLaneEltIdx = (NumLanes == 2) ? MaskVals[i + NumElemsInLane] : -1; - int EltIdx = MaskVals[i]; + // If there are only inputs from one 128-bit lane, splitting will in fact be + // less expensive. The flags track wether the given lane contains an element + // that crosses to another lane. + bool LaneCrossing[2] = {false, false}; + for (int i = 0, Size = Mask.size(); i < Size; ++i) + if (Mask[i] >= 0 && (Mask[i] % Size) / LaneSize != i / LaneSize) + LaneCrossing[(Mask[i] % Size) / LaneSize] = true; + if (!LaneCrossing[0] || !LaneCrossing[1]) + return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG); - if ((EltIdx < 0 || EltIdx == (int)i) && - (SndLaneEltIdx < 0 || SndLaneEltIdx == (int)(i + NumElemsInLane))) - continue; + if (isSingleInputShuffleMask(Mask)) { + SmallVector FlippedBlendMask; + for (int i = 0, Size = Mask.size(); i < Size; ++i) + FlippedBlendMask.push_back( + Mask[i] < 0 ? -1 : (((Mask[i] % Size) / LaneSize == i / LaneSize) + ? Mask[i] + : Mask[i] % LaneSize + + (i / LaneSize) * LaneSize + Size)); - if (((unsigned)EltIdx == (i + NumElems)) && - (SndLaneEltIdx < 0 || - (unsigned)SndLaneEltIdx == i + NumElems + NumElemsInLane)) - MaskValue |= (1 << i); - else - return false; + // Flip the vector, and blend the results which should now be in-lane. The + // VPERM2X128 mask uses the low 2 bits for the low source and bits 4 and + // 5 for the high source. The value 3 selects the high half of source 2 and + // the value 2 selects the low half of source 2. We only use source 2 to + // allow folding it into a memory operand. + unsigned PERMMask = 3 | 2 << 4; + SDValue Flipped = DAG.getNode(X86ISD::VPERM2X128, DL, VT, DAG.getUNDEF(VT), + V1, DAG.getConstant(PERMMask, MVT::i8)); + return DAG.getVectorShuffle(VT, DL, V1, Flipped, FlippedBlendMask); } - if (MaskOut) - *MaskOut = MaskValue; - return true; + // This now reduces to two single-input shuffles of V1 and V2 which at worst + // will be handled by the above logic and a blend of the results, much like + // other patterns in AVX. + return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask, DAG); } -// Try to lower a shuffle node into a simple blend instruction. -// This function assumes isBlendMask returns true for this -// SuffleVectorSDNode -static SDValue LowerVECTOR_SHUFFLEtoBlend(ShuffleVectorSDNode *SVOp, - unsigned MaskValue, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - MVT VT = SVOp->getSimpleValueType(0); - MVT EltVT = VT.getVectorElementType(); - assert(isBlendMask(SVOp->getMask(), VT, Subtarget->hasSSE41(), - Subtarget->hasInt256() && "Trying to lower a " - "VECTOR_SHUFFLE to a Blend but " - "with the wrong mask")); - SDValue V1 = SVOp->getOperand(0); - SDValue V2 = SVOp->getOperand(1); - SDLoc dl(SVOp); - unsigned NumElems = VT.getVectorNumElements(); +/// \brief Handle lowering 2-lane 128-bit shuffles. +static SDValue lowerV2X128VectorShuffle(SDLoc DL, MVT VT, SDValue V1, + SDValue V2, ArrayRef Mask, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + // Blends are faster and handle all the non-lane-crossing cases. + if (SDValue Blend = lowerVectorShuffleAsBlend(DL, VT, V1, V2, Mask, + Subtarget, DAG)) + return Blend; - // Convert i32 vectors to floating point if it is not AVX2. - // AVX2 introduced VPBLENDD instruction for 128 and 256-bit vectors. - MVT BlendVT = VT; - if (EltVT == MVT::i64 || (EltVT == MVT::i32 && !Subtarget->hasInt256())) { - BlendVT = MVT::getVectorVT(MVT::getFloatingPointVT(EltVT.getSizeInBits()), - NumElems); - V1 = DAG.getNode(ISD::BITCAST, dl, VT, V1); - V2 = DAG.getNode(ISD::BITCAST, dl, VT, V2); + MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), + VT.getVectorNumElements() / 2); + // Check for patterns which can be matched with a single insert of a 128-bit + // subvector. + if (isShuffleEquivalent(V1, V2, Mask, 0, 1, 0, 1) || + isShuffleEquivalent(V1, V2, Mask, 0, 1, 4, 5)) { + SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1, + DAG.getIntPtrConstant(0)); + SDValue HiV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, + Mask[2] < 4 ? V1 : V2, DAG.getIntPtrConstant(0)); + return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LoV, HiV); + } + if (isShuffleEquivalent(V1, V2, Mask, 0, 1, 6, 7)) { + SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1, + DAG.getIntPtrConstant(0)); + SDValue HiV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V2, + DAG.getIntPtrConstant(2)); + return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LoV, HiV); } - SDValue Ret = DAG.getNode(X86ISD::BLENDI, dl, BlendVT, V1, V2, - DAG.getConstant(MaskValue, MVT::i32)); - return DAG.getNode(ISD::BITCAST, dl, VT, Ret); + // Otherwise form a 128-bit permutation. + // FIXME: Detect zero-vector inputs and use the VPERM2X128 to zero that half. + unsigned PermMask = Mask[0] / 2 | (Mask[2] / 2) << 4; + return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2, + DAG.getConstant(PermMask, MVT::i8)); } -/// In vector type \p VT, return true if the element at index \p InputIdx -/// falls on a different 128-bit lane than \p OutputIdx. -static bool ShuffleCrosses128bitLane(MVT VT, unsigned InputIdx, - unsigned OutputIdx) { - unsigned EltSize = VT.getVectorElementType().getSizeInBits(); - return InputIdx * EltSize / 128 != OutputIdx * EltSize / 128; -} +/// \brief Lower a vector shuffle by first fixing the 128-bit lanes and then +/// shuffling each lane. +/// +/// This will only succeed when the result of fixing the 128-bit lanes results +/// in a single-input non-lane-crossing shuffle with a repeating shuffle mask in +/// each 128-bit lanes. This handles many cases where we can quickly blend away +/// the lane crosses early and then use simpler shuffles within each lane. +/// +/// FIXME: It might be worthwhile at some point to support this without +/// requiring the 128-bit lane-relative shuffles to be repeating, but currently +/// in x86 only floating point has interesting non-repeating shuffles, and even +/// those are still *marginally* more expensive. +static SDValue lowerVectorShuffleByMerging128BitLanes( + SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef Mask, + const X86Subtarget *Subtarget, SelectionDAG &DAG) { + assert(!isSingleInputShuffleMask(Mask) && + "This is only useful with multiple inputs."); -/// Generate a PSHUFB if possible. Selects elements from \p V1 according to -/// \p MaskVals. MaskVals[OutputIdx] = InputIdx specifies that we want to -/// shuffle the element at InputIdx in V1 to OutputIdx in the result. If \p -/// MaskVals refers to elements outside of \p V1 or is undef (-1), insert a -/// zero. -static SDValue getPSHUFB(ArrayRef MaskVals, SDValue V1, SDLoc &dl, - SelectionDAG &DAG) { - MVT VT = V1.getSimpleValueType(); - assert(VT.is128BitVector() || VT.is256BitVector()); + int Size = Mask.size(); + int LaneSize = 128 / VT.getScalarSizeInBits(); + int NumLanes = Size / LaneSize; + assert(NumLanes > 1 && "Only handles 256-bit and wider shuffles."); - MVT EltVT = VT.getVectorElementType(); - unsigned EltSizeInBytes = EltVT.getSizeInBits() / 8; - unsigned NumElts = VT.getVectorNumElements(); + // See if we can build a hypothetical 128-bit lane-fixing shuffle mask. Also + // check whether the in-128-bit lane shuffles share a repeating pattern. + SmallVector Lanes; + Lanes.resize(NumLanes, -1); + SmallVector InLaneMask; + InLaneMask.resize(LaneSize, -1); + for (int i = 0; i < Size; ++i) { + if (Mask[i] < 0) + continue; - SmallVector PshufbMask; - for (unsigned OutputIdx = 0; OutputIdx < NumElts; ++OutputIdx) { - int InputIdx = MaskVals[OutputIdx]; - unsigned InputByteIdx; + int j = i / LaneSize; - if (InputIdx < 0 || NumElts <= (unsigned)InputIdx) - InputByteIdx = 0x80; - else { - // Cross lane is not allowed. - if (ShuffleCrosses128bitLane(VT, InputIdx, OutputIdx)) - return SDValue(); - InputByteIdx = InputIdx * EltSizeInBytes; - // Index is an byte offset within the 128-bit lane. - InputByteIdx &= 0xf; + if (Lanes[j] < 0) { + // First entry we've seen for this lane. + Lanes[j] = Mask[i] / LaneSize; + } else if (Lanes[j] != Mask[i] / LaneSize) { + // This doesn't match the lane selected previously! + return SDValue(); } - for (unsigned j = 0; j < EltSizeInBytes; ++j) { - PshufbMask.push_back(DAG.getConstant(InputByteIdx, MVT::i8)); - if (InputByteIdx != 0x80) - ++InputByteIdx; + // Check that within each lane we have a consistent shuffle mask. + int k = i % LaneSize; + if (InLaneMask[k] < 0) { + InLaneMask[k] = Mask[i] % LaneSize; + } else if (InLaneMask[k] != Mask[i] % LaneSize) { + // This doesn't fit a repeating in-lane mask. + return SDValue(); } } - MVT ShufVT = MVT::getVectorVT(MVT::i8, PshufbMask.size()); - if (ShufVT != VT) - V1 = DAG.getNode(ISD::BITCAST, dl, ShufVT, V1); - return DAG.getNode(X86ISD::PSHUFB, dl, ShufVT, V1, - DAG.getNode(ISD::BUILD_VECTOR, dl, ShufVT, PshufbMask)); + // First shuffle the lanes into place. + MVT LaneVT = MVT::getVectorVT(VT.isFloatingPoint() ? MVT::f64 : MVT::i64, + VT.getSizeInBits() / 64); + SmallVector LaneMask; + LaneMask.resize(NumLanes * 2, -1); + for (int i = 0; i < NumLanes; ++i) + if (Lanes[i] >= 0) { + LaneMask[2 * i + 0] = 2*Lanes[i] + 0; + LaneMask[2 * i + 1] = 2*Lanes[i] + 1; + } + + V1 = DAG.getNode(ISD::BITCAST, DL, LaneVT, V1); + V2 = DAG.getNode(ISD::BITCAST, DL, LaneVT, V2); + SDValue LaneShuffle = DAG.getVectorShuffle(LaneVT, DL, V1, V2, LaneMask); + + // Cast it back to the type we actually want. + LaneShuffle = DAG.getNode(ISD::BITCAST, DL, VT, LaneShuffle); + + // Now do a simple shuffle that isn't lane crossing. + SmallVector NewMask; + NewMask.resize(Size, -1); + for (int i = 0; i < Size; ++i) + if (Mask[i] >= 0) + NewMask[i] = (i / LaneSize) * LaneSize + Mask[i] % LaneSize; + assert(!is128BitLaneCrossingShuffleMask(VT, NewMask) && + "Must not introduce lane crosses at this point!"); + + return DAG.getVectorShuffle(VT, DL, LaneShuffle, DAG.getUNDEF(VT), NewMask); } -// v8i16 shuffles - Prefer shuffles in the following order: -// 1. [all] pshuflw, pshufhw, optional move -// 2. [ssse3] 1 x pshufb -// 3. [ssse3] 2 x pshufb + 1 x por -// 4. [all] mov + pshuflw + pshufhw + N x (pextrw + pinsrw) -static SDValue -LowerVECTOR_SHUFFLEv8i16(SDValue Op, const X86Subtarget *Subtarget, - SelectionDAG &DAG) { +/// \brief Test whether the specified input (0 or 1) is in-place blended by the +/// given mask. +/// +/// This returns true if the elements from a particular input are already in the +/// slot required by the given mask and require no permutation. +static bool isShuffleMaskInputInPlace(int Input, ArrayRef Mask) { + assert((Input == 0 || Input == 1) && "Only two inputs to shuffles."); + int Size = Mask.size(); + for (int i = 0; i < Size; ++i) + if (Mask[i] >= 0 && Mask[i] / Size == Input && Mask[i] % Size != i) + return false; + + return true; +} + +/// \brief Handle lowering of 4-lane 64-bit floating point shuffles. +/// +/// Also ends up handling lowering of 4-lane 64-bit integer shuffles when AVX2 +/// isn't available. +static SDValue lowerV4F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); + assert(V1.getSimpleValueType() == MVT::v4f64 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v4f64 && "Bad operand type!"); ShuffleVectorSDNode *SVOp = cast(Op); - SDValue V1 = SVOp->getOperand(0); - SDValue V2 = SVOp->getOperand(1); - SDLoc dl(SVOp); - SmallVector MaskVals; + ArrayRef Mask = SVOp->getMask(); + assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"); - // Determine if more than 1 of the words in each of the low and high quadwords - // of the result come from the same quadword of one of the two inputs. Undef - // mask values count as coming from any quadword, for better codegen. - // - // Lo/HiQuad[i] = j indicates how many words from the ith quad of the input - // feeds this quad. For i, 0 and 1 refer to V1, 2 and 3 refer to V2. - unsigned LoQuad[] = { 0, 0, 0, 0 }; - unsigned HiQuad[] = { 0, 0, 0, 0 }; - // Indices of quads used. - std::bitset<4> InputQuads; - for (unsigned i = 0; i < 8; ++i) { - unsigned *Quad = i < 4 ? LoQuad : HiQuad; - int EltIdx = SVOp->getMaskElt(i); - MaskVals.push_back(EltIdx); - if (EltIdx < 0) { - ++Quad[0]; - ++Quad[1]; - ++Quad[2]; - ++Quad[3]; - continue; - } - ++Quad[EltIdx / 4]; - InputQuads.set(EltIdx / 4); - } + SmallVector WidenedMask; + if (canWidenShuffleElements(Mask, WidenedMask)) + return lowerV2X128VectorShuffle(DL, MVT::v4f64, V1, V2, Mask, Subtarget, + DAG); + + if (isSingleInputShuffleMask(Mask)) { + // Check for being able to broadcast a single element. + if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v4f64, DL, V1, + Mask, Subtarget, DAG)) + return Broadcast; - int BestLoQuad = -1; - unsigned MaxQuad = 1; - for (unsigned i = 0; i < 4; ++i) { - if (LoQuad[i] > MaxQuad) { - BestLoQuad = i; - MaxQuad = LoQuad[i]; + // Use low duplicate instructions for masks that match their pattern. + if (isShuffleEquivalent(V1, V2, Mask, 0, 0, 2, 2)) + return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1); + + if (!is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask)) { + // Non-half-crossing single input shuffles can be lowerid with an + // interleaved permutation. + unsigned VPERMILPMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1) | + ((Mask[2] == 3) << 2) | ((Mask[3] == 3) << 3); + return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1, + DAG.getConstant(VPERMILPMask, MVT::i8)); } - } - int BestHiQuad = -1; - MaxQuad = 1; - for (unsigned i = 0; i < 4; ++i) { - if (HiQuad[i] > MaxQuad) { - BestHiQuad = i; - MaxQuad = HiQuad[i]; - } + // With AVX2 we have direct support for this permutation. + if (Subtarget->hasAVX2()) + return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1, + getV4X86ShuffleImm8ForMask(Mask, DAG)); + + // Otherwise, fall back. + return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v4f64, V1, V2, Mask, + DAG); } - // For SSSE3, If all 8 words of the result come from only 1 quadword of each - // of the two input vectors, shuffle them into one input vector so only a - // single pshufb instruction is necessary. If there are more than 2 input - // quads, disable the next transformation since it does not help SSSE3. - bool V1Used = InputQuads[0] || InputQuads[1]; - bool V2Used = InputQuads[2] || InputQuads[3]; - if (Subtarget->hasSSSE3()) { - if (InputQuads.count() == 2 && V1Used && V2Used) { - BestLoQuad = InputQuads[0] ? 0 : 1; - BestHiQuad = InputQuads[2] ? 2 : 3; - } - if (InputQuads.count() > 2) { - BestLoQuad = -1; - BestHiQuad = -1; - } - } + // X86 has dedicated unpack instructions that can handle specific blend + // operations: UNPCKH and UNPCKL. + if (isShuffleEquivalent(V1, V2, Mask, 0, 4, 2, 6)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f64, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, 1, 5, 3, 7)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f64, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, 4, 0, 6, 2)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f64, V2, V1); + if (isShuffleEquivalent(V1, V2, Mask, 5, 1, 7, 3)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f64, V2, V1); - // If BestLoQuad or BestHiQuad are set, shuffle the quads together and update - // the shuffle mask. If a quad is scored as -1, that means that it contains - // words from all 4 input quadwords. - SDValue NewV; - if (BestLoQuad >= 0 || BestHiQuad >= 0) { - int MaskV[] = { - BestLoQuad < 0 ? 0 : BestLoQuad, - BestHiQuad < 0 ? 1 : BestHiQuad - }; - NewV = DAG.getVectorShuffle(MVT::v2i64, dl, - DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1), - DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V2), &MaskV[0]); - NewV = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, NewV); - - // Rewrite the MaskVals and assign NewV to V1 if NewV now contains all the - // source words for the shuffle, to aid later transformations. - bool AllWordsInNewV = true; - bool InOrder[2] = { true, true }; - for (unsigned i = 0; i != 8; ++i) { - int idx = MaskVals[i]; - if (idx != (int)i) - InOrder[i/4] = false; - if (idx < 0 || (idx/4) == BestLoQuad || (idx/4) == BestHiQuad) - continue; - AllWordsInNewV = false; - break; - } + // If we have a single input to the zero element, insert that into V1 if we + // can do so cheaply. + int NumV2Elements = + std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; }); + if (NumV2Elements == 1 && Mask[0] >= 4) + if (SDValue Insertion = lowerVectorShuffleAsElementInsertion( + MVT::v4f64, DL, V1, V2, Mask, Subtarget, DAG)) + return Insertion; - bool pshuflw = AllWordsInNewV, pshufhw = AllWordsInNewV; - if (AllWordsInNewV) { - for (int i = 0; i != 8; ++i) { - int idx = MaskVals[i]; - if (idx < 0) - continue; - idx = MaskVals[i] = (idx / 4) == BestLoQuad ? (idx & 3) : (idx & 3) + 4; - if ((idx != i) && idx < 4) - pshufhw = false; - if ((idx != i) && idx > 3) - pshuflw = false; - } - V1 = NewV; - V2Used = false; - BestLoQuad = 0; - BestHiQuad = 1; - } - - // If we've eliminated the use of V2, and the new mask is a pshuflw or - // pshufhw, that's as cheap as it gets. Return the new shuffle. - if ((pshufhw && InOrder[0]) || (pshuflw && InOrder[1])) { - unsigned Opc = pshufhw ? X86ISD::PSHUFHW : X86ISD::PSHUFLW; - unsigned TargetMask = 0; - NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, - DAG.getUNDEF(MVT::v8i16), &MaskVals[0]); - ShuffleVectorSDNode *SVOp = cast(NewV.getNode()); - TargetMask = pshufhw ? getShufflePSHUFHWImmediate(SVOp): - getShufflePSHUFLWImmediate(SVOp); - V1 = NewV.getOperand(0); - return getTargetShuffleNode(Opc, dl, MVT::v8i16, V1, TargetMask, DAG); - } - } - - // Promote splats to a larger type which usually leads to more efficient code. - // FIXME: Is this true if pshufb is available? - if (SVOp->isSplat()) - return PromoteSplat(SVOp, DAG); - - // If we have SSSE3, and all words of the result are from 1 input vector, - // case 2 is generated, otherwise case 3 is generated. If no SSSE3 - // is present, fall back to case 4. - if (Subtarget->hasSSSE3()) { - SmallVector pshufbMask; - - // If we have elements from both input vectors, set the high bit of the - // shuffle mask element to zero out elements that come from V2 in the V1 - // mask, and elements that come from V1 in the V2 mask, so that the two - // results can be OR'd together. - bool TwoInputs = V1Used && V2Used; - V1 = getPSHUFB(MaskVals, V1, dl, DAG); - if (!TwoInputs) - return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); - - // Calculate the shuffle mask for the second input, shuffle it, and - // OR it with the first shuffled input. - CommuteVectorShuffleMask(MaskVals, 8); - V2 = getPSHUFB(MaskVals, V2, dl, DAG); - V1 = DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2); - return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); - } - - // If BestLoQuad >= 0, generate a pshuflw to put the low elements in order, - // and update MaskVals with new element order. - std::bitset<8> InOrder; - if (BestLoQuad >= 0) { - int MaskV[] = { -1, -1, -1, -1, 4, 5, 6, 7 }; - for (int i = 0; i != 4; ++i) { - int idx = MaskVals[i]; - if (idx < 0) { - InOrder.set(i); - } else if ((idx / 4) == BestLoQuad) { - MaskV[i] = idx & 3; - InOrder.set(i); - } - } - NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16), - &MaskV[0]); + if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask, + Subtarget, DAG)) + return Blend; - if (NewV.getOpcode() == ISD::VECTOR_SHUFFLE && Subtarget->hasSSE2()) { - ShuffleVectorSDNode *SVOp = cast(NewV.getNode()); - NewV = getTargetShuffleNode(X86ISD::PSHUFLW, dl, MVT::v8i16, - NewV.getOperand(0), - getShufflePSHUFLWImmediate(SVOp), DAG); - } + // Check if the blend happens to exactly fit that of SHUFPD. + if ((Mask[0] == -1 || Mask[0] < 2) && + (Mask[1] == -1 || (Mask[1] >= 4 && Mask[1] < 6)) && + (Mask[2] == -1 || (Mask[2] >= 2 && Mask[2] < 4)) && + (Mask[3] == -1 || Mask[3] >= 6)) { + unsigned SHUFPDMask = (Mask[0] == 1) | ((Mask[1] == 5) << 1) | + ((Mask[2] == 3) << 2) | ((Mask[3] == 7) << 3); + return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f64, V1, V2, + DAG.getConstant(SHUFPDMask, MVT::i8)); } - - // If BestHi >= 0, generate a pshufhw to put the high elements in order, - // and update MaskVals with the new element order. - if (BestHiQuad >= 0) { - int MaskV[] = { 0, 1, 2, 3, -1, -1, -1, -1 }; - for (unsigned i = 4; i != 8; ++i) { - int idx = MaskVals[i]; - if (idx < 0) { - InOrder.set(i); - } else if ((idx / 4) == BestHiQuad) { - MaskV[i] = (idx & 3) + 4; - InOrder.set(i); - } - } - NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16), - &MaskV[0]); - - if (NewV.getOpcode() == ISD::VECTOR_SHUFFLE && Subtarget->hasSSE2()) { - ShuffleVectorSDNode *SVOp = cast(NewV.getNode()); - NewV = getTargetShuffleNode(X86ISD::PSHUFHW, dl, MVT::v8i16, - NewV.getOperand(0), - getShufflePSHUFHWImmediate(SVOp), DAG); - } + if ((Mask[0] == -1 || (Mask[0] >= 4 && Mask[0] < 6)) && + (Mask[1] == -1 || Mask[1] < 2) && + (Mask[2] == -1 || Mask[2] >= 6) && + (Mask[3] == -1 || (Mask[3] >= 2 && Mask[3] < 4))) { + unsigned SHUFPDMask = (Mask[0] == 5) | ((Mask[1] == 1) << 1) | + ((Mask[2] == 7) << 2) | ((Mask[3] == 3) << 3); + return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f64, V2, V1, + DAG.getConstant(SHUFPDMask, MVT::i8)); } - // In case BestHi & BestLo were both -1, which means each quadword has a word - // from each of the four input quadwords, calculate the InOrder bitvector now - // before falling through to the insert/extract cleanup. - if (BestLoQuad == -1 && BestHiQuad == -1) { - NewV = V1; - for (int i = 0; i != 8; ++i) - if (MaskVals[i] < 0 || MaskVals[i] == i) - InOrder.set(i); - } + // Try to simplify this by merging 128-bit lanes to enable a lane-based + // shuffle. However, if we have AVX2 and either inputs are already in place, + // we will be able to shuffle even across lanes the other input in a single + // instruction so skip this pattern. + if (!(Subtarget->hasAVX2() && (isShuffleMaskInputInPlace(0, Mask) || + isShuffleMaskInputInPlace(1, Mask)))) + if (SDValue Result = lowerVectorShuffleByMerging128BitLanes( + DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG)) + return Result; - // The other elements are put in the right place using pextrw and pinsrw. - for (unsigned i = 0; i != 8; ++i) { - if (InOrder[i]) - continue; - int EltIdx = MaskVals[i]; - if (EltIdx < 0) - continue; - SDValue ExtOp = (EltIdx < 8) ? - DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V1, - DAG.getIntPtrConstant(EltIdx)) : - DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V2, - DAG.getIntPtrConstant(EltIdx - 8)); - NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, ExtOp, - DAG.getIntPtrConstant(i)); - } - return NewV; + // If we have AVX2 then we always want to lower with a blend because an v4 we + // can fully permute the elements. + if (Subtarget->hasAVX2()) + return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4f64, V1, V2, + Mask, DAG); + + // Otherwise fall back on generic lowering. + return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask, DAG); } -/// \brief v16i16 shuffles +/// \brief Handle lowering of 4-lane 64-bit integer shuffles. /// -/// FIXME: We only support generation of a single pshufb currently. We can -/// generalize the other applicable cases from LowerVECTOR_SHUFFLEv8i16 as -/// well (e.g 2 x pshufb + 1 x por). -static SDValue -LowerVECTOR_SHUFFLEv16i16(SDValue Op, SelectionDAG &DAG) { +/// This routine is only called when we have AVX2 and thus a reasonable +/// instruction set for v4i64 shuffling.. +static SDValue lowerV4I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); + assert(V1.getSimpleValueType() == MVT::v4i64 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v4i64 && "Bad operand type!"); ShuffleVectorSDNode *SVOp = cast(Op); - SDValue V1 = SVOp->getOperand(0); - SDValue V2 = SVOp->getOperand(1); - SDLoc dl(SVOp); - - if (V2.getOpcode() != ISD::UNDEF) - return SDValue(); - - SmallVector MaskVals(SVOp->getMask().begin(), SVOp->getMask().end()); - return getPSHUFB(MaskVals, V1, dl, DAG); -} + ArrayRef Mask = SVOp->getMask(); + assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"); + assert(Subtarget->hasAVX2() && "We can only lower v4i64 with AVX2!"); -// v16i8 shuffles - Prefer shuffles in the following order: -// 1. [ssse3] 1 x pshufb -// 2. [ssse3] 2 x pshufb + 1 x por -// 3. [all] v8i16 shuffle + N x pextrw + rotate + pinsrw -static SDValue LowerVECTOR_SHUFFLEv16i8(ShuffleVectorSDNode *SVOp, - const X86Subtarget* Subtarget, - SelectionDAG &DAG) { - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - SDValue V1 = SVOp->getOperand(0); - SDValue V2 = SVOp->getOperand(1); - SDLoc dl(SVOp); - ArrayRef MaskVals = SVOp->getMask(); + SmallVector WidenedMask; + if (canWidenShuffleElements(Mask, WidenedMask)) + return lowerV2X128VectorShuffle(DL, MVT::v4i64, V1, V2, Mask, Subtarget, + DAG); - // Promote splats to a larger type which usually leads to more efficient code. - // FIXME: Is this true if pshufb is available? - if (SVOp->isSplat()) - return PromoteSplat(SVOp, DAG); + if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask, + Subtarget, DAG)) + return Blend; - // If we have SSSE3, case 1 is generated when all result bytes come from - // one of the inputs. Otherwise, case 2 is generated. If no SSSE3 is - // present, fall back to case 3. + // Check for being able to broadcast a single element. + if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v4i64, DL, V1, + Mask, Subtarget, DAG)) + return Broadcast; - // If SSSE3, use 1 pshufb instruction per vector with elements in the result. - if (Subtarget->hasSSSE3()) { - SmallVector pshufbMask; + // When the shuffle is mirrored between the 128-bit lanes of the unit, we can + // use lower latency instructions that will operate on both 128-bit lanes. + SmallVector RepeatedMask; + if (is128BitLaneRepeatedShuffleMask(MVT::v4i64, Mask, RepeatedMask)) { + if (isSingleInputShuffleMask(Mask)) { + int PSHUFDMask[] = {-1, -1, -1, -1}; + for (int i = 0; i < 2; ++i) + if (RepeatedMask[i] >= 0) { + PSHUFDMask[2 * i] = 2 * RepeatedMask[i]; + PSHUFDMask[2 * i + 1] = 2 * RepeatedMask[i] + 1; + } + return DAG.getNode( + ISD::BITCAST, DL, MVT::v4i64, + DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, + DAG.getNode(ISD::BITCAST, DL, MVT::v8i32, V1), + getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG))); + } + } - // If all result elements are from one input vector, then only translate - // undef mask values to 0x80 (zero out result) in the pshufb mask. - // - // Otherwise, we have elements from both input vectors, and must zero out - // elements that come from V2 in the first mask, and V1 in the second mask - // so that we can OR them together. - for (unsigned i = 0; i != 16; ++i) { - int EltIdx = MaskVals[i]; - if (EltIdx < 0 || EltIdx >= 16) - EltIdx = 0x80; - pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8)); - } - V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1, - DAG.getNode(ISD::BUILD_VECTOR, dl, - MVT::v16i8, pshufbMask)); - - // As PSHUFB will zero elements with negative indices, it's safe to ignore - // the 2nd operand if it's undefined or zero. - if (V2.getOpcode() == ISD::UNDEF || - ISD::isBuildVectorAllZeros(V2.getNode())) - return V1; + // AVX2 provides a direct instruction for permuting a single input across + // lanes. + if (isSingleInputShuffleMask(Mask)) + return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1, + getV4X86ShuffleImm8ForMask(Mask, DAG)); - // Calculate the shuffle mask for the second input, shuffle it, and - // OR it with the first shuffled input. - pshufbMask.clear(); - for (unsigned i = 0; i != 16; ++i) { - int EltIdx = MaskVals[i]; - EltIdx = (EltIdx < 16) ? 0x80 : EltIdx - 16; - pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8)); - } - V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2, - DAG.getNode(ISD::BUILD_VECTOR, dl, - MVT::v16i8, pshufbMask)); - return DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2); - } - - // No SSSE3 - Calculate in place words and then fix all out of place words - // With 0-16 extracts & inserts. Worst case is 16 bytes out of order from - // the 16 different words that comprise the two doublequadword input vectors. - V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); - V2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V2); - SDValue NewV = V1; - for (int i = 0; i != 8; ++i) { - int Elt0 = MaskVals[i*2]; - int Elt1 = MaskVals[i*2+1]; - - // This word of the result is all undef, skip it. - if (Elt0 < 0 && Elt1 < 0) - continue; + // Try to use shift instructions. + if (SDValue Shift = + lowerVectorShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask, DAG)) + return Shift; - // This word of the result is already in the correct place, skip it. - if ((Elt0 == i*2) && (Elt1 == i*2+1)) - continue; + // Use dedicated unpack instructions for masks that match their pattern. + if (isShuffleEquivalent(V1, V2, Mask, 0, 4, 2, 6)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i64, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, 1, 5, 3, 7)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i64, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, 4, 0, 6, 2)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i64, V2, V1); + if (isShuffleEquivalent(V1, V2, Mask, 5, 1, 7, 3)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i64, V2, V1); - SDValue Elt0Src = Elt0 < 16 ? V1 : V2; - SDValue Elt1Src = Elt1 < 16 ? V1 : V2; - SDValue InsElt; - - // If Elt0 and Elt1 are defined, are consecutive, and can be load - // using a single extract together, load it and store it. - if ((Elt0 >= 0) && ((Elt0 + 1) == Elt1) && ((Elt0 & 1) == 0)) { - InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src, - DAG.getIntPtrConstant(Elt1 / 2)); - NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt, - DAG.getIntPtrConstant(i)); - continue; - } + // Try to simplify this by merging 128-bit lanes to enable a lane-based + // shuffle. However, if we have AVX2 and either inputs are already in place, + // we will be able to shuffle even across lanes the other input in a single + // instruction so skip this pattern. + if (!(Subtarget->hasAVX2() && (isShuffleMaskInputInPlace(0, Mask) || + isShuffleMaskInputInPlace(1, Mask)))) + if (SDValue Result = lowerVectorShuffleByMerging128BitLanes( + DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG)) + return Result; - // If Elt1 is defined, extract it from the appropriate source. If the - // source byte is not also odd, shift the extracted word left 8 bits - // otherwise clear the bottom 8 bits if we need to do an or. - if (Elt1 >= 0) { - InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src, - DAG.getIntPtrConstant(Elt1 / 2)); - if ((Elt1 & 1) == 0) - InsElt = DAG.getNode(ISD::SHL, dl, MVT::i16, InsElt, - DAG.getConstant(8, - TLI.getShiftAmountTy(InsElt.getValueType()))); - else if (Elt0 >= 0) - InsElt = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt, - DAG.getConstant(0xFF00, MVT::i16)); - } - // If Elt0 is defined, extract it from the appropriate source. If the - // source byte is not also even, shift the extracted word right 8 bits. If - // Elt1 was also defined, OR the extracted values together before - // inserting them in the result. - if (Elt0 >= 0) { - SDValue InsElt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, - Elt0Src, DAG.getIntPtrConstant(Elt0 / 2)); - if ((Elt0 & 1) != 0) - InsElt0 = DAG.getNode(ISD::SRL, dl, MVT::i16, InsElt0, - DAG.getConstant(8, - TLI.getShiftAmountTy(InsElt0.getValueType()))); - else if (Elt1 >= 0) - InsElt0 = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt0, - DAG.getConstant(0x00FF, MVT::i16)); - InsElt = Elt1 >= 0 ? DAG.getNode(ISD::OR, dl, MVT::i16, InsElt, InsElt0) - : InsElt0; - } - NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt, - DAG.getIntPtrConstant(i)); - } - return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, NewV); -} - -// v32i8 shuffles - Translate to VPSHUFB if possible. -static -SDValue LowerVECTOR_SHUFFLEv32i8(ShuffleVectorSDNode *SVOp, - const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - MVT VT = SVOp->getSimpleValueType(0); - SDValue V1 = SVOp->getOperand(0); - SDValue V2 = SVOp->getOperand(1); - SDLoc dl(SVOp); - SmallVector MaskVals(SVOp->getMask().begin(), SVOp->getMask().end()); + // Otherwise fall back on generic blend lowering. + return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i64, V1, V2, + Mask, DAG); +} - bool V2IsUndef = V2.getOpcode() == ISD::UNDEF; - bool V1IsAllZero = ISD::isBuildVectorAllZeros(V1.getNode()); - bool V2IsAllZero = ISD::isBuildVectorAllZeros(V2.getNode()); - - // VPSHUFB may be generated if - // (1) one of input vector is undefined or zeroinitializer. - // The mask value 0x80 puts 0 in the corresponding slot of the vector. - // And (2) the mask indexes don't cross the 128-bit lane. - if (VT != MVT::v32i8 || !Subtarget->hasInt256() || - (!V2IsUndef && !V2IsAllZero && !V1IsAllZero)) - return SDValue(); +/// \brief Handle lowering of 8-lane 32-bit floating point shuffles. +/// +/// Also ends up handling lowering of 8-lane 32-bit integer shuffles when AVX2 +/// isn't available. +static SDValue lowerV8F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); + assert(V1.getSimpleValueType() == MVT::v8f32 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v8f32 && "Bad operand type!"); + ShuffleVectorSDNode *SVOp = cast(Op); + ArrayRef Mask = SVOp->getMask(); + assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"); - if (V1IsAllZero && !V2IsAllZero) { - CommuteVectorShuffleMask(MaskVals, 32); - V1 = V2; - } - return getPSHUFB(MaskVals, V1, dl, DAG); -} + if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask, + Subtarget, DAG)) + return Blend; -/// RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide -/// ones, or rewriting v4i32 / v4f32 as 2 wide ones if possible. This can be -/// done when every pair / quad of shuffle mask elements point to elements in -/// the right sequence. e.g. -/// vector_shuffle X, Y, <2, 3, | 10, 11, | 0, 1, | 14, 15> -static -SDValue RewriteAsNarrowerShuffle(ShuffleVectorSDNode *SVOp, - SelectionDAG &DAG) { - MVT VT = SVOp->getSimpleValueType(0); - SDLoc dl(SVOp); - unsigned NumElems = VT.getVectorNumElements(); - MVT NewVT; - unsigned Scale; - switch (VT.SimpleTy) { - default: llvm_unreachable("Unexpected!"); - case MVT::v2i64: - case MVT::v2f64: - return SDValue(SVOp, 0); - case MVT::v4f32: NewVT = MVT::v2f64; Scale = 2; break; - case MVT::v4i32: NewVT = MVT::v2i64; Scale = 2; break; - case MVT::v8i16: NewVT = MVT::v4i32; Scale = 2; break; - case MVT::v16i8: NewVT = MVT::v4i32; Scale = 4; break; - case MVT::v16i16: NewVT = MVT::v8i32; Scale = 2; break; - case MVT::v32i8: NewVT = MVT::v8i32; Scale = 4; break; - } - - SmallVector MaskVec; - for (unsigned i = 0; i != NumElems; i += Scale) { - int StartIdx = -1; - for (unsigned j = 0; j != Scale; ++j) { - int EltIdx = SVOp->getMaskElt(i+j); - if (EltIdx < 0) - continue; - if (StartIdx < 0) - StartIdx = (EltIdx / Scale); - if (EltIdx != (int)(StartIdx*Scale + j)) - return SDValue(); - } - MaskVec.push_back(StartIdx); - } + // Check for being able to broadcast a single element. + if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v8f32, DL, V1, + Mask, Subtarget, DAG)) + return Broadcast; - SDValue V1 = DAG.getNode(ISD::BITCAST, dl, NewVT, SVOp->getOperand(0)); - SDValue V2 = DAG.getNode(ISD::BITCAST, dl, NewVT, SVOp->getOperand(1)); - return DAG.getVectorShuffle(NewVT, dl, V1, V2, &MaskVec[0]); -} + // If the shuffle mask is repeated in each 128-bit lane, we have many more + // options to efficiently lower the shuffle. + SmallVector RepeatedMask; + if (is128BitLaneRepeatedShuffleMask(MVT::v8f32, Mask, RepeatedMask)) { + assert(RepeatedMask.size() == 4 && + "Repeated masks must be half the mask width!"); -/// getVZextMovL - Return a zero-extending vector move low node. -/// -static SDValue getVZextMovL(MVT VT, MVT OpVT, - SDValue SrcOp, SelectionDAG &DAG, - const X86Subtarget *Subtarget, SDLoc dl) { - if (VT == MVT::v2f64 || VT == MVT::v4f32) { - LoadSDNode *LD = nullptr; - if (!isScalarLoadToVector(SrcOp.getNode(), &LD)) - LD = dyn_cast(SrcOp); - if (!LD) { - // movssrr and movsdrr do not clear top bits. Try to use movd, movq - // instead. - MVT ExtVT = (OpVT == MVT::v2f64) ? MVT::i64 : MVT::i32; - if ((ExtVT != MVT::i64 || Subtarget->is64Bit()) && - SrcOp.getOpcode() == ISD::SCALAR_TO_VECTOR && - SrcOp.getOperand(0).getOpcode() == ISD::BITCAST && - SrcOp.getOperand(0).getOperand(0).getValueType() == ExtVT) { - // PR2108 - OpVT = (OpVT == MVT::v2f64) ? MVT::v2i64 : MVT::v4i32; - return DAG.getNode(ISD::BITCAST, dl, VT, - DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT, - DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, - OpVT, - SrcOp.getOperand(0) - .getOperand(0)))); - } - } - } + // Use even/odd duplicate instructions for masks that match their pattern. + if (isShuffleEquivalent(V1, V2, Mask, 0, 0, 2, 2, 4, 4, 6, 6)) + return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1); + if (isShuffleEquivalent(V1, V2, Mask, 1, 1, 3, 3, 5, 5, 7, 7)) + return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1); - return DAG.getNode(ISD::BITCAST, dl, VT, - DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT, - DAG.getNode(ISD::BITCAST, dl, - OpVT, SrcOp))); -} + if (isSingleInputShuffleMask(Mask)) + return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1, + getV4X86ShuffleImm8ForMask(RepeatedMask, DAG)); -/// LowerVECTOR_SHUFFLE_256 - Handle all 256-bit wide vectors shuffles -/// which could not be matched by any known target speficic shuffle -static SDValue -LowerVECTOR_SHUFFLE_256(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) { + // Use dedicated unpack instructions for masks that match their pattern. + if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 1, 9, 4, 12, 5, 13)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8f32, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, 2, 10, 3, 11, 6, 14, 7, 15)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8f32, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, 8, 0, 9, 1, 12, 4, 13, 5)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8f32, V2, V1); + if (isShuffleEquivalent(V1, V2, Mask, 10, 2, 11, 3, 14, 6, 15, 7)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8f32, V2, V1); - SDValue NewOp = Compact8x32ShuffleNode(SVOp, DAG); - if (NewOp.getNode()) - return NewOp; + // Otherwise, fall back to a SHUFPS sequence. Here it is important that we + // have already handled any direct blends. We also need to squash the + // repeated mask into a simulated v4f32 mask. + for (int i = 0; i < 4; ++i) + if (RepeatedMask[i] >= 8) + RepeatedMask[i] -= 4; + return lowerVectorShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG); + } - MVT VT = SVOp->getSimpleValueType(0); + // If we have a single input shuffle with different shuffle patterns in the + // two 128-bit lanes use the variable mask to VPERMILPS. + if (isSingleInputShuffleMask(Mask)) { + SDValue VPermMask[8]; + for (int i = 0; i < 8; ++i) + VPermMask[i] = Mask[i] < 0 ? DAG.getUNDEF(MVT::i32) + : DAG.getConstant(Mask[i], MVT::i32); + if (!is128BitLaneCrossingShuffleMask(MVT::v8f32, Mask)) + return DAG.getNode( + X86ISD::VPERMILPV, DL, MVT::v8f32, V1, + DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i32, VPermMask)); - unsigned NumElems = VT.getVectorNumElements(); - unsigned NumLaneElems = NumElems / 2; + if (Subtarget->hasAVX2()) + return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32, + DAG.getNode(ISD::BITCAST, DL, MVT::v8f32, + DAG.getNode(ISD::BUILD_VECTOR, DL, + MVT::v8i32, VPermMask)), + V1); - SDLoc dl(SVOp); - MVT EltVT = VT.getVectorElementType(); - MVT NVT = MVT::getVectorVT(EltVT, NumLaneElems); - SDValue Output[2]; + // Otherwise, fall back. + return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v8f32, V1, V2, Mask, + DAG); + } - SmallVector Mask; - for (unsigned l = 0; l < 2; ++l) { - // Build a shuffle mask for the output, discovering on the fly which - // input vectors to use as shuffle operands (recorded in InputUsed). - // If building a suitable shuffle vector proves too hard, then bail - // out with UseBuildVector set. - bool UseBuildVector = false; - int InputUsed[2] = { -1, -1 }; // Not yet discovered. - unsigned LaneStart = l * NumLaneElems; - for (unsigned i = 0; i != NumLaneElems; ++i) { - // The mask element. This indexes into the input. - int Idx = SVOp->getMaskElt(i+LaneStart); - if (Idx < 0) { - // the mask element does not index into any input vector. - Mask.push_back(-1); - continue; - } + // Try to simplify this by merging 128-bit lanes to enable a lane-based + // shuffle. + if (SDValue Result = lowerVectorShuffleByMerging128BitLanes( + DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG)) + return Result; - // The input vector this mask element indexes into. - int Input = Idx / NumLaneElems; + // If we have AVX2 then we always want to lower with a blend because at v8 we + // can fully permute the elements. + if (Subtarget->hasAVX2()) + return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8f32, V1, V2, + Mask, DAG); - // Turn the index into an offset from the start of the input vector. - Idx -= Input * NumLaneElems; + // Otherwise fall back on generic lowering. + return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, DAG); +} - // Find or create a shuffle vector operand to hold this input. - unsigned OpNo; - for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { - if (InputUsed[OpNo] == Input) - // This input vector is already an operand. - break; - if (InputUsed[OpNo] < 0) { - // Create a new operand for this input vector. - InputUsed[OpNo] = Input; - break; - } - } +/// \brief Handle lowering of 8-lane 32-bit integer shuffles. +/// +/// This routine is only called when we have AVX2 and thus a reasonable +/// instruction set for v8i32 shuffling.. +static SDValue lowerV8I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); + assert(V1.getSimpleValueType() == MVT::v8i32 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v8i32 && "Bad operand type!"); + ShuffleVectorSDNode *SVOp = cast(Op); + ArrayRef Mask = SVOp->getMask(); + assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"); + assert(Subtarget->hasAVX2() && "We can only lower v8i32 with AVX2!"); - if (OpNo >= array_lengthof(InputUsed)) { - // More than two input vectors used! Give up on trying to create a - // shuffle vector. Insert all elements into a BUILD_VECTOR instead. - UseBuildVector = true; - break; - } + // Whenever we can lower this as a zext, that instruction is strictly faster + // than any alternative. It also allows us to fold memory operands into the + // shuffle in many cases. + if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, + Mask, Subtarget, DAG)) + return ZExt; - // Add the mask index for the new shuffle vector. - Mask.push_back(Idx + OpNo * NumLaneElems); - } + if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask, + Subtarget, DAG)) + return Blend; - if (UseBuildVector) { - SmallVector SVOps; - for (unsigned i = 0; i != NumLaneElems; ++i) { - // The mask element. This indexes into the input. - int Idx = SVOp->getMaskElt(i+LaneStart); - if (Idx < 0) { - SVOps.push_back(DAG.getUNDEF(EltVT)); - continue; - } + // Check for being able to broadcast a single element. + if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v8i32, DL, V1, + Mask, Subtarget, DAG)) + return Broadcast; - // The input vector this mask element indexes into. - int Input = Idx / NumElems; + // If the shuffle mask is repeated in each 128-bit lane we can use more + // efficient instructions that mirror the shuffles across the two 128-bit + // lanes. + SmallVector RepeatedMask; + if (is128BitLaneRepeatedShuffleMask(MVT::v8i32, Mask, RepeatedMask)) { + assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!"); + if (isSingleInputShuffleMask(Mask)) + return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1, + getV4X86ShuffleImm8ForMask(RepeatedMask, DAG)); - // Turn the index into an offset from the start of the input vector. - Idx -= Input * NumElems; + // Use dedicated unpack instructions for masks that match their pattern. + if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 1, 9, 4, 12, 5, 13)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i32, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, 2, 10, 3, 11, 6, 14, 7, 15)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i32, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, 8, 0, 9, 1, 12, 4, 13, 5)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i32, V2, V1); + if (isShuffleEquivalent(V1, V2, Mask, 10, 2, 11, 3, 14, 6, 15, 7)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i32, V2, V1); + } - // Extract the vector element by hand. - SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, - SVOp->getOperand(Input), - DAG.getIntPtrConstant(Idx))); - } + // Try to use shift instructions. + if (SDValue Shift = + lowerVectorShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask, DAG)) + return Shift; - // Construct the output using a BUILD_VECTOR. - Output[l] = DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, SVOps); - } else if (InputUsed[0] < 0) { - // No input vectors were used! The result is undefined. - Output[l] = DAG.getUNDEF(NVT); - } else { - SDValue Op0 = Extract128BitVector(SVOp->getOperand(InputUsed[0] / 2), - (InputUsed[0] % 2) * NumLaneElems, - DAG, dl); - // If only one input was used, use an undefined vector for the other. - SDValue Op1 = (InputUsed[1] < 0) ? DAG.getUNDEF(NVT) : - Extract128BitVector(SVOp->getOperand(InputUsed[1] / 2), - (InputUsed[1] % 2) * NumLaneElems, DAG, dl); - // At least one input vector was used. Create a new shuffle vector. - Output[l] = DAG.getVectorShuffle(NVT, dl, Op0, Op1, &Mask[0]); - } + if (SDValue Rotate = lowerVectorShuffleAsByteRotate( + DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG)) + return Rotate; - Mask.clear(); + // If the shuffle patterns aren't repeated but it is a single input, directly + // generate a cross-lane VPERMD instruction. + if (isSingleInputShuffleMask(Mask)) { + SDValue VPermMask[8]; + for (int i = 0; i < 8; ++i) + VPermMask[i] = Mask[i] < 0 ? DAG.getUNDEF(MVT::i32) + : DAG.getConstant(Mask[i], MVT::i32); + return DAG.getNode( + X86ISD::VPERMV, DL, MVT::v8i32, + DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i32, VPermMask), V1); } - // Concatenate the result back - return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Output[0], Output[1]); -} + // Try to simplify this by merging 128-bit lanes to enable a lane-based + // shuffle. + if (SDValue Result = lowerVectorShuffleByMerging128BitLanes( + DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG)) + return Result; -/// LowerVECTOR_SHUFFLE_128v4 - Handle all 128-bit wide vectors with -/// 4 elements, and match them with several different shuffle types. -static SDValue -LowerVECTOR_SHUFFLE_128v4(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) { - SDValue V1 = SVOp->getOperand(0); - SDValue V2 = SVOp->getOperand(1); - SDLoc dl(SVOp); - MVT VT = SVOp->getSimpleValueType(0); + // Otherwise fall back on generic blend lowering. + return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i32, V1, V2, + Mask, DAG); +} - assert(VT.is128BitVector() && "Unsupported vector size"); +/// \brief Handle lowering of 16-lane 16-bit integer shuffles. +/// +/// This routine is only called when we have AVX2 and thus a reasonable +/// instruction set for v16i16 shuffling.. +static SDValue lowerV16I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); + assert(V1.getSimpleValueType() == MVT::v16i16 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v16i16 && "Bad operand type!"); + ShuffleVectorSDNode *SVOp = cast(Op); + ArrayRef Mask = SVOp->getMask(); + assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!"); + assert(Subtarget->hasAVX2() && "We can only lower v16i16 with AVX2!"); - std::pair Locs[4]; - int Mask1[] = { -1, -1, -1, -1 }; - SmallVector PermMask(SVOp->getMask().begin(), SVOp->getMask().end()); + // Whenever we can lower this as a zext, that instruction is strictly faster + // than any alternative. It also allows us to fold memory operands into the + // shuffle in many cases. + if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v16i16, V1, V2, + Mask, Subtarget, DAG)) + return ZExt; - unsigned NumHi = 0; - unsigned NumLo = 0; - for (unsigned i = 0; i != 4; ++i) { - int Idx = PermMask[i]; - if (Idx < 0) { - Locs[i] = std::make_pair(-1, -1); - } else { - assert(Idx < 8 && "Invalid VECTOR_SHUFFLE index!"); - if (Idx < 4) { - Locs[i] = std::make_pair(0, NumLo); - Mask1[NumLo] = Idx; - NumLo++; - } else { - Locs[i] = std::make_pair(1, NumHi); - if (2+NumHi < 4) - Mask1[2+NumHi] = Idx; - NumHi++; - } - } - } + // Check for being able to broadcast a single element. + if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v16i16, DL, V1, + Mask, Subtarget, DAG)) + return Broadcast; - if (NumLo <= 2 && NumHi <= 2) { - // If no more than two elements come from either vector. This can be - // implemented with two shuffles. First shuffle gather the elements. - // The second shuffle, which takes the first shuffle as both of its - // vector operands, put the elements into the right order. - V1 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); + if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask, + Subtarget, DAG)) + return Blend; - int Mask2[] = { -1, -1, -1, -1 }; + // Use dedicated unpack instructions for masks that match their pattern. + if (isShuffleEquivalent(V1, V2, Mask, + // First 128-bit lane: + 0, 16, 1, 17, 2, 18, 3, 19, + // Second 128-bit lane: + 8, 24, 9, 25, 10, 26, 11, 27)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i16, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, + // First 128-bit lane: + 4, 20, 5, 21, 6, 22, 7, 23, + // Second 128-bit lane: + 12, 28, 13, 29, 14, 30, 15, 31)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i16, V1, V2); - for (unsigned i = 0; i != 4; ++i) - if (Locs[i].first != -1) { - unsigned Idx = (i < 2) ? 0 : 4; - Idx += Locs[i].first * 2 + Locs[i].second; - Mask2[i] = Idx; - } + // Try to use shift instructions. + if (SDValue Shift = + lowerVectorShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask, DAG)) + return Shift; - return DAG.getVectorShuffle(VT, dl, V1, V1, &Mask2[0]); - } + // Try to use byte rotation instructions. + if (SDValue Rotate = lowerVectorShuffleAsByteRotate( + DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG)) + return Rotate; - if (NumLo == 3 || NumHi == 3) { - // Otherwise, we must have three elements from one vector, call it X, and - // one element from the other, call it Y. First, use a shufps to build an - // intermediate vector with the one element from Y and the element from X - // that will be in the same half in the final destination (the indexes don't - // matter). Then, use a shufps to build the final vector, taking the half - // containing the element from Y from the intermediate, and the other half - // from X. - if (NumHi == 3) { - // Normalize it so the 3 elements come from V1. - CommuteVectorShuffleMask(PermMask, 4); - std::swap(V1, V2); - } + if (isSingleInputShuffleMask(Mask)) { + // There are no generalized cross-lane shuffle operations available on i16 + // element types. + if (is128BitLaneCrossingShuffleMask(MVT::v16i16, Mask)) + return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v16i16, V1, V2, + Mask, DAG); - // Find the element from V2. - unsigned HiIndex; - for (HiIndex = 0; HiIndex < 3; ++HiIndex) { - int Val = PermMask[HiIndex]; - if (Val < 0) + SDValue PSHUFBMask[32]; + for (int i = 0; i < 16; ++i) { + if (Mask[i] == -1) { + PSHUFBMask[2 * i] = PSHUFBMask[2 * i + 1] = DAG.getUNDEF(MVT::i8); continue; - if (Val >= 4) - break; - } + } - Mask1[0] = PermMask[HiIndex]; - Mask1[1] = -1; - Mask1[2] = PermMask[HiIndex^1]; - Mask1[3] = -1; - V2 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); - - if (HiIndex >= 2) { - Mask1[0] = PermMask[0]; - Mask1[1] = PermMask[1]; - Mask1[2] = HiIndex & 1 ? 6 : 4; - Mask1[3] = HiIndex & 1 ? 4 : 6; - return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]); - } - - Mask1[0] = HiIndex & 1 ? 2 : 0; - Mask1[1] = HiIndex & 1 ? 0 : 2; - Mask1[2] = PermMask[2]; - Mask1[3] = PermMask[3]; - if (Mask1[2] >= 0) - Mask1[2] += 4; - if (Mask1[3] >= 0) - Mask1[3] += 4; - return DAG.getVectorShuffle(VT, dl, V2, V1, &Mask1[0]); - } - - // Break it into (shuffle shuffle_hi, shuffle_lo). - int LoMask[] = { -1, -1, -1, -1 }; - int HiMask[] = { -1, -1, -1, -1 }; - - int *MaskPtr = LoMask; - unsigned MaskIdx = 0; - unsigned LoIdx = 0; - unsigned HiIdx = 2; - for (unsigned i = 0; i != 4; ++i) { - if (i == 2) { - MaskPtr = HiMask; - MaskIdx = 1; - LoIdx = 0; - HiIdx = 2; - } - int Idx = PermMask[i]; - if (Idx < 0) { - Locs[i] = std::make_pair(-1, -1); - } else if (Idx < 4) { - Locs[i] = std::make_pair(MaskIdx, LoIdx); - MaskPtr[LoIdx] = Idx; - LoIdx++; - } else { - Locs[i] = std::make_pair(MaskIdx, HiIdx); - MaskPtr[HiIdx] = Idx; - HiIdx++; + int M = i < 8 ? Mask[i] : Mask[i] - 8; + assert(M >= 0 && M < 8 && "Invalid single-input mask!"); + PSHUFBMask[2 * i] = DAG.getConstant(2 * M, MVT::i8); + PSHUFBMask[2 * i + 1] = DAG.getConstant(2 * M + 1, MVT::i8); } + return DAG.getNode( + ISD::BITCAST, DL, MVT::v16i16, + DAG.getNode( + X86ISD::PSHUFB, DL, MVT::v32i8, + DAG.getNode(ISD::BITCAST, DL, MVT::v32i8, V1), + DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, PSHUFBMask))); } - SDValue LoShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &LoMask[0]); - SDValue HiShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &HiMask[0]); - int MaskOps[] = { -1, -1, -1, -1 }; - for (unsigned i = 0; i != 4; ++i) - if (Locs[i].first != -1) - MaskOps[i] = Locs[i].first * 4 + Locs[i].second; - return DAG.getVectorShuffle(VT, dl, LoShuffle, HiShuffle, &MaskOps[0]); -} - -static bool MayFoldVectorLoad(SDValue V) { - while (V.hasOneUse() && V.getOpcode() == ISD::BITCAST) - V = V.getOperand(0); - - if (V.hasOneUse() && V.getOpcode() == ISD::SCALAR_TO_VECTOR) - V = V.getOperand(0); - if (V.hasOneUse() && V.getOpcode() == ISD::BUILD_VECTOR && - V.getNumOperands() == 2 && V.getOperand(1).getOpcode() == ISD::UNDEF) - // BUILD_VECTOR (load), undef - V = V.getOperand(0); - - return MayFoldLoad(V); -} - -static -SDValue getMOVDDup(SDValue &Op, SDLoc &dl, SDValue V1, SelectionDAG &DAG) { - MVT VT = Op.getSimpleValueType(); + // Try to simplify this by merging 128-bit lanes to enable a lane-based + // shuffle. + if (SDValue Result = lowerVectorShuffleByMerging128BitLanes( + DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG)) + return Result; - // Canonicalize to v2f64. - V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, V1); - return DAG.getNode(ISD::BITCAST, dl, VT, - getTargetShuffleNode(X86ISD::MOVDDUP, dl, MVT::v2f64, - V1, DAG)); + // Otherwise fall back on generic lowering. + return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask, DAG); } -static -SDValue getMOVLowToHigh(SDValue &Op, SDLoc &dl, SelectionDAG &DAG, - bool HasSSE2) { - SDValue V1 = Op.getOperand(0); - SDValue V2 = Op.getOperand(1); - MVT VT = Op.getSimpleValueType(); +/// \brief Handle lowering of 32-lane 8-bit integer shuffles. +/// +/// This routine is only called when we have AVX2 and thus a reasonable +/// instruction set for v32i8 shuffling.. +static SDValue lowerV32I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); + assert(V1.getSimpleValueType() == MVT::v32i8 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v32i8 && "Bad operand type!"); + ShuffleVectorSDNode *SVOp = cast(Op); + ArrayRef Mask = SVOp->getMask(); + assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!"); + assert(Subtarget->hasAVX2() && "We can only lower v32i8 with AVX2!"); - assert(VT != MVT::v2i64 && "unsupported shuffle type"); + // Whenever we can lower this as a zext, that instruction is strictly faster + // than any alternative. It also allows us to fold memory operands into the + // shuffle in many cases. + if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v32i8, V1, V2, + Mask, Subtarget, DAG)) + return ZExt; - if (HasSSE2 && VT == MVT::v2f64) - return getTargetShuffleNode(X86ISD::MOVLHPD, dl, VT, V1, V2, DAG); + // Check for being able to broadcast a single element. + if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v32i8, DL, V1, + Mask, Subtarget, DAG)) + return Broadcast; - // v4f32 or v4i32: canonicalize to v4f32 (which is legal for SSE1) - return DAG.getNode(ISD::BITCAST, dl, VT, - getTargetShuffleNode(X86ISD::MOVLHPS, dl, MVT::v4f32, - DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V1), - DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V2), DAG)); -} + if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask, + Subtarget, DAG)) + return Blend; -static -SDValue getMOVHighToLow(SDValue &Op, SDLoc &dl, SelectionDAG &DAG) { - SDValue V1 = Op.getOperand(0); - SDValue V2 = Op.getOperand(1); - MVT VT = Op.getSimpleValueType(); + // Use dedicated unpack instructions for masks that match their pattern. + // Note that these are repeated 128-bit lane unpacks, not unpacks across all + // 256-bit lanes. + if (isShuffleEquivalent( + V1, V2, Mask, + // First 128-bit lane: + 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, + // Second 128-bit lane: + 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v32i8, V1, V2); + if (isShuffleEquivalent( + V1, V2, Mask, + // First 128-bit lane: + 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47, + // Second 128-bit lane: + 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v32i8, V1, V2); - assert((VT == MVT::v4i32 || VT == MVT::v4f32) && - "unsupported shuffle type"); + // Try to use shift instructions. + if (SDValue Shift = + lowerVectorShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask, DAG)) + return Shift; - if (V2.getOpcode() == ISD::UNDEF) - V2 = V1; + // Try to use byte rotation instructions. + if (SDValue Rotate = lowerVectorShuffleAsByteRotate( + DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG)) + return Rotate; - // v4i32 or v4f32 - return getTargetShuffleNode(X86ISD::MOVHLPS, dl, VT, V1, V2, DAG); -} + if (isSingleInputShuffleMask(Mask)) { + // There are no generalized cross-lane shuffle operations available on i8 + // element types. + if (is128BitLaneCrossingShuffleMask(MVT::v32i8, Mask)) + return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v32i8, V1, V2, + Mask, DAG); -static -SDValue getMOVLP(SDValue &Op, SDLoc &dl, SelectionDAG &DAG, bool HasSSE2) { - SDValue V1 = Op.getOperand(0); - SDValue V2 = Op.getOperand(1); - MVT VT = Op.getSimpleValueType(); - unsigned NumElems = VT.getVectorNumElements(); + SDValue PSHUFBMask[32]; + for (int i = 0; i < 32; ++i) + PSHUFBMask[i] = + Mask[i] < 0 + ? DAG.getUNDEF(MVT::i8) + : DAG.getConstant(Mask[i] < 16 ? Mask[i] : Mask[i] - 16, MVT::i8); - // Use MOVLPS and MOVLPD in case V1 or V2 are loads. During isel, the second - // operand of these instructions is only memory, so check if there's a - // potencial load folding here, otherwise use SHUFPS or MOVSD to match the - // same masks. - bool CanFoldLoad = false; + return DAG.getNode( + X86ISD::PSHUFB, DL, MVT::v32i8, V1, + DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, PSHUFBMask)); + } - // Trivial case, when V2 comes from a load. - if (MayFoldVectorLoad(V2)) - CanFoldLoad = true; + // Try to simplify this by merging 128-bit lanes to enable a lane-based + // shuffle. + if (SDValue Result = lowerVectorShuffleByMerging128BitLanes( + DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG)) + return Result; - // When V1 is a load, it can be folded later into a store in isel, example: - // (store (v4f32 (X86Movlps (load addr:$src1), VR128:$src2)), addr:$src1) - // turns into: - // (MOVLPSmr addr:$src1, VR128:$src2) - // So, recognize this potential and also use MOVLPS or MOVLPD - else if (MayFoldVectorLoad(V1) && MayFoldIntoStore(Op)) - CanFoldLoad = true; + // Otherwise fall back on generic lowering. + return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask, DAG); +} +/// \brief High-level routine to lower various 256-bit x86 vector shuffles. +/// +/// This routine either breaks down the specific type of a 256-bit x86 vector +/// shuffle or splits it into two 128-bit shuffles and fuses the results back +/// together based on the available instructions. +static SDValue lower256BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2, + MVT VT, const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); ShuffleVectorSDNode *SVOp = cast(Op); - if (CanFoldLoad) { - if (HasSSE2 && NumElems == 2) - return getTargetShuffleNode(X86ISD::MOVLPD, dl, VT, V1, V2, DAG); + ArrayRef Mask = SVOp->getMask(); - if (NumElems == 4) - // If we don't care about the second element, proceed to use movss. - if (SVOp->getMaskElt(1) != -1) - return getTargetShuffleNode(X86ISD::MOVLPS, dl, VT, V1, V2, DAG); - } + // There is a really nice hard cut-over between AVX1 and AVX2 that means we can + // check for those subtargets here and avoid much of the subtarget querying in + // the per-vector-type lowering routines. With AVX1 we have essentially *zero* + // ability to manipulate a 256-bit vector with integer types. Since we'll use + // floating point types there eventually, just immediately cast everything to + // a float and operate entirely in that domain. + if (VT.isInteger() && !Subtarget->hasAVX2()) { + int ElementBits = VT.getScalarSizeInBits(); + if (ElementBits < 32) + // No floating point type available, decompose into 128-bit vectors. + return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG); - // movl and movlp will both match v2i64, but v2i64 is never matched by - // movl earlier because we make it strict to avoid messing with the movlp load - // folding logic (see the code above getMOVLP call). Match it here then, - // this is horrible, but will stay like this until we move all shuffle - // matching to x86 specific nodes. Note that for the 1st condition all - // types are matched with movsd. - if (HasSSE2) { - // FIXME: isMOVLMask should be checked and matched before getMOVLP, - // as to remove this logic from here, as much as possible - if (NumElems == 2 || !isMOVLMask(SVOp->getMask(), VT)) - return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG); - return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG); + MVT FpVT = MVT::getVectorVT(MVT::getFloatingPointVT(ElementBits), + VT.getVectorNumElements()); + V1 = DAG.getNode(ISD::BITCAST, DL, FpVT, V1); + V2 = DAG.getNode(ISD::BITCAST, DL, FpVT, V2); + return DAG.getNode(ISD::BITCAST, DL, VT, + DAG.getVectorShuffle(FpVT, DL, V1, V2, Mask)); } - assert(VT != MVT::v4i32 && "unsupported shuffle type"); + switch (VT.SimpleTy) { + case MVT::v4f64: + return lowerV4F64VectorShuffle(Op, V1, V2, Subtarget, DAG); + case MVT::v4i64: + return lowerV4I64VectorShuffle(Op, V1, V2, Subtarget, DAG); + case MVT::v8f32: + return lowerV8F32VectorShuffle(Op, V1, V2, Subtarget, DAG); + case MVT::v8i32: + return lowerV8I32VectorShuffle(Op, V1, V2, Subtarget, DAG); + case MVT::v16i16: + return lowerV16I16VectorShuffle(Op, V1, V2, Subtarget, DAG); + case MVT::v32i8: + return lowerV32I8VectorShuffle(Op, V1, V2, Subtarget, DAG); - // Invert the operand order and use SHUFPS to match it. - return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V2, V1, - getShuffleSHUFImmediate(SVOp), DAG); + default: + llvm_unreachable("Not a valid 256-bit x86 vector type!"); + } } -static SDValue NarrowVectorLoadToElement(LoadSDNode *Load, unsigned Index, - SelectionDAG &DAG) { - SDLoc dl(Load); - MVT VT = Load->getSimpleValueType(0); - MVT EVT = VT.getVectorElementType(); - SDValue Addr = Load->getOperand(1); - SDValue NewAddr = DAG.getNode( - ISD::ADD, dl, Addr.getSimpleValueType(), Addr, - DAG.getConstant(Index * EVT.getStoreSize(), Addr.getSimpleValueType())); +/// \brief Handle lowering of 8-lane 64-bit floating point shuffles. +static SDValue lowerV8F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); + assert(V1.getSimpleValueType() == MVT::v8f64 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v8f64 && "Bad operand type!"); + ShuffleVectorSDNode *SVOp = cast(Op); + ArrayRef Mask = SVOp->getMask(); + assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"); - SDValue NewLoad = - DAG.getLoad(EVT, dl, Load->getChain(), NewAddr, - DAG.getMachineFunction().getMachineMemOperand( - Load->getMemOperand(), 0, EVT.getStoreSize())); - return NewLoad; -} + // X86 has dedicated unpack instructions that can handle specific blend + // operations: UNPCKH and UNPCKL. + if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 2, 10, 4, 12, 6, 14)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8f64, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, 1, 9, 3, 11, 5, 13, 7, 15)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8f64, V1, V2); -// It is only safe to call this function if isINSERTPSMask is true for -// this shufflevector mask. -static SDValue getINSERTPS(ShuffleVectorSDNode *SVOp, SDLoc &dl, - SelectionDAG &DAG) { - // Generate an insertps instruction when inserting an f32 from memory onto a - // v4f32 or when copying a member from one v4f32 to another. - // We also use it for transferring i32 from one register to another, - // since it simply copies the same bits. - // If we're transferring an i32 from memory to a specific element in a - // register, we output a generic DAG that will match the PINSRD - // instruction. - MVT VT = SVOp->getSimpleValueType(0); - MVT EVT = VT.getVectorElementType(); - SDValue V1 = SVOp->getOperand(0); - SDValue V2 = SVOp->getOperand(1); - auto Mask = SVOp->getMask(); - assert((VT == MVT::v4f32 || VT == MVT::v4i32) && - "unsupported vector type for insertps/pinsrd"); - - auto FromV1Predicate = [](const int &i) { return i < 4 && i > -1; }; - auto FromV2Predicate = [](const int &i) { return i >= 4; }; - int FromV1 = std::count_if(Mask.begin(), Mask.end(), FromV1Predicate); - - SDValue From; - SDValue To; - unsigned DestIndex; - if (FromV1 == 1) { - From = V1; - To = V2; - DestIndex = std::find_if(Mask.begin(), Mask.end(), FromV1Predicate) - - Mask.begin(); + // FIXME: Implement direct support for this type! + return splitAndLowerVectorShuffle(DL, MVT::v8f64, V1, V2, Mask, DAG); +} - // If we have 1 element from each vector, we have to check if we're - // changing V1's element's place. If so, we're done. Otherwise, we - // should assume we're changing V2's element's place and behave - // accordingly. - int FromV2 = std::count_if(Mask.begin(), Mask.end(), FromV2Predicate); - assert(DestIndex <= INT32_MAX && "truncated destination index"); - if (FromV1 == FromV2 && - static_cast(DestIndex) == Mask[DestIndex] % 4) { - From = V2; - To = V1; - DestIndex = - std::find_if(Mask.begin(), Mask.end(), FromV2Predicate) - Mask.begin(); - } - } else { - assert(std::count_if(Mask.begin(), Mask.end(), FromV2Predicate) == 1 && - "More than one element from V1 and from V2, or no elements from one " - "of the vectors. This case should not have returned true from " - "isINSERTPSMask"); - From = V2; - To = V1; - DestIndex = - std::find_if(Mask.begin(), Mask.end(), FromV2Predicate) - Mask.begin(); - } - - // Get an index into the source vector in the range [0,4) (the mask is - // in the range [0,8) because it can address V1 and V2) - unsigned SrcIndex = Mask[DestIndex] % 4; - if (MayFoldLoad(From)) { - // Trivial case, when From comes from a load and is only used by the - // shuffle. Make it use insertps from the vector that we need from that - // load. - SDValue NewLoad = - NarrowVectorLoadToElement(cast(From), SrcIndex, DAG); - if (!NewLoad.getNode()) - return SDValue(); +/// \brief Handle lowering of 16-lane 32-bit floating point shuffles. +static SDValue lowerV16F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); + assert(V1.getSimpleValueType() == MVT::v16f32 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v16f32 && "Bad operand type!"); + ShuffleVectorSDNode *SVOp = cast(Op); + ArrayRef Mask = SVOp->getMask(); + assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!"); - if (EVT == MVT::f32) { - // Create this as a scalar to vector to match the instruction pattern. - SDValue LoadScalarToVector = - DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, NewLoad); - SDValue InsertpsMask = DAG.getIntPtrConstant(DestIndex << 4); - return DAG.getNode(X86ISD::INSERTPS, dl, VT, To, LoadScalarToVector, - InsertpsMask); - } else { // EVT == MVT::i32 - // If we're getting an i32 from memory, use an INSERT_VECTOR_ELT - // instruction, to match the PINSRD instruction, which loads an i32 to a - // certain vector element. - return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, To, NewLoad, - DAG.getConstant(DestIndex, MVT::i32)); - } - } + // Use dedicated unpack instructions for masks that match their pattern. + if (isShuffleEquivalent(V1, V2, Mask, + 0, 16, 1, 17, 4, 20, 5, 21, + 8, 24, 9, 25, 12, 28, 13, 29)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16f32, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, + 2, 18, 3, 19, 6, 22, 7, 23, + 10, 26, 11, 27, 14, 30, 15, 31)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16f32, V1, V2); - // Vector-element-to-vector - SDValue InsertpsMask = DAG.getIntPtrConstant(DestIndex << 4 | SrcIndex << 6); - return DAG.getNode(X86ISD::INSERTPS, dl, VT, To, From, InsertpsMask); + // FIXME: Implement direct support for this type! + return splitAndLowerVectorShuffle(DL, MVT::v16f32, V1, V2, Mask, DAG); } -// Reduce a vector shuffle to zext. -static SDValue LowerVectorIntExtend(SDValue Op, const X86Subtarget *Subtarget, - SelectionDAG &DAG) { - // PMOVZX is only available from SSE41. - if (!Subtarget->hasSSE41()) - return SDValue(); +/// \brief Handle lowering of 8-lane 64-bit integer shuffles. +static SDValue lowerV8I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); + assert(V1.getSimpleValueType() == MVT::v8i64 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v8i64 && "Bad operand type!"); + ShuffleVectorSDNode *SVOp = cast(Op); + ArrayRef Mask = SVOp->getMask(); + assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"); - MVT VT = Op.getSimpleValueType(); + // X86 has dedicated unpack instructions that can handle specific blend + // operations: UNPCKH and UNPCKL. + if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 2, 10, 4, 12, 6, 14)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i64, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, 1, 9, 3, 11, 5, 13, 7, 15)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i64, V1, V2); - // Only AVX2 support 256-bit vector integer extending. - if (!Subtarget->hasInt256() && VT.is256BitVector()) - return SDValue(); + // FIXME: Implement direct support for this type! + return splitAndLowerVectorShuffle(DL, MVT::v8i64, V1, V2, Mask, DAG); +} - ShuffleVectorSDNode *SVOp = cast(Op); +/// \brief Handle lowering of 16-lane 32-bit integer shuffles. +static SDValue lowerV16I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { SDLoc DL(Op); - SDValue V1 = Op.getOperand(0); - SDValue V2 = Op.getOperand(1); - unsigned NumElems = VT.getVectorNumElements(); + assert(V1.getSimpleValueType() == MVT::v16i32 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v16i32 && "Bad operand type!"); + ShuffleVectorSDNode *SVOp = cast(Op); + ArrayRef Mask = SVOp->getMask(); + assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!"); - // Extending is an unary operation and the element type of the source vector - // won't be equal to or larger than i64. - if (V2.getOpcode() != ISD::UNDEF || !VT.isInteger() || - VT.getVectorElementType() == MVT::i64) - return SDValue(); + // Use dedicated unpack instructions for masks that match their pattern. + if (isShuffleEquivalent(V1, V2, Mask, + 0, 16, 1, 17, 4, 20, 5, 21, + 8, 24, 9, 25, 12, 28, 13, 29)) + return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i32, V1, V2); + if (isShuffleEquivalent(V1, V2, Mask, + 2, 18, 3, 19, 6, 22, 7, 23, + 10, 26, 11, 27, 14, 30, 15, 31)) + return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i32, V1, V2); - // Find the expansion ratio, e.g. expanding from i8 to i32 has a ratio of 4. - unsigned Shift = 1; // Start from 2, i.e. 1 << 1. - while ((1U << Shift) < NumElems) { - if (SVOp->getMaskElt(1U << Shift) == 1) - break; - Shift += 1; - // The maximal ratio is 8, i.e. from i8 to i64. - if (Shift > 3) - return SDValue(); - } + // FIXME: Implement direct support for this type! + return splitAndLowerVectorShuffle(DL, MVT::v16i32, V1, V2, Mask, DAG); +} - // Check the shuffle mask. - unsigned Mask = (1U << Shift) - 1; - for (unsigned i = 0; i != NumElems; ++i) { - int EltIdx = SVOp->getMaskElt(i); - if ((i & Mask) != 0 && EltIdx != -1) - return SDValue(); - if ((i & Mask) == 0 && (unsigned)EltIdx != (i >> Shift)) - return SDValue(); - } +/// \brief Handle lowering of 32-lane 16-bit integer shuffles. +static SDValue lowerV32I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); + assert(V1.getSimpleValueType() == MVT::v32i16 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v32i16 && "Bad operand type!"); + ShuffleVectorSDNode *SVOp = cast(Op); + ArrayRef Mask = SVOp->getMask(); + assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!"); + assert(Subtarget->hasBWI() && "We can only lower v32i16 with AVX-512-BWI!"); - unsigned NBits = VT.getVectorElementType().getSizeInBits() << Shift; - MVT NeVT = MVT::getIntegerVT(NBits); - MVT NVT = MVT::getVectorVT(NeVT, NumElems >> Shift); + // FIXME: Implement direct support for this type! + return splitAndLowerVectorShuffle(DL, MVT::v32i16, V1, V2, Mask, DAG); +} - if (!DAG.getTargetLoweringInfo().isTypeLegal(NVT)) - return SDValue(); +/// \brief Handle lowering of 64-lane 8-bit integer shuffles. +static SDValue lowerV64I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2, + const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); + assert(V1.getSimpleValueType() == MVT::v64i8 && "Bad operand type!"); + assert(V2.getSimpleValueType() == MVT::v64i8 && "Bad operand type!"); + ShuffleVectorSDNode *SVOp = cast(Op); + ArrayRef Mask = SVOp->getMask(); + assert(Mask.size() == 64 && "Unexpected mask size for v64 shuffle!"); + assert(Subtarget->hasBWI() && "We can only lower v64i8 with AVX-512-BWI!"); - return DAG.getNode(ISD::BITCAST, DL, VT, - DAG.getNode(X86ISD::VZEXT, DL, NVT, V1)); + // FIXME: Implement direct support for this type! + return splitAndLowerVectorShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG); } -static SDValue NormalizeVectorShuffle(SDValue Op, const X86Subtarget *Subtarget, - SelectionDAG &DAG) { +/// \brief High-level routine to lower various 512-bit x86 vector shuffles. +/// +/// This routine either breaks down the specific type of a 512-bit x86 vector +/// shuffle or splits it into two 256-bit shuffles and fuses the results back +/// together based on the available instructions. +static SDValue lower512BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2, + MVT VT, const X86Subtarget *Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); ShuffleVectorSDNode *SVOp = cast(Op); - MVT VT = Op.getSimpleValueType(); - SDLoc dl(Op); - SDValue V1 = Op.getOperand(0); - SDValue V2 = Op.getOperand(1); - - if (isZeroShuffle(SVOp)) - return getZeroVector(VT, Subtarget, DAG, dl); + ArrayRef Mask = SVOp->getMask(); + assert(Subtarget->hasAVX512() && + "Cannot lower 512-bit vectors w/ basic ISA!"); - // Handle splat operations - if (SVOp->isSplat()) { - // Use vbroadcast whenever the splat comes from a foldable load - SDValue Broadcast = LowerVectorBroadcast(Op, Subtarget, DAG); - if (Broadcast.getNode()) - return Broadcast; - } + // Check for being able to broadcast a single element. + if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(VT.SimpleTy, DL, V1, + Mask, Subtarget, DAG)) + return Broadcast; - // Check integer expanding shuffles. - SDValue NewOp = LowerVectorIntExtend(Op, Subtarget, DAG); - if (NewOp.getNode()) - return NewOp; + // Dispatch to each element type for lowering. If we don't have supprot for + // specific element type shuffles at 512 bits, immediately split them and + // lower them. Each lowering routine of a given type is allowed to assume that + // the requisite ISA extensions for that element type are available. + switch (VT.SimpleTy) { + case MVT::v8f64: + return lowerV8F64VectorShuffle(Op, V1, V2, Subtarget, DAG); + case MVT::v16f32: + return lowerV16F32VectorShuffle(Op, V1, V2, Subtarget, DAG); + case MVT::v8i64: + return lowerV8I64VectorShuffle(Op, V1, V2, Subtarget, DAG); + case MVT::v16i32: + return lowerV16I32VectorShuffle(Op, V1, V2, Subtarget, DAG); + case MVT::v32i16: + if (Subtarget->hasBWI()) + return lowerV32I16VectorShuffle(Op, V1, V2, Subtarget, DAG); + break; + case MVT::v64i8: + if (Subtarget->hasBWI()) + return lowerV64I8VectorShuffle(Op, V1, V2, Subtarget, DAG); + break; - // If the shuffle can be profitably rewritten as a narrower shuffle, then - // do it! - if (VT == MVT::v8i16 || VT == MVT::v16i8 || VT == MVT::v16i16 || - VT == MVT::v32i8) { - SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG); - if (NewOp.getNode()) - return DAG.getNode(ISD::BITCAST, dl, VT, NewOp); - } else if (VT.is128BitVector() && Subtarget->hasSSE2()) { - // FIXME: Figure out a cleaner way to do this. - if (ISD::isBuildVectorAllZeros(V2.getNode())) { - SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG); - if (NewOp.getNode()) { - MVT NewVT = NewOp.getSimpleValueType(); - if (isCommutedMOVLMask(cast(NewOp)->getMask(), - NewVT, true, false)) - return getVZextMovL(VT, NewVT, NewOp.getOperand(0), DAG, Subtarget, - dl); - } - } else if (ISD::isBuildVectorAllZeros(V1.getNode())) { - SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG); - if (NewOp.getNode()) { - MVT NewVT = NewOp.getSimpleValueType(); - if (isMOVLMask(cast(NewOp)->getMask(), NewVT)) - return getVZextMovL(VT, NewVT, NewOp.getOperand(1), DAG, Subtarget, - dl); - } - } + default: + llvm_unreachable("Not a valid 512-bit x86 vector type!"); } - return SDValue(); + + // Otherwise fall back on splitting. + return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG); } -SDValue -X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const { +/// \brief Top-level lowering for x86 vector shuffles. +/// +/// This handles decomposition, canonicalization, and lowering of all x86 +/// vector shuffles. Most of the specific lowering strategies are encapsulated +/// above in helper routines. The canonicalization attempts to widen shuffles +/// to involve fewer lanes of wider elements, consolidate symmetric patterns +/// s.t. only one of the two inputs needs to be tested, etc. +static SDValue lowerVectorShuffle(SDValue Op, const X86Subtarget *Subtarget, + SelectionDAG &DAG) { ShuffleVectorSDNode *SVOp = cast(Op); + ArrayRef Mask = SVOp->getMask(); SDValue V1 = Op.getOperand(0); SDValue V2 = Op.getOperand(1); MVT VT = Op.getSimpleValueType(); + int NumElements = VT.getVectorNumElements(); SDLoc dl(Op); - unsigned NumElems = VT.getVectorNumElements(); - bool V1IsUndef = V1.getOpcode() == ISD::UNDEF; - bool V2IsUndef = V2.getOpcode() == ISD::UNDEF; - bool V1IsSplat = false; - bool V2IsSplat = false; - bool HasSSE2 = Subtarget->hasSSE2(); - bool HasFp256 = Subtarget->hasFp256(); - bool HasInt256 = Subtarget->hasInt256(); - MachineFunction &MF = DAG.getMachineFunction(); - bool OptForSize = - MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize); - - // Check if we should use the experimental vector shuffle lowering. If so, - // delegate completely to that code path. - if (ExperimentalVectorShuffleLowering) - return lowerVectorShuffle(Op, Subtarget, DAG); assert(VT.getSizeInBits() != 64 && "Can't lower MMX shuffles"); + bool V1IsUndef = V1.getOpcode() == ISD::UNDEF; + bool V2IsUndef = V2.getOpcode() == ISD::UNDEF; if (V1IsUndef && V2IsUndef) return DAG.getUNDEF(VT); @@ -12665,322 +9941,111 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const { if (V1IsUndef) return DAG.getCommutedVectorShuffle(*SVOp); - // Vector shuffle lowering takes 3 steps: - // - // 1) Normalize the input vectors. Here splats, zeroed vectors, profitable - // narrowing and commutation of operands should be handled. - // 2) Matching of shuffles with known shuffle masks to x86 target specific - // shuffle nodes. - // 3) Rewriting of unmatched masks into new generic shuffle operations, - // so the shuffle can be broken into other shuffles and the legalizer can - // try the lowering again. - // - // The general idea is that no vector_shuffle operation should be left to - // be matched during isel, all of them must be converted to a target specific - // node here. - - // Normalize the input vectors. Here splats, zeroed vectors, profitable - // narrowing and commutation of operands should be handled. The actual code - // doesn't include all of those, work in progress... - SDValue NewOp = NormalizeVectorShuffle(Op, Subtarget, DAG); - if (NewOp.getNode()) - return NewOp; - - SmallVector M(SVOp->getMask().begin(), SVOp->getMask().end()); - - // NOTE: isPSHUFDMask can also match both masks below (unpckl_undef and - // unpckh_undef). Only use pshufd if speed is more important than size. - if (OptForSize && isUNPCKL_v_undef_Mask(M, VT, HasInt256)) - return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG); - if (OptForSize && isUNPCKH_v_undef_Mask(M, VT, HasInt256)) - return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG); - - if (isMOVDDUPMask(M, VT) && Subtarget->hasSSE3() && - V2IsUndef && MayFoldVectorLoad(V1)) - return getMOVDDup(Op, dl, V1, DAG); - - if (isMOVHLPS_v_undef_Mask(M, VT)) - return getMOVHighToLow(Op, dl, DAG); - - // Use to match splats - if (HasSSE2 && isUNPCKHMask(M, VT, HasInt256) && V2IsUndef && - (VT == MVT::v2f64 || VT == MVT::v2i64)) - return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG); - - if (isPSHUFDMask(M, VT)) { - // The actual implementation will match the mask in the if above and then - // during isel it can match several different instructions, not only pshufd - // as its name says, sad but true, emulate the behavior for now... - if (isMOVDDUPMask(M, VT) && ((VT == MVT::v4f32 || VT == MVT::v2i64))) - return getTargetShuffleNode(X86ISD::MOVLHPS, dl, VT, V1, V1, DAG); - - unsigned TargetMask = getShuffleSHUFImmediate(SVOp); - - if (HasSSE2 && (VT == MVT::v4f32 || VT == MVT::v4i32)) - return getTargetShuffleNode(X86ISD::PSHUFD, dl, VT, V1, TargetMask, DAG); - - if (HasFp256 && (VT == MVT::v4f32 || VT == MVT::v2f64)) - return getTargetShuffleNode(X86ISD::VPERMILPI, dl, VT, V1, TargetMask, - DAG); - - return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V1, V1, - TargetMask, DAG); - } - - if (isPALIGNRMask(M, VT, Subtarget)) - return getTargetShuffleNode(X86ISD::PALIGNR, dl, VT, V1, V2, - getShufflePALIGNRImmediate(SVOp), - DAG); - - if (isVALIGNMask(M, VT, Subtarget)) - return getTargetShuffleNode(X86ISD::VALIGN, dl, VT, V1, V2, - getShuffleVALIGNImmediate(SVOp), - DAG); - - // Check if this can be converted into a logical shift. - bool isLeft = false; - unsigned ShAmt = 0; - SDValue ShVal; - bool isShift = HasSSE2 && isVectorShift(SVOp, DAG, isLeft, ShVal, ShAmt); - if (isShift && ShVal.hasOneUse()) { - // If the shifted value has multiple uses, it may be cheaper to use - // v_set0 + movlhps or movhlps, etc. - MVT EltVT = VT.getVectorElementType(); - ShAmt *= EltVT.getSizeInBits(); - return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl); - } + // Check for non-undef masks pointing at an undef vector and make the masks + // undef as well. This makes it easier to match the shuffle based solely on + // the mask. + if (V2IsUndef) + for (int M : Mask) + if (M >= NumElements) { + SmallVector NewMask(Mask.begin(), Mask.end()); + for (int &M : NewMask) + if (M >= NumElements) + M = -1; + return DAG.getVectorShuffle(VT, dl, V1, V2, NewMask); + } - if (isMOVLMask(M, VT)) { - if (ISD::isBuildVectorAllZeros(V1.getNode())) - return getVZextMovL(VT, VT, V2, DAG, Subtarget, dl); - if (!isMOVLPMask(M, VT)) { - if (HasSSE2 && (VT == MVT::v2i64 || VT == MVT::v2f64)) - return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG); + // We actually see shuffles that are entirely re-arrangements of a set of + // zero inputs. This mostly happens while decomposing complex shuffles into + // simple ones. Directly lower these as a buildvector of zeros. + SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2); + if (Zeroable.all()) + return getZeroVector(VT, Subtarget, DAG, dl); - if (VT == MVT::v4i32 || VT == MVT::v4f32) - return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG); + // Try to collapse shuffles into using a vector type with fewer elements but + // wider element types. We cap this to not form integers or floating point + // elements wider than 64 bits, but it might be interesting to form i128 + // integers to handle flipping the low and high halves of AVX 256-bit vectors. + SmallVector WidenedMask; + if (VT.getScalarSizeInBits() < 64 && + canWidenShuffleElements(Mask, WidenedMask)) { + MVT NewEltVT = VT.isFloatingPoint() + ? MVT::getFloatingPointVT(VT.getScalarSizeInBits() * 2) + : MVT::getIntegerVT(VT.getScalarSizeInBits() * 2); + MVT NewVT = MVT::getVectorVT(NewEltVT, VT.getVectorNumElements() / 2); + // Make sure that the new vector type is legal. For example, v2f64 isn't + // legal on SSE1. + if (DAG.getTargetLoweringInfo().isTypeLegal(NewVT)) { + V1 = DAG.getNode(ISD::BITCAST, dl, NewVT, V1); + V2 = DAG.getNode(ISD::BITCAST, dl, NewVT, V2); + return DAG.getNode(ISD::BITCAST, dl, VT, + DAG.getVectorShuffle(NewVT, dl, V1, V2, WidenedMask)); } } - // FIXME: fold these into legal mask. - if (isMOVLHPSMask(M, VT) && !isUNPCKLMask(M, VT, HasInt256)) - return getMOVLowToHigh(Op, dl, DAG, HasSSE2); - - if (isMOVHLPSMask(M, VT)) - return getMOVHighToLow(Op, dl, DAG); - - if (V2IsUndef && isMOVSHDUPMask(M, VT, Subtarget)) - return getTargetShuffleNode(X86ISD::MOVSHDUP, dl, VT, V1, DAG); - - if (V2IsUndef && isMOVSLDUPMask(M, VT, Subtarget)) - return getTargetShuffleNode(X86ISD::MOVSLDUP, dl, VT, V1, DAG); - - if (isMOVLPMask(M, VT)) - return getMOVLP(Op, dl, DAG, HasSSE2); - - if (ShouldXformToMOVHLPS(M, VT) || - ShouldXformToMOVLP(V1.getNode(), V2.getNode(), M, VT)) - return DAG.getCommutedVectorShuffle(*SVOp); - - if (isShift) { - // No better options. Use a vshldq / vsrldq. - MVT EltVT = VT.getVectorElementType(); - ShAmt *= EltVT.getSizeInBits(); - return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl); - } - - bool Commuted = false; - // FIXME: This should also accept a bitcast of a splat? Be careful, not - // 1,1,1,1 -> v8i16 though. - BitVector UndefElements; - if (auto *BVOp = dyn_cast(V1.getNode())) - if (BVOp->getConstantSplatNode(&UndefElements) && UndefElements.none()) - V1IsSplat = true; - if (auto *BVOp = dyn_cast(V2.getNode())) - if (BVOp->getConstantSplatNode(&UndefElements) && UndefElements.none()) - V2IsSplat = true; - - // Canonicalize the splat or undef, if present, to be on the RHS. - if (!V2IsUndef && V1IsSplat && !V2IsSplat) { - CommuteVectorShuffleMask(M, NumElems); - std::swap(V1, V2); - std::swap(V1IsSplat, V2IsSplat); - Commuted = true; - } - - if (isCommutedMOVLMask(M, VT, V2IsSplat, V2IsUndef)) { - // Shuffling low element of v1 into undef, just return v1. - if (V2IsUndef) - return V1; - // If V2 is a splat, the mask may be malformed such as <4,3,3,3>, which - // the instruction selector will not match, so get a canonical MOVL with - // swapped operands to undo the commute. - return getMOVL(DAG, dl, VT, V2, V1); - } - - if (isUNPCKLMask(M, VT, HasInt256)) - return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG); - - if (isUNPCKHMask(M, VT, HasInt256)) - return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG); - - if (V2IsSplat) { - // Normalize mask so all entries that point to V2 points to its first - // element then try to match unpck{h|l} again. If match, return a - // new vector_shuffle with the corrected mask.p - SmallVector NewMask(M.begin(), M.end()); - NormalizeMask(NewMask, NumElems); - if (isUNPCKLMask(NewMask, VT, HasInt256, true)) - return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG); - if (isUNPCKHMask(NewMask, VT, HasInt256, true)) - return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG); - } - - if (Commuted) { - // Commute is back and try unpck* again. - // FIXME: this seems wrong. - CommuteVectorShuffleMask(M, NumElems); - std::swap(V1, V2); - std::swap(V1IsSplat, V2IsSplat); - - if (isUNPCKLMask(M, VT, HasInt256)) - return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG); - - if (isUNPCKHMask(M, VT, HasInt256)) - return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG); - } + int NumV1Elements = 0, NumUndefElements = 0, NumV2Elements = 0; + for (int M : SVOp->getMask()) + if (M < 0) + ++NumUndefElements; + else if (M < NumElements) + ++NumV1Elements; + else + ++NumV2Elements; - // Normalize the node to match x86 shuffle ops if needed - if (!V2IsUndef && (isSHUFPMask(M, VT, /* Commuted */ true))) + // Commute the shuffle as needed such that more elements come from V1 than + // V2. This allows us to match the shuffle pattern strictly on how many + // elements come from V1 without handling the symmetric cases. + if (NumV2Elements > NumV1Elements) return DAG.getCommutedVectorShuffle(*SVOp); - // The checks below are all present in isShuffleMaskLegal, but they are - // inlined here right now to enable us to directly emit target specific - // nodes, and remove one by one until they don't return Op anymore. - - if (ShuffleVectorSDNode::isSplatMask(&M[0], VT) && - SVOp->getSplatIndex() == 0 && V2IsUndef) { - if (VT == MVT::v2f64 || VT == MVT::v2i64) - return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG); - } - - if (isPSHUFHWMask(M, VT, HasInt256)) - return getTargetShuffleNode(X86ISD::PSHUFHW, dl, VT, V1, - getShufflePSHUFHWImmediate(SVOp), - DAG); - - if (isPSHUFLWMask(M, VT, HasInt256)) - return getTargetShuffleNode(X86ISD::PSHUFLW, dl, VT, V1, - getShufflePSHUFLWImmediate(SVOp), - DAG); - - unsigned MaskValue; - if (isBlendMask(M, VT, Subtarget->hasSSE41(), HasInt256, &MaskValue)) - return LowerVECTOR_SHUFFLEtoBlend(SVOp, MaskValue, Subtarget, DAG); - - if (isSHUFPMask(M, VT)) - return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V1, V2, - getShuffleSHUFImmediate(SVOp), DAG); - - if (isUNPCKL_v_undef_Mask(M, VT, HasInt256)) - return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG); - if (isUNPCKH_v_undef_Mask(M, VT, HasInt256)) - return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG); - - //===--------------------------------------------------------------------===// - // Generate target specific nodes for 128 or 256-bit shuffles only - // supported in the AVX instruction set. - // - - // Handle VMOVDDUPY permutations - if (V2IsUndef && isMOVDDUPYMask(M, VT, HasFp256)) - return getTargetShuffleNode(X86ISD::MOVDDUP, dl, VT, V1, DAG); - - // Handle VPERMILPS/D* permutations - if (isVPERMILPMask(M, VT)) { - if ((HasInt256 && VT == MVT::v8i32) || VT == MVT::v16i32) - return getTargetShuffleNode(X86ISD::PSHUFD, dl, VT, V1, - getShuffleSHUFImmediate(SVOp), DAG); - return getTargetShuffleNode(X86ISD::VPERMILPI, dl, VT, V1, - getShuffleSHUFImmediate(SVOp), DAG); - } - - unsigned Idx; - if (VT.is512BitVector() && isINSERT64x4Mask(M, VT, &Idx)) - return Insert256BitVector(V1, Extract256BitVector(V2, 0, DAG, dl), - Idx*(NumElems/2), DAG, dl); - - // Handle VPERM2F128/VPERM2I128 permutations - if (isVPERM2X128Mask(M, VT, HasFp256)) - return getTargetShuffleNode(X86ISD::VPERM2X128, dl, VT, V1, - V2, getShuffleVPERM2X128Immediate(SVOp), DAG); - - if (Subtarget->hasSSE41() && isINSERTPSMask(M, VT)) - return getINSERTPS(SVOp, dl, DAG); - - unsigned Imm8; - if (V2IsUndef && HasInt256 && isPermImmMask(M, VT, Imm8)) - return getTargetShuffleNode(X86ISD::VPERMI, dl, VT, V1, Imm8, DAG); - - if ((V2IsUndef && HasInt256 && VT.is256BitVector() && NumElems == 8) || - VT.is512BitVector()) { - MVT MaskEltVT = MVT::getIntegerVT(VT.getVectorElementType().getSizeInBits()); - MVT MaskVectorVT = MVT::getVectorVT(MaskEltVT, NumElems); - SmallVector permclMask; - for (unsigned i = 0; i != NumElems; ++i) { - permclMask.push_back(DAG.getConstant((M[i]>=0) ? M[i] : 0, MaskEltVT)); + // When the number of V1 and V2 elements are the same, try to minimize the + // number of uses of V2 in the low half of the vector. When that is tied, + // ensure that the sum of indices for V1 is equal to or lower than the sum + // indices for V2. When those are equal, try to ensure that the number of odd + // indices for V1 is lower than the number of odd indices for V2. + if (NumV1Elements == NumV2Elements) { + int LowV1Elements = 0, LowV2Elements = 0; + for (int M : SVOp->getMask().slice(0, NumElements / 2)) + if (M >= NumElements) + ++LowV2Elements; + else if (M >= 0) + ++LowV1Elements; + if (LowV2Elements > LowV1Elements) { + return DAG.getCommutedVectorShuffle(*SVOp); + } else if (LowV2Elements == LowV1Elements) { + int SumV1Indices = 0, SumV2Indices = 0; + for (int i = 0, Size = SVOp->getMask().size(); i < Size; ++i) + if (SVOp->getMask()[i] >= NumElements) + SumV2Indices += i; + else if (SVOp->getMask()[i] >= 0) + SumV1Indices += i; + if (SumV2Indices < SumV1Indices) { + return DAG.getCommutedVectorShuffle(*SVOp); + } else if (SumV2Indices == SumV1Indices) { + int NumV1OddIndices = 0, NumV2OddIndices = 0; + for (int i = 0, Size = SVOp->getMask().size(); i < Size; ++i) + if (SVOp->getMask()[i] >= NumElements) + NumV2OddIndices += i % 2; + else if (SVOp->getMask()[i] >= 0) + NumV1OddIndices += i % 2; + if (NumV2OddIndices < NumV1OddIndices) + return DAG.getCommutedVectorShuffle(*SVOp); + } } - - SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, dl, MaskVectorVT, permclMask); - if (V2IsUndef) - // Bitcast is for VPERMPS since mask is v8i32 but node takes v8f32 - return DAG.getNode(X86ISD::VPERMV, dl, VT, - DAG.getNode(ISD::BITCAST, dl, VT, Mask), V1); - return DAG.getNode(X86ISD::VPERMV3, dl, VT, V1, - DAG.getNode(ISD::BITCAST, dl, VT, Mask), V2); - } - - //===--------------------------------------------------------------------===// - // Since no target specific shuffle was selected for this generic one, - // lower it into other known shuffles. FIXME: this isn't true yet, but - // this is the plan. - // - - // Handle v8i16 specifically since SSE can do byte extraction and insertion. - if (VT == MVT::v8i16) { - SDValue NewOp = LowerVECTOR_SHUFFLEv8i16(Op, Subtarget, DAG); - if (NewOp.getNode()) - return NewOp; - } - - if (VT == MVT::v16i16 && HasInt256) { - SDValue NewOp = LowerVECTOR_SHUFFLEv16i16(Op, DAG); - if (NewOp.getNode()) - return NewOp; - } - - if (VT == MVT::v16i8) { - SDValue NewOp = LowerVECTOR_SHUFFLEv16i8(SVOp, Subtarget, DAG); - if (NewOp.getNode()) - return NewOp; } - if (VT == MVT::v32i8) { - SDValue NewOp = LowerVECTOR_SHUFFLEv32i8(SVOp, Subtarget, DAG); - if (NewOp.getNode()) - return NewOp; - } + // For each vector width, delegate to a specialized lowering routine. + if (VT.getSizeInBits() == 128) + return lower128BitVectorShuffle(Op, V1, V2, VT, Subtarget, DAG); - // Handle all 128-bit wide vectors with 4 elements, and match them with - // several different shuffle types. - if (NumElems == 4 && VT.is128BitVector()) - return LowerVECTOR_SHUFFLE_128v4(SVOp, DAG); + if (VT.getSizeInBits() == 256) + return lower256BitVectorShuffle(Op, V1, V2, VT, Subtarget, DAG); - // Handle general 256-bit shuffles - if (VT.is256BitVector()) - return LowerVECTOR_SHUFFLE_256(SVOp, DAG); + // Force AVX-512 vectors to be scalarized for now. + // FIXME: Implement AVX-512 support! + if (VT.getSizeInBits() == 512) + return lower512BitVectorShuffle(Op, V1, V2, VT, Subtarget, DAG); - return SDValue(); + llvm_unreachable("Unimplemented!"); } // This function assumes its argument is a BUILD_VECTOR of constants or @@ -19904,7 +16969,7 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { case ISD::ATOMIC_STORE: return LowerATOMIC_STORE(Op,DAG); case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); - case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); + case ISD::VECTOR_SHUFFLE: return lowerVectorShuffle(Op, Subtarget, DAG); case ISD::VSELECT: return LowerVSELECT(Op, DAG); case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); @@ -25921,6 +22986,23 @@ static SDValue PerformISDSETCCCombine(SDNode *N, SelectionDAG &DAG, return SDValue(); } +static SDValue NarrowVectorLoadToElement(LoadSDNode *Load, unsigned Index, + SelectionDAG &DAG) { + SDLoc dl(Load); + MVT VT = Load->getSimpleValueType(0); + MVT EVT = VT.getVectorElementType(); + SDValue Addr = Load->getOperand(1); + SDValue NewAddr = DAG.getNode( + ISD::ADD, dl, Addr.getSimpleValueType(), Addr, + DAG.getConstant(Index * EVT.getStoreSize(), Addr.getSimpleValueType())); + + SDValue NewLoad = + DAG.getLoad(EVT, dl, Load->getChain(), NewAddr, + DAG.getMachineFunction().getMachineMemOperand( + Load->getMemOperand(), 0, EVT.getStoreSize())); + return NewLoad; +} + static SDValue PerformINSERTPSCombine(SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget) { SDLoc dl(N); diff --git a/llvm/test/CodeGen/X86/avx2-pmovx-256-old-shuffle.ll b/llvm/test/CodeGen/X86/avx2-pmovx-256-old-shuffle.ll deleted file mode 100644 index 4054931..0000000 --- a/llvm/test/CodeGen/X86/avx2-pmovx-256-old-shuffle.ll +++ /dev/null @@ -1,27 +0,0 @@ -; RUN: llc < %s -x86-experimental-vector-shuffle-lowering=false -mattr=+avx2 | FileCheck %s - -target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" -target triple = "x86_64-apple-darwin" - -; PR21876 -; The old shuffle lowering sometimes generates VZEXT nodes with both input -; and output same-sized types, here 256-bits. For instance, a v8i8 to v8i32 -; zero-extend would become a (v8i32 (VZEXT v32i8)) node, which can't happen -; otherwise. The companion commit r223996 added those patterns temporarily. -; This test, along with the VR256 for AVX2 PMOVXrr instructions, should be -; removed once the old vector shuffle lowering goes away. - -define void @test_avx2_pmovx_256(<8 x i8>* %tmp64, <8 x float>* %tmp75) { -; CHECK-LABEL: test_avx2_pmovx_256 -; We really don't care about the generated code. -; CHECK: vpmovzxbd -; CHECK: vcvtdq2ps -; CHECK: vmovups -; CHECK: vzeroupper -; CHECK: retq - - %wide.load458 = load <8 x i8>* %tmp64, align 1 - %tmp68 = uitofp <8 x i8> %wide.load458 to <8 x float> - store <8 x float> %tmp68, <8 x float>* %tmp75, align 4 - ret void -} -- 2.7.4