From 73cdbbea02ac2f5470916520e92ede75e13fae06 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 18 Jan 2023 11:21:06 +0000 Subject: [PATCH] [DAG] combineInsertEltToShuffle - split off mergeInsertEltWithShuffle fold. NFC. combineInsertEltToShuffle was performing 2 very different folds in the same call, merging "(insert_vector_elt (vector_shuffle X, Y), (extract_vector_elt X, N), IdxC) --> (vector_shuffle X, Y)" and "(insert_vector_elt V, (bitcast X from vector type), IdxC) --> bitcast(shuffle (bitcast V), (extended X), Mask)" The folds are currently still attempted in the same order as before (just as 2 seperate calls) so there should be no change in behaviour. First step towards some adjustments to mergeInsertEltWithShuffle for D127115. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 158 ++++++++++++++------------ 1 file changed, 86 insertions(+), 72 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 0271ffc..4cb2280 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -577,6 +577,7 @@ namespace { SDValue CombineExtLoad(SDNode *N); SDValue CombineZExtLogicopShiftLoad(SDNode *N); SDValue combineRepeatedFPDivisors(SDNode *N); + SDValue mergeInsertEltWithShuffle(SDNode *N, unsigned InsIndex); SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex); SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT); SDValue BuildSDIV(SDNode *N); @@ -19965,94 +19966,104 @@ SDValue DAGCombiner::splitMergedValStore(StoreSDNode *ST) { return St1; } -/// Convert a disguised subvector insertion into a shuffle: -SDValue DAGCombiner::combineInsertEltToShuffle(SDNode *N, unsigned InsIndex) { +// Merge an insertion into an existing shuffle: +// (insert_vector_elt (vector_shuffle X, Y), (extract_vector_elt X, N), +// InsIndex) +// --> (vector_shuffle X, Y) and variations where shuffle operands may be +// CONCAT_VECTORS. +SDValue DAGCombiner::mergeInsertEltWithShuffle(SDNode *N, unsigned InsIndex) { assert(N->getOpcode() == ISD::INSERT_VECTOR_ELT && "Expected extract_vector_elt"); SDValue InsertVal = N->getOperand(1); SDValue Vec = N->getOperand(0); - // (insert_vector_elt (vector_shuffle X, Y), (extract_vector_elt X, N), - // InsIndex) - // --> (vector_shuffle X, Y) and variations where shuffle operands may be - // CONCAT_VECTORS. - if (Vec.getOpcode() == ISD::VECTOR_SHUFFLE && Vec.hasOneUse() && - InsertVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT && - isa(InsertVal.getOperand(1))) { - ShuffleVectorSDNode *SVN = cast(Vec.getNode()); - ArrayRef Mask = SVN->getMask(); + if (Vec.getOpcode() != ISD::VECTOR_SHUFFLE || + InsertVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT || + !isa(InsertVal.getOperand(1)) || !Vec.hasOneUse()) + return SDValue(); - SDValue X = Vec.getOperand(0); - SDValue Y = Vec.getOperand(1); - - // Vec's operand 0 is using indices from 0 to N-1 and - // operand 1 from N to 2N - 1, where N is the number of - // elements in the vectors. - SDValue InsertVal0 = InsertVal.getOperand(0); - int ElementOffset = -1; - - // We explore the inputs of the shuffle in order to see if we find the - // source of the extract_vector_elt. If so, we can use it to modify the - // shuffle rather than perform an insert_vector_elt. - SmallVector, 8> ArgWorkList; - ArgWorkList.emplace_back(Mask.size(), Y); - ArgWorkList.emplace_back(0, X); - - while (!ArgWorkList.empty()) { - int ArgOffset; - SDValue ArgVal; - std::tie(ArgOffset, ArgVal) = ArgWorkList.pop_back_val(); - - if (ArgVal == InsertVal0) { - ElementOffset = ArgOffset; - break; - } + auto *SVN = cast(Vec.getNode()); + ArrayRef Mask = SVN->getMask(); - // Peek through concat_vector. - if (ArgVal.getOpcode() == ISD::CONCAT_VECTORS) { - int CurrentArgOffset = - ArgOffset + ArgVal.getValueType().getVectorNumElements(); - int Step = ArgVal.getOperand(0).getValueType().getVectorNumElements(); - for (SDValue Op : reverse(ArgVal->ops())) { - CurrentArgOffset -= Step; - ArgWorkList.emplace_back(CurrentArgOffset, Op); - } + SDValue X = Vec.getOperand(0); + SDValue Y = Vec.getOperand(1); + + // Vec's operand 0 is using indices from 0 to N-1 and + // operand 1 from N to 2N - 1, where N is the number of + // elements in the vectors. + SDValue InsertVal0 = InsertVal.getOperand(0); + int ElementOffset = -1; + + // We explore the inputs of the shuffle in order to see if we find the + // source of the extract_vector_elt. If so, we can use it to modify the + // shuffle rather than perform an insert_vector_elt. + SmallVector, 8> ArgWorkList; + ArgWorkList.emplace_back(Mask.size(), Y); + ArgWorkList.emplace_back(0, X); + + while (!ArgWorkList.empty()) { + int ArgOffset; + SDValue ArgVal; + std::tie(ArgOffset, ArgVal) = ArgWorkList.pop_back_val(); + + if (ArgVal == InsertVal0) { + ElementOffset = ArgOffset; + break; + } - // Make sure we went through all the elements and did not screw up index - // computation. - assert(CurrentArgOffset == ArgOffset); + // Peek through concat_vector. + if (ArgVal.getOpcode() == ISD::CONCAT_VECTORS) { + int CurrentArgOffset = + ArgOffset + ArgVal.getValueType().getVectorNumElements(); + int Step = ArgVal.getOperand(0).getValueType().getVectorNumElements(); + for (SDValue Op : reverse(ArgVal->ops())) { + CurrentArgOffset -= Step; + ArgWorkList.emplace_back(CurrentArgOffset, Op); } - } - // If we failed to find a match, see if we can replace an UNDEF shuffle - // operand. - if (ElementOffset == -1 && Y.isUndef() && - InsertVal0.getValueType() == Y.getValueType()) { - ElementOffset = Mask.size(); - Y = InsertVal0; + // Make sure we went through all the elements and did not screw up index + // computation. + assert(CurrentArgOffset == ArgOffset); } + } - if (ElementOffset != -1) { - SmallVector NewMask(Mask); + // If we failed to find a match, see if we can replace an UNDEF shuffle + // operand. + if (ElementOffset == -1 && Y.isUndef() && + InsertVal0.getValueType() == Y.getValueType()) { + ElementOffset = Mask.size(); + Y = InsertVal0; + } - auto *ExtrIndex = cast(InsertVal.getOperand(1)); - NewMask[InsIndex] = ElementOffset + ExtrIndex->getZExtValue(); - assert(NewMask[InsIndex] < - (int)(2 * Vec.getValueType().getVectorNumElements()) && - NewMask[InsIndex] >= 0 && "NewMask[InsIndex] is out of bound"); + if (ElementOffset != -1) { + SmallVector NewMask(Mask); - SDValue LegalShuffle = - TLI.buildLegalVectorShuffle(Vec.getValueType(), SDLoc(N), X, - Y, NewMask, DAG); - if (LegalShuffle) - return LegalShuffle; - } + auto *ExtrIndex = cast(InsertVal.getOperand(1)); + NewMask[InsIndex] = ElementOffset + ExtrIndex->getZExtValue(); + assert(NewMask[InsIndex] < + (int)(2 * Vec.getValueType().getVectorNumElements()) && + NewMask[InsIndex] >= 0 && "NewMask[InsIndex] is out of bound"); + + SDValue LegalShuffle = TLI.buildLegalVectorShuffle( + Vec.getValueType(), SDLoc(N), X, Y, NewMask, DAG); + if (LegalShuffle) + return LegalShuffle; } - // insert_vector_elt V, (bitcast X from vector type), IdxC --> - // bitcast(shuffle (bitcast V), (extended X), Mask) - // Note: We do not use an insert_subvector node because that requires a - // legal subvector type. + return SDValue(); +} + +// Convert a disguised subvector insertion into a shuffle: +// insert_vector_elt V, (bitcast X from vector type), IdxC --> +// bitcast(shuffle (bitcast V), (extended X), Mask) +// Note: We do not use an insert_subvector node because that requires a +// legal subvector type. +SDValue DAGCombiner::combineInsertEltToShuffle(SDNode *N, unsigned InsIndex) { + assert(N->getOpcode() == ISD::INSERT_VECTOR_ELT && + "Expected extract_vector_elt"); + SDValue InsertVal = N->getOperand(1); + SDValue Vec = N->getOperand(0); + if (InsertVal.getOpcode() != ISD::BITCAST || !InsertVal.hasOneUse() || !InsertVal.getOperand(0).getValueType().isVector()) return SDValue(); @@ -20140,6 +20151,9 @@ SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) { // We must know which element is being inserted for folds below here. unsigned Elt = IndexC->getZExtValue(); + if (SDValue Shuf = mergeInsertEltWithShuffle(N, Elt)) + return Shuf; + if (SDValue Shuf = combineInsertEltToShuffle(N, Elt)) return Shuf; -- 2.7.4