1 //===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file implements the SelectionDAG::LegalizeVectors method.
11 // The vector legalizer looks for vector operations which might need to be
12 // scalarized and legalizes them. This is a separate step from Legalize because
13 // scalarizing can introduce illegal types. For example, suppose we have an
14 // ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition
15 // on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
16 // operation, which introduces nodes with the illegal type i64 which must be
17 // expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
18 // the operation must be unrolled, which introduces nodes with the illegal
19 // type i8 which must be promoted.
21 // This does not legalize vector manipulations like ISD::BUILD_VECTOR,
22 // or operations that happen to take a vector which are custom-lowered;
23 // the legalization for such operations never produces nodes
24 // with illegal types, so it's okay to put off legalizing them until
25 // SelectionDAG::Legalize runs.
27 //===----------------------------------------------------------------------===//
29 #include "llvm/ADT/DenseMap.h"
30 #include "llvm/ADT/SmallVector.h"
31 #include "llvm/CodeGen/ISDOpcodes.h"
32 #include "llvm/CodeGen/SelectionDAG.h"
33 #include "llvm/CodeGen/SelectionDAGNodes.h"
34 #include "llvm/CodeGen/TargetLowering.h"
35 #include "llvm/CodeGen/ValueTypes.h"
36 #include "llvm/IR/DataLayout.h"
37 #include "llvm/Support/Casting.h"
38 #include "llvm/Support/Compiler.h"
39 #include "llvm/Support/Debug.h"
40 #include "llvm/Support/ErrorHandling.h"
41 #include "llvm/Support/MachineValueType.h"
49 #define DEBUG_TYPE "legalizevectorops"
53 class VectorLegalizer {
55 const TargetLowering &TLI;
56 bool Changed = false; // Keep track of whether anything changed
58 /// For nodes that are of legal width, and that have more than one use, this
59 /// map indicates what regularized operand to use. This allows us to avoid
60 /// legalizing the same thing more than once.
61 SmallDenseMap<SDValue, SDValue, 64> LegalizedNodes;
63 /// Adds a node to the translation cache.
64 void AddLegalizedOperand(SDValue From, SDValue To) {
65 LegalizedNodes.insert(std::make_pair(From, To));
66 // If someone requests legalization of the new node, return itself.
68 LegalizedNodes.insert(std::make_pair(To, To));
71 /// Legalizes the given node.
72 SDValue LegalizeOp(SDValue Op);
74 /// Assuming the node is legal, "legalize" the results.
75 SDValue TranslateLegalizeResults(SDValue Op, SDNode *Result);
77 /// Make sure Results are legal and update the translation cache.
78 SDValue RecursivelyLegalizeResults(SDValue Op,
79 MutableArrayRef<SDValue> Results);
81 /// Wrapper to interface LowerOperation with a vector of Results.
82 /// Returns false if the target wants to use default expansion. Otherwise
83 /// returns true. If return is true and the Results are empty, then the
84 /// target wants to keep the input node as is.
85 bool LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results);
87 /// Implements unrolling a VSETCC.
88 SDValue UnrollVSETCC(SDNode *Node);
90 /// Implement expand-based legalization of vector operations.
92 /// This is just a high-level routine to dispatch to specific code paths for
93 /// operations to legalize them.
94 void Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results);
96 /// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if
97 /// FP_TO_SINT isn't legal.
98 void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
100 /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
101 /// SINT_TO_FLOAT and SHR on vectors isn't legal.
102 void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
104 /// Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
105 SDValue ExpandSEXTINREG(SDNode *Node);
107 /// Implement expansion for ANY_EXTEND_VECTOR_INREG.
109 /// Shuffles the low lanes of the operand into place and bitcasts to the proper
110 /// type. The contents of the bits in the extended part of each element are
112 SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);
114 /// Implement expansion for SIGN_EXTEND_VECTOR_INREG.
116 /// Shuffles the low lanes of the operand into place, bitcasts to the proper
117 /// type, then shifts left and arithmetic shifts right to introduce a sign
119 SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);
121 /// Implement expansion for ZERO_EXTEND_VECTOR_INREG.
123 /// Shuffles the low lanes of the operand into place and blends zeros into
124 /// the remaining lanes, finally bitcasting to the proper type.
125 SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);
127 /// Expand bswap of vectors into a shuffle if legal.
128 SDValue ExpandBSWAP(SDNode *Node);
130 /// Implement vselect in terms of XOR, AND, OR when blend is not
131 /// supported by the target.
132 SDValue ExpandVSELECT(SDNode *Node);
133 SDValue ExpandVP_SELECT(SDNode *Node);
134 SDValue ExpandVP_MERGE(SDNode *Node);
135 SDValue ExpandSELECT(SDNode *Node);
136 std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);
137 SDValue ExpandStore(SDNode *N);
138 SDValue ExpandFNEG(SDNode *Node);
139 void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
140 void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
141 void ExpandBITREVERSE(SDNode *Node, SmallVectorImpl<SDValue> &Results);
142 void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
143 void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
144 void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
145 void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);
146 void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
147 void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);
149 void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
151 /// Implements vector promotion.
153 /// This is essentially just bitcasting the operands to a different type and
154 /// bitcasting the result back to the original type.
155 void Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157 /// Implements [SU]INT_TO_FP vector promotion.
159 /// This is a [zs]ext of the input operand to a larger integer type.
160 void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);
162 /// Implements FP_TO_[SU]INT vector promotion of the result type.
164 /// It is promoted to a larger integer type. The result is then
165 /// truncated back to the original type.
166 void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
169 VectorLegalizer(SelectionDAG& dag) :
170 DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
172 /// Begin legalizer the vector operations in the DAG.
176 } // end anonymous namespace
178 bool VectorLegalizer::Run() {
179 // Before we start legalizing vector nodes, check if there are any vectors.
180 bool HasVectors = false;
181 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
182 E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) {
183 // Check if the values of the nodes contain vectors. We don't need to check
184 // the operands because we are going to check their values at some point.
185 HasVectors = llvm::any_of(I->values(), [](EVT T) { return T.isVector(); });
187 // If we found a vector node we can start the legalization.
192 // If this basic block has no vectors then no need to legalize vectors.
196 // The legalize process is inherently a bottom-up recursive process (users
197 // legalize their uses before themselves). Given infinite stack space, we
198 // could just start legalizing on the root and traverse the whole graph. In
199 // practice however, this causes us to run out of stack space on large basic
200 // blocks. To avoid this problem, compute an ordering of the nodes where each
201 // node is only legalized after all of its operands are legalized.
202 DAG.AssignTopologicalOrder();
203 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
204 E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I)
205 LegalizeOp(SDValue(&*I, 0));
207 // Finally, it's possible the root changed. Get the new root.
208 SDValue OldRoot = DAG.getRoot();
209 assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
210 DAG.setRoot(LegalizedNodes[OldRoot]);
212 LegalizedNodes.clear();
214 // Remove dead nodes now.
215 DAG.RemoveDeadNodes();
220 SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDNode *Result) {
221 assert(Op->getNumValues() == Result->getNumValues() &&
222 "Unexpected number of results");
223 // Generic legalization: just pass the operand through.
224 for (unsigned i = 0, e = Op->getNumValues(); i != e; ++i)
225 AddLegalizedOperand(Op.getValue(i), SDValue(Result, i));
226 return SDValue(Result, Op.getResNo());
230 VectorLegalizer::RecursivelyLegalizeResults(SDValue Op,
231 MutableArrayRef<SDValue> Results) {
232 assert(Results.size() == Op->getNumValues() &&
233 "Unexpected number of results");
234 // Make sure that the generated code is itself legal.
235 for (unsigned i = 0, e = Results.size(); i != e; ++i) {
236 Results[i] = LegalizeOp(Results[i]);
237 AddLegalizedOperand(Op.getValue(i), Results[i]);
240 return Results[Op.getResNo()];
243 SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
244 // Note that LegalizeOp may be reentered even from single-use nodes, which
245 // means that we always must cache transformed nodes.
246 DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);
247 if (I != LegalizedNodes.end()) return I->second;
249 // Legalize the operands
250 SmallVector<SDValue, 8> Ops;
251 for (const SDValue &Oper : Op->op_values())
252 Ops.push_back(LegalizeOp(Oper));
254 SDNode *Node = DAG.UpdateNodeOperands(Op.getNode(), Ops);
256 bool HasVectorValueOrOp =
257 llvm::any_of(Node->values(), [](EVT T) { return T.isVector(); }) ||
258 llvm::any_of(Node->op_values(),
259 [](SDValue O) { return O.getValueType().isVector(); });
260 if (!HasVectorValueOrOp)
261 return TranslateLegalizeResults(Op, Node);
263 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
265 switch (Op.getOpcode()) {
267 return TranslateLegalizeResults(Op, Node);
269 LoadSDNode *LD = cast<LoadSDNode>(Node);
270 ISD::LoadExtType ExtType = LD->getExtensionType();
271 EVT LoadedVT = LD->getMemoryVT();
272 if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD)
273 Action = TLI.getLoadExtAction(ExtType, LD->getValueType(0), LoadedVT);
277 StoreSDNode *ST = cast<StoreSDNode>(Node);
278 EVT StVT = ST->getMemoryVT();
279 MVT ValVT = ST->getValue().getSimpleValueType();
280 if (StVT.isVector() && ST->isTruncatingStore())
281 Action = TLI.getTruncStoreAction(ValVT, StVT);
284 case ISD::MERGE_VALUES:
285 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
286 // This operation lies about being legal: when it claims to be legal,
287 // it should actually be expanded.
288 if (Action == TargetLowering::Legal)
289 Action = TargetLowering::Expand;
291 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
292 case ISD::STRICT_##DAGN:
293 #include "llvm/IR/ConstrainedOps.def"
294 ValVT = Node->getValueType(0);
295 if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
296 Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
297 ValVT = Node->getOperand(1).getValueType();
298 Action = TLI.getOperationAction(Node->getOpcode(), ValVT);
299 // If we're asked to expand a strict vector floating-point operation,
300 // by default we're going to simply unroll it. That is usually the
301 // best approach, except in the case where the resulting strict (scalar)
302 // operations would themselves use the fallback mutation to non-strict.
303 // In that specific case, just do the fallback on the vector op.
304 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
305 TLI.getStrictFPOperationAction(Node->getOpcode(), ValVT) ==
306 TargetLowering::Legal) {
307 EVT EltVT = ValVT.getVectorElementType();
308 if (TLI.getOperationAction(Node->getOpcode(), EltVT)
309 == TargetLowering::Expand &&
310 TLI.getStrictFPOperationAction(Node->getOpcode(), EltVT)
311 == TargetLowering::Legal)
312 Action = TargetLowering::Legal;
343 case ISD::BITREVERSE:
346 case ISD::CTLZ_ZERO_UNDEF:
347 case ISD::CTTZ_ZERO_UNDEF:
352 case ISD::ZERO_EXTEND:
353 case ISD::ANY_EXTEND:
355 case ISD::SIGN_EXTEND:
356 case ISD::FP_TO_SINT:
357 case ISD::FP_TO_UINT:
362 case ISD::FMINNUM_IEEE:
363 case ISD::FMAXNUM_IEEE:
380 case ISD::FNEARBYINT:
382 case ISD::FROUNDEVEN:
387 case ISD::SIGN_EXTEND_INREG:
388 case ISD::ANY_EXTEND_VECTOR_INREG:
389 case ISD::SIGN_EXTEND_VECTOR_INREG:
390 case ISD::ZERO_EXTEND_VECTOR_INREG:
403 case ISD::FCANONICALIZE:
410 case ISD::FP_TO_SINT_SAT:
411 case ISD::FP_TO_UINT_SAT:
413 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
416 case ISD::SMULFIXSAT:
418 case ISD::UMULFIXSAT:
420 case ISD::SDIVFIXSAT:
422 case ISD::UDIVFIXSAT: {
423 unsigned Scale = Node->getConstantOperandVal(2);
424 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
425 Node->getValueType(0), Scale);
428 case ISD::SINT_TO_FP:
429 case ISD::UINT_TO_FP:
430 case ISD::VECREDUCE_ADD:
431 case ISD::VECREDUCE_MUL:
432 case ISD::VECREDUCE_AND:
433 case ISD::VECREDUCE_OR:
434 case ISD::VECREDUCE_XOR:
435 case ISD::VECREDUCE_SMAX:
436 case ISD::VECREDUCE_SMIN:
437 case ISD::VECREDUCE_UMAX:
438 case ISD::VECREDUCE_UMIN:
439 case ISD::VECREDUCE_FADD:
440 case ISD::VECREDUCE_FMUL:
441 case ISD::VECREDUCE_FMAX:
442 case ISD::VECREDUCE_FMIN:
443 Action = TLI.getOperationAction(Node->getOpcode(),
444 Node->getOperand(0).getValueType());
446 case ISD::VECREDUCE_SEQ_FADD:
447 case ISD::VECREDUCE_SEQ_FMUL:
448 Action = TLI.getOperationAction(Node->getOpcode(),
449 Node->getOperand(1).getValueType());
452 MVT OpVT = Node->getOperand(0).getSimpleValueType();
453 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
454 Action = TLI.getCondCodeAction(CCCode, OpVT);
455 if (Action == TargetLowering::Legal)
456 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
460 #define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
462 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
463 : Node->getOperand(LEGALPOS).getValueType(); \
464 if (ISD::VPID == ISD::VP_SETCC) { \
465 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \
466 Action = TLI.getCondCodeAction(CCCode, LegalizeVT.getSimpleVT()); \
467 if (Action != TargetLowering::Legal) \
470 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
472 #include "llvm/IR/VPIntrinsics.def"
475 LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
477 SmallVector<SDValue, 8> ResultVals;
479 default: llvm_unreachable("This action is not supported yet!");
480 case TargetLowering::Promote:
481 assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) &&
482 "This action is not supported yet!");
483 LLVM_DEBUG(dbgs() << "Promoting\n");
484 Promote(Node, ResultVals);
485 assert(!ResultVals.empty() && "No results for promotion?");
487 case TargetLowering::Legal:
488 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
490 case TargetLowering::Custom:
491 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
492 if (LowerOperationWrapper(Node, ResultVals))
494 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
496 case TargetLowering::Expand:
497 LLVM_DEBUG(dbgs() << "Expanding\n");
498 Expand(Node, ResultVals);
502 if (ResultVals.empty())
503 return TranslateLegalizeResults(Op, Node);
506 return RecursivelyLegalizeResults(Op, ResultVals);
509 // FIXME: This is very similar to TargetLowering::LowerOperationWrapper. Can we
510 // merge them somehow?
511 bool VectorLegalizer::LowerOperationWrapper(SDNode *Node,
512 SmallVectorImpl<SDValue> &Results) {
513 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
518 if (Res == SDValue(Node, 0))
521 // If the original node has one result, take the return value from
522 // LowerOperation as is. It might not be result number 0.
523 if (Node->getNumValues() == 1) {
524 Results.push_back(Res);
528 // If the original node has multiple results, then the return node should
529 // have the same number of results.
530 assert((Node->getNumValues() == Res->getNumValues()) &&
531 "Lowering returned the wrong number of results!");
533 // Places new result values base on N result number.
534 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
535 Results.push_back(Res.getValue(I));
540 void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
541 // For a few operations there is a specific concept for promotion based on
542 // the operand's type.
543 switch (Node->getOpcode()) {
544 case ISD::SINT_TO_FP:
545 case ISD::UINT_TO_FP:
546 case ISD::STRICT_SINT_TO_FP:
547 case ISD::STRICT_UINT_TO_FP:
548 // "Promote" the operation by extending the operand.
549 PromoteINT_TO_FP(Node, Results);
551 case ISD::FP_TO_UINT:
552 case ISD::FP_TO_SINT:
553 case ISD::STRICT_FP_TO_UINT:
554 case ISD::STRICT_FP_TO_SINT:
555 // Promote the operation by extending the operand.
556 PromoteFP_TO_INT(Node, Results);
560 // These operations are used to do promotion so they can't be promoted
562 llvm_unreachable("Don't know how to promote this operation!");
565 // There are currently two cases of vector promotion:
566 // 1) Bitcasting a vector of integers to a different type to a vector of the
567 // same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.
568 // 2) Extending a vector of floats to a vector of the same number of larger
569 // floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.
570 assert(Node->getNumValues() == 1 &&
571 "Can't promote a vector with multiple results!");
572 MVT VT = Node->getSimpleValueType(0);
573 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
575 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
577 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
578 if (Node->getOperand(j).getValueType().isVector())
579 if (Node->getOperand(j)
581 .getVectorElementType()
582 .isFloatingPoint() &&
583 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint())
584 Operands[j] = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(j));
586 Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(j));
588 Operands[j] = Node->getOperand(j);
592 DAG.getNode(Node->getOpcode(), dl, NVT, Operands, Node->getFlags());
594 if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
595 (VT.isVector() && VT.getVectorElementType().isFloatingPoint() &&
596 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint()))
597 Res = DAG.getNode(ISD::FP_ROUND, dl, VT, Res, DAG.getIntPtrConstant(0, dl));
599 Res = DAG.getNode(ISD::BITCAST, dl, VT, Res);
601 Results.push_back(Res);
604 void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node,
605 SmallVectorImpl<SDValue> &Results) {
606 // INT_TO_FP operations may require the input operand be promoted even
607 // when the type is otherwise legal.
608 bool IsStrict = Node->isStrictFPOpcode();
609 MVT VT = Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();
610 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
611 assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
612 "Vectors have different number of elements!");
615 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
617 unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP ||
618 Node->getOpcode() == ISD::STRICT_UINT_TO_FP)
621 for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
622 if (Node->getOperand(j).getValueType().isVector())
623 Operands[j] = DAG.getNode(Opc, dl, NVT, Node->getOperand(j));
625 Operands[j] = Node->getOperand(j);
629 SDValue Res = DAG.getNode(Node->getOpcode(), dl,
630 {Node->getValueType(0), MVT::Other}, Operands);
631 Results.push_back(Res);
632 Results.push_back(Res.getValue(1));
637 DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Operands);
638 Results.push_back(Res);
641 // For FP_TO_INT we promote the result type to a vector type with wider
642 // elements and then truncate the result. This is different from the default
643 // PromoteVector which uses bitcast to promote thus assumning that the
644 // promoted vector type has the same overall size.
645 void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,
646 SmallVectorImpl<SDValue> &Results) {
647 MVT VT = Node->getSimpleValueType(0);
648 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
649 bool IsStrict = Node->isStrictFPOpcode();
650 assert(NVT.getVectorNumElements() == VT.getVectorNumElements() &&
651 "Vectors have different number of elements!");
653 unsigned NewOpc = Node->getOpcode();
654 // Change FP_TO_UINT to FP_TO_SINT if possible.
655 // TODO: Should we only do this if FP_TO_UINT itself isn't legal?
656 if (NewOpc == ISD::FP_TO_UINT &&
657 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
658 NewOpc = ISD::FP_TO_SINT;
660 if (NewOpc == ISD::STRICT_FP_TO_UINT &&
661 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
662 NewOpc = ISD::STRICT_FP_TO_SINT;
665 SDValue Promoted, Chain;
667 Promoted = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
668 {Node->getOperand(0), Node->getOperand(1)});
669 Chain = Promoted.getValue(1);
671 Promoted = DAG.getNode(NewOpc, dl, NVT, Node->getOperand(0));
673 // Assert that the converted value fits in the original type. If it doesn't
674 // (eg: because the value being converted is too big), then the result of the
675 // original operation was undefined anyway, so the assert is still correct.
676 if (Node->getOpcode() == ISD::FP_TO_UINT ||
677 Node->getOpcode() == ISD::STRICT_FP_TO_UINT)
678 NewOpc = ISD::AssertZext;
680 NewOpc = ISD::AssertSext;
682 Promoted = DAG.getNode(NewOpc, dl, NVT, Promoted,
683 DAG.getValueType(VT.getScalarType()));
684 Promoted = DAG.getNode(ISD::TRUNCATE, dl, VT, Promoted);
685 Results.push_back(Promoted);
687 Results.push_back(Chain);
690 std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {
691 LoadSDNode *LD = cast<LoadSDNode>(N);
692 return TLI.scalarizeVectorLoad(LD, DAG);
695 SDValue VectorLegalizer::ExpandStore(SDNode *N) {
696 StoreSDNode *ST = cast<StoreSDNode>(N);
697 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
701 void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
702 switch (Node->getOpcode()) {
704 std::pair<SDValue, SDValue> Tmp = ExpandLoad(Node);
705 Results.push_back(Tmp.first);
706 Results.push_back(Tmp.second);
710 Results.push_back(ExpandStore(Node));
712 case ISD::MERGE_VALUES:
713 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
714 Results.push_back(Node->getOperand(i));
716 case ISD::SIGN_EXTEND_INREG:
717 Results.push_back(ExpandSEXTINREG(Node));
719 case ISD::ANY_EXTEND_VECTOR_INREG:
720 Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(Node));
722 case ISD::SIGN_EXTEND_VECTOR_INREG:
723 Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(Node));
725 case ISD::ZERO_EXTEND_VECTOR_INREG:
726 Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(Node));
729 Results.push_back(ExpandBSWAP(Node));
732 Results.push_back(ExpandVSELECT(Node));
735 Results.push_back(ExpandVP_SELECT(Node));
738 Results.push_back(ExpandSELECT(Node));
740 case ISD::SELECT_CC: {
741 if (Node->getValueType(0).isScalableVector()) {
742 EVT CondVT = TLI.getSetCCResultType(
743 DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
745 DAG.getNode(ISD::SETCC, SDLoc(Node), CondVT, Node->getOperand(0),
746 Node->getOperand(1), Node->getOperand(4));
747 Results.push_back(DAG.getSelect(SDLoc(Node), Node->getValueType(0), SetCC,
749 Node->getOperand(3)));
754 case ISD::FP_TO_UINT:
755 ExpandFP_TO_UINT(Node, Results);
757 case ISD::UINT_TO_FP:
758 ExpandUINT_TO_FLOAT(Node, Results);
761 Results.push_back(ExpandFNEG(Node));
764 ExpandFSUB(Node, Results);
768 ExpandSETCC(Node, Results);
771 if (SDValue Expanded = TLI.expandABS(Node, DAG)) {
772 Results.push_back(Expanded);
776 case ISD::BITREVERSE:
777 ExpandBITREVERSE(Node, Results);
780 if (SDValue Expanded = TLI.expandCTPOP(Node, DAG)) {
781 Results.push_back(Expanded);
786 case ISD::CTLZ_ZERO_UNDEF:
787 if (SDValue Expanded = TLI.expandCTLZ(Node, DAG)) {
788 Results.push_back(Expanded);
793 case ISD::CTTZ_ZERO_UNDEF:
794 if (SDValue Expanded = TLI.expandCTTZ(Node, DAG)) {
795 Results.push_back(Expanded);
801 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG)) {
802 Results.push_back(Expanded);
808 if (SDValue Expanded = TLI.expandROT(Node, false /*AllowVectorOps*/, DAG)) {
809 Results.push_back(Expanded);
815 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) {
816 Results.push_back(Expanded);
824 if (SDValue Expanded = TLI.expandIntMINMAX(Node, DAG)) {
825 Results.push_back(Expanded);
831 ExpandUADDSUBO(Node, Results);
835 ExpandSADDSUBO(Node, Results);
839 ExpandMULO(Node, Results);
845 if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) {
846 Results.push_back(Expanded);
850 case ISD::FP_TO_SINT_SAT:
851 case ISD::FP_TO_UINT_SAT:
852 // Expand the fpsosisat if it is scalable to prevent it from unrolling below.
853 if (Node->getValueType(0).isScalableVector()) {
854 if (SDValue Expanded = TLI.expandFP_TO_INT_SAT(Node, DAG)) {
855 Results.push_back(Expanded);
862 if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) {
863 Results.push_back(Expanded);
867 case ISD::SMULFIXSAT:
868 case ISD::UMULFIXSAT:
869 // FIXME: We do not expand SMULFIXSAT/UMULFIXSAT here yet, not sure exactly
870 // why. Maybe it results in worse codegen compared to the unroll for some
871 // targets? This should probably be investigated. And if we still prefer to
872 // unroll an explanation could be helpful.
876 ExpandFixedPointDiv(Node, Results);
878 case ISD::SDIVFIXSAT:
879 case ISD::UDIVFIXSAT:
881 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
882 case ISD::STRICT_##DAGN:
883 #include "llvm/IR/ConstrainedOps.def"
884 ExpandStrictFPOp(Node, Results);
886 case ISD::VECREDUCE_ADD:
887 case ISD::VECREDUCE_MUL:
888 case ISD::VECREDUCE_AND:
889 case ISD::VECREDUCE_OR:
890 case ISD::VECREDUCE_XOR:
891 case ISD::VECREDUCE_SMAX:
892 case ISD::VECREDUCE_SMIN:
893 case ISD::VECREDUCE_UMAX:
894 case ISD::VECREDUCE_UMIN:
895 case ISD::VECREDUCE_FADD:
896 case ISD::VECREDUCE_FMUL:
897 case ISD::VECREDUCE_FMAX:
898 case ISD::VECREDUCE_FMIN:
899 Results.push_back(TLI.expandVecReduce(Node, DAG));
901 case ISD::VECREDUCE_SEQ_FADD:
902 case ISD::VECREDUCE_SEQ_FMUL:
903 Results.push_back(TLI.expandVecReduceSeq(Node, DAG));
907 ExpandREM(Node, Results);
910 Results.push_back(ExpandVP_MERGE(Node));
914 Results.push_back(DAG.UnrollVectorOp(Node));
917 SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {
918 // Lower a select instruction where the condition is a scalar and the
919 // operands are vectors. Lower this select to VSELECT and implement it
920 // using XOR AND OR. The selector bit is broadcasted.
921 EVT VT = Node->getValueType(0);
924 SDValue Mask = Node->getOperand(0);
925 SDValue Op1 = Node->getOperand(1);
926 SDValue Op2 = Node->getOperand(2);
928 assert(VT.isVector() && !Mask.getValueType().isVector()
929 && Op1.getValueType() == Op2.getValueType() && "Invalid type");
931 // If we can't even use the basic vector operations of
932 // AND,OR,XOR, we will have to scalarize the op.
933 // Notice that the operation may be 'promoted' which means that it is
934 // 'bitcasted' to another type which is handled.
935 // Also, we need to be able to construct a splat vector using either
936 // BUILD_VECTOR or SPLAT_VECTOR.
937 // FIXME: Should we also permit fixed-length SPLAT_VECTOR as a fallback to
939 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
940 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
941 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand ||
942 TLI.getOperationAction(VT.isFixedLengthVector() ? ISD::BUILD_VECTOR
944 VT) == TargetLowering::Expand)
945 return DAG.UnrollVectorOp(Node);
947 // Generate a mask operand.
948 EVT MaskTy = VT.changeVectorElementTypeToInteger();
950 // What is the size of each element in the vector mask.
951 EVT BitTy = MaskTy.getScalarType();
953 Mask = DAG.getSelect(DL, BitTy, Mask, DAG.getAllOnesConstant(DL, BitTy),
954 DAG.getConstant(0, DL, BitTy));
956 // Broadcast the mask so that the entire vector is all one or all zero.
957 if (VT.isFixedLengthVector())
958 Mask = DAG.getSplatBuildVector(MaskTy, DL, Mask);
960 Mask = DAG.getSplatVector(MaskTy, DL, Mask);
962 // Bitcast the operands to be the same type as the mask.
963 // This is needed when we select between FP types because
964 // the mask is a vector of integers.
965 Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1);
966 Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2);
968 SDValue NotMask = DAG.getNOT(DL, Mask, MaskTy);
970 Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask);
971 Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask);
972 SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2);
973 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
976 SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) {
977 EVT VT = Node->getValueType(0);
979 // Make sure that the SRA and SHL instructions are available.
980 if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand ||
981 TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand)
982 return DAG.UnrollVectorOp(Node);
985 EVT OrigTy = cast<VTSDNode>(Node->getOperand(1))->getVT();
987 unsigned BW = VT.getScalarSizeInBits();
988 unsigned OrigBW = OrigTy.getScalarSizeInBits();
989 SDValue ShiftSz = DAG.getConstant(BW - OrigBW, DL, VT);
991 SDValue Op = DAG.getNode(ISD::SHL, DL, VT, Node->getOperand(0), ShiftSz);
992 return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz);
995 // Generically expand a vector anyext in register to a shuffle of the relevant
996 // lanes into the appropriate locations, with other lanes left undef.
997 SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) {
999 EVT VT = Node->getValueType(0);
1000 int NumElements = VT.getVectorNumElements();
1001 SDValue Src = Node->getOperand(0);
1002 EVT SrcVT = Src.getValueType();
1003 int NumSrcElements = SrcVT.getVectorNumElements();
1005 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1006 // into a larger vector type.
1007 if (SrcVT.bitsLE(VT)) {
1008 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1009 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1010 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1011 SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
1013 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT),
1014 Src, DAG.getVectorIdxConstant(0, DL));
1017 // Build a base mask of undef shuffles.
1018 SmallVector<int, 16> ShuffleMask;
1019 ShuffleMask.resize(NumSrcElements, -1);
1021 // Place the extended lanes into the correct locations.
1022 int ExtLaneScale = NumSrcElements / NumElements;
1023 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1024 for (int i = 0; i < NumElements; ++i)
1025 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1028 ISD::BITCAST, DL, VT,
1029 DAG.getVectorShuffle(SrcVT, DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
1032 SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) {
1034 EVT VT = Node->getValueType(0);
1035 SDValue Src = Node->getOperand(0);
1036 EVT SrcVT = Src.getValueType();
1038 // First build an any-extend node which can be legalized above when we
1039 // recurse through it.
1040 SDValue Op = DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Src);
1042 // Now we need sign extend. Do this by shifting the elements. Even if these
1043 // aren't legal operations, they have a better chance of being legalized
1044 // without full scalarization than the sign extension does.
1045 unsigned EltWidth = VT.getScalarSizeInBits();
1046 unsigned SrcEltWidth = SrcVT.getScalarSizeInBits();
1047 SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, DL, VT);
1048 return DAG.getNode(ISD::SRA, DL, VT,
1049 DAG.getNode(ISD::SHL, DL, VT, Op, ShiftAmount),
1053 // Generically expand a vector zext in register to a shuffle of the relevant
1054 // lanes into the appropriate locations, a blend of zero into the high bits,
1055 // and a bitcast to the wider element type.
1056 SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) {
1058 EVT VT = Node->getValueType(0);
1059 int NumElements = VT.getVectorNumElements();
1060 SDValue Src = Node->getOperand(0);
1061 EVT SrcVT = Src.getValueType();
1062 int NumSrcElements = SrcVT.getVectorNumElements();
1064 // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1065 // into a larger vector type.
1066 if (SrcVT.bitsLE(VT)) {
1067 assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1068 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1069 NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1070 SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
1072 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT),
1073 Src, DAG.getVectorIdxConstant(0, DL));
1076 // Build up a zero vector to blend into this one.
1077 SDValue Zero = DAG.getConstant(0, DL, SrcVT);
1079 // Shuffle the incoming lanes into the correct position, and pull all other
1080 // lanes from the zero vector.
1081 auto ShuffleMask = llvm::to_vector<16>(llvm::seq<int>(0, NumSrcElements));
1083 int ExtLaneScale = NumSrcElements / NumElements;
1084 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1085 for (int i = 0; i < NumElements; ++i)
1086 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1088 return DAG.getNode(ISD::BITCAST, DL, VT,
1089 DAG.getVectorShuffle(SrcVT, DL, Zero, Src, ShuffleMask));
1092 static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) {
1093 int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
1094 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)
1095 for (int J = ScalarSizeInBytes - 1; J >= 0; --J)
1096 ShuffleMask.push_back((I * ScalarSizeInBytes) + J);
1099 SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) {
1100 EVT VT = Node->getValueType(0);
1102 // Scalable vectors can't use shuffle expansion.
1103 if (VT.isScalableVector())
1104 return TLI.expandBSWAP(Node, DAG);
1106 // Generate a byte wise shuffle mask for the BSWAP.
1107 SmallVector<int, 16> ShuffleMask;
1108 createBSWAPShuffleMask(VT, ShuffleMask);
1109 EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, ShuffleMask.size());
1111 // Only emit a shuffle if the mask is legal.
1112 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1114 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1115 Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT), ShuffleMask);
1116 return DAG.getNode(ISD::BITCAST, DL, VT, Op);
1119 // If we have the appropriate vector bit operations, it is better to use them
1120 // than unrolling and expanding each component.
1121 if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
1122 TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
1123 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
1124 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT))
1125 return TLI.expandBSWAP(Node, DAG);
1127 // Otherwise unroll.
1128 return DAG.UnrollVectorOp(Node);
1131 void VectorLegalizer::ExpandBITREVERSE(SDNode *Node,
1132 SmallVectorImpl<SDValue> &Results) {
1133 EVT VT = Node->getValueType(0);
1135 // We can't unroll or use shuffles for scalable vectors.
1136 if (VT.isScalableVector()) {
1137 Results.push_back(TLI.expandBITREVERSE(Node, DAG));
1141 // If we have the scalar operation, it's probably cheaper to unroll it.
1142 if (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, VT.getScalarType())) {
1143 SDValue Tmp = DAG.UnrollVectorOp(Node);
1144 Results.push_back(Tmp);
1148 // If the vector element width is a whole number of bytes, test if its legal
1149 // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte
1150 // vector. This greatly reduces the number of bit shifts necessary.
1151 unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
1152 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1153 SmallVector<int, 16> BSWAPMask;
1154 createBSWAPShuffleMask(VT, BSWAPMask);
1156 EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, BSWAPMask.size());
1157 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1158 (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, ByteVT) ||
1159 (TLI.isOperationLegalOrCustom(ISD::SHL, ByteVT) &&
1160 TLI.isOperationLegalOrCustom(ISD::SRL, ByteVT) &&
1161 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, ByteVT) &&
1162 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, ByteVT)))) {
1164 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1165 Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT),
1167 Op = DAG.getNode(ISD::BITREVERSE, DL, ByteVT, Op);
1168 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
1169 Results.push_back(Op);
1174 // If we have the appropriate vector bit operations, it is better to use them
1175 // than unrolling and expanding each component.
1176 if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
1177 TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
1178 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
1179 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT)) {
1180 Results.push_back(TLI.expandBITREVERSE(Node, DAG));
1184 // Otherwise unroll.
1185 SDValue Tmp = DAG.UnrollVectorOp(Node);
1186 Results.push_back(Tmp);
1189 SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
1190 // Implement VSELECT in terms of XOR, AND, OR
1191 // on platforms which do not support blend natively.
1194 SDValue Mask = Node->getOperand(0);
1195 SDValue Op1 = Node->getOperand(1);
1196 SDValue Op2 = Node->getOperand(2);
1198 EVT VT = Mask.getValueType();
1200 // If we can't even use the basic vector operations of
1201 // AND,OR,XOR, we will have to scalarize the op.
1202 // Notice that the operation may be 'promoted' which means that it is
1203 // 'bitcasted' to another type which is handled.
1204 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
1205 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
1206 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand)
1207 return DAG.UnrollVectorOp(Node);
1209 // This operation also isn't safe with AND, OR, XOR when the boolean type is
1210 // 0/1 and the select operands aren't also booleans, as we need an all-ones
1211 // vector constant to mask with.
1212 // FIXME: Sign extend 1 to all ones if that's legal on the target.
1213 auto BoolContents = TLI.getBooleanContents(Op1.getValueType());
1214 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1215 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1216 Op1.getValueType().getVectorElementType() == MVT::i1))
1217 return DAG.UnrollVectorOp(Node);
1219 // If the mask and the type are different sizes, unroll the vector op. This
1220 // can occur when getSetCCResultType returns something that is different in
1221 // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
1222 if (VT.getSizeInBits() != Op1.getValueSizeInBits())
1223 return DAG.UnrollVectorOp(Node);
1225 // Bitcast the operands to be the same type as the mask.
1226 // This is needed when we select between FP types because
1227 // the mask is a vector of integers.
1228 Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1);
1229 Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2);
1231 SDValue NotMask = DAG.getNOT(DL, Mask, VT);
1233 Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask);
1234 Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask);
1235 SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2);
1236 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
1239 SDValue VectorLegalizer::ExpandVP_SELECT(SDNode *Node) {
1240 // Implement VP_SELECT in terms of VP_XOR, VP_AND and VP_OR on platforms which
1241 // do not support it natively.
1244 SDValue Mask = Node->getOperand(0);
1245 SDValue Op1 = Node->getOperand(1);
1246 SDValue Op2 = Node->getOperand(2);
1247 SDValue EVL = Node->getOperand(3);
1249 EVT VT = Mask.getValueType();
1251 // If we can't even use the basic vector operations of
1252 // VP_AND,VP_OR,VP_XOR, we will have to scalarize the op.
1253 if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand ||
1254 TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand ||
1255 TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand)
1256 return DAG.UnrollVectorOp(Node);
1258 // This operation also isn't safe when the operands aren't also booleans.
1259 if (Op1.getValueType().getVectorElementType() != MVT::i1)
1260 return DAG.UnrollVectorOp(Node);
1262 SDValue Ones = DAG.getAllOnesConstant(DL, VT);
1263 SDValue NotMask = DAG.getNode(ISD::VP_XOR, DL, VT, Mask, Ones, Mask, EVL);
1265 Op1 = DAG.getNode(ISD::VP_AND, DL, VT, Op1, Mask, Mask, EVL);
1266 Op2 = DAG.getNode(ISD::VP_AND, DL, VT, Op2, NotMask, Mask, EVL);
1267 return DAG.getNode(ISD::VP_OR, DL, VT, Op1, Op2, Mask, EVL);
1270 SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) {
1271 // Implement VP_MERGE in terms of VSELECT. Construct a mask where vector
1272 // indices less than the EVL/pivot are true. Combine that with the original
1273 // mask for a full-length mask. Use a full-length VSELECT to select between
1274 // the true and false values.
1277 SDValue Mask = Node->getOperand(0);
1278 SDValue Op1 = Node->getOperand(1);
1279 SDValue Op2 = Node->getOperand(2);
1280 SDValue EVL = Node->getOperand(3);
1282 EVT MaskVT = Mask.getValueType();
1283 bool IsFixedLen = MaskVT.isFixedLengthVector();
1285 EVT EVLVecVT = EVT::getVectorVT(*DAG.getContext(), EVL.getValueType(),
1286 MaskVT.getVectorElementCount());
1288 // If we can't construct the EVL mask efficiently, it's better to unroll.
1290 !TLI.isOperationLegalOrCustom(ISD::BUILD_VECTOR, EVLVecVT)) ||
1292 (!TLI.isOperationLegalOrCustom(ISD::STEP_VECTOR, EVLVecVT) ||
1293 !TLI.isOperationLegalOrCustom(ISD::SPLAT_VECTOR, EVLVecVT))))
1294 return DAG.UnrollVectorOp(Node);
1296 // If using a SETCC would result in a different type than the mask type,
1298 if (TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1299 EVLVecVT) != MaskVT)
1300 return DAG.UnrollVectorOp(Node);
1302 SDValue StepVec = DAG.getStepVector(DL, EVLVecVT);
1303 SDValue SplatEVL = IsFixedLen ? DAG.getSplatBuildVector(EVLVecVT, DL, EVL)
1304 : DAG.getSplatVector(EVLVecVT, DL, EVL);
1306 DAG.getSetCC(DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT);
1308 SDValue FullMask = DAG.getNode(ISD::AND, DL, MaskVT, Mask, EVLMask);
1309 return DAG.getSelect(DL, Node->getValueType(0), FullMask, Op1, Op2);
1312 void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,
1313 SmallVectorImpl<SDValue> &Results) {
1314 // Attempt to expand using TargetLowering.
1315 SDValue Result, Chain;
1316 if (TLI.expandFP_TO_UINT(Node, Result, Chain, DAG)) {
1317 Results.push_back(Result);
1318 if (Node->isStrictFPOpcode())
1319 Results.push_back(Chain);
1323 // Otherwise go ahead and unroll.
1324 if (Node->isStrictFPOpcode()) {
1325 UnrollStrictFPOp(Node, Results);
1329 Results.push_back(DAG.UnrollVectorOp(Node));
1332 void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node,
1333 SmallVectorImpl<SDValue> &Results) {
1334 bool IsStrict = Node->isStrictFPOpcode();
1335 unsigned OpNo = IsStrict ? 1 : 0;
1336 SDValue Src = Node->getOperand(OpNo);
1337 EVT VT = Src.getValueType();
1340 // Attempt to expand using TargetLowering.
1343 if (TLI.expandUINT_TO_FP(Node, Result, Chain, DAG)) {
1344 Results.push_back(Result);
1346 Results.push_back(Chain);
1350 // Make sure that the SINT_TO_FP and SRL instructions are available.
1351 if (((!IsStrict && TLI.getOperationAction(ISD::SINT_TO_FP, VT) ==
1352 TargetLowering::Expand) ||
1353 (IsStrict && TLI.getOperationAction(ISD::STRICT_SINT_TO_FP, VT) ==
1354 TargetLowering::Expand)) ||
1355 TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Expand) {
1357 UnrollStrictFPOp(Node, Results);
1361 Results.push_back(DAG.UnrollVectorOp(Node));
1365 unsigned BW = VT.getScalarSizeInBits();
1366 assert((BW == 64 || BW == 32) &&
1367 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1369 SDValue HalfWord = DAG.getConstant(BW / 2, DL, VT);
1371 // Constants to clear the upper part of the word.
1372 // Notice that we can also use SHL+SHR, but using a constant is slightly
1374 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1375 SDValue HalfWordMask = DAG.getConstant(HWMask, DL, VT);
1377 // Two to the power of half-word-size.
1379 DAG.getConstantFP(1ULL << (BW / 2), DL, Node->getValueType(0));
1381 // Clear upper part of LO, lower HI
1382 SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Src, HalfWord);
1383 SDValue LO = DAG.getNode(ISD::AND, DL, VT, Src, HalfWordMask);
1386 // Convert hi and lo to floats
1387 // Convert the hi part back to the upper values
1388 // TODO: Can any fast-math-flags be set on these nodes?
1389 SDValue fHI = DAG.getNode(ISD::STRICT_SINT_TO_FP, DL,
1390 {Node->getValueType(0), MVT::Other},
1391 {Node->getOperand(0), HI});
1392 fHI = DAG.getNode(ISD::STRICT_FMUL, DL, {Node->getValueType(0), MVT::Other},
1393 {fHI.getValue(1), fHI, TWOHW});
1394 SDValue fLO = DAG.getNode(ISD::STRICT_SINT_TO_FP, DL,
1395 {Node->getValueType(0), MVT::Other},
1396 {Node->getOperand(0), LO});
1398 SDValue TF = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, fHI.getValue(1),
1401 // Add the two halves
1403 DAG.getNode(ISD::STRICT_FADD, DL, {Node->getValueType(0), MVT::Other},
1406 Results.push_back(Result);
1407 Results.push_back(Result.getValue(1));
1411 // Convert hi and lo to floats
1412 // Convert the hi part back to the upper values
1413 // TODO: Can any fast-math-flags be set on these nodes?
1414 SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), HI);
1415 fHI = DAG.getNode(ISD::FMUL, DL, Node->getValueType(0), fHI, TWOHW);
1416 SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), LO);
1418 // Add the two halves
1420 DAG.getNode(ISD::FADD, DL, Node->getValueType(0), fHI, fLO));
1423 SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) {
1424 if (TLI.isOperationLegalOrCustom(ISD::FSUB, Node->getValueType(0))) {
1426 SDValue Zero = DAG.getConstantFP(-0.0, DL, Node->getValueType(0));
1427 // TODO: If FNEG had fast-math-flags, they'd get propagated to this FSUB.
1428 return DAG.getNode(ISD::FSUB, DL, Node->getValueType(0), Zero,
1429 Node->getOperand(0));
1431 return DAG.UnrollVectorOp(Node);
1434 void VectorLegalizer::ExpandFSUB(SDNode *Node,
1435 SmallVectorImpl<SDValue> &Results) {
1436 // For floating-point values, (a-b) is the same as a+(-b). If FNEG is legal,
1437 // we can defer this to operation legalization where it will be lowered as
1439 EVT VT = Node->getValueType(0);
1440 if (TLI.isOperationLegalOrCustom(ISD::FNEG, VT) &&
1441 TLI.isOperationLegalOrCustom(ISD::FADD, VT))
1442 return; // Defer to LegalizeDAG
1444 SDValue Tmp = DAG.UnrollVectorOp(Node);
1445 Results.push_back(Tmp);
1448 void VectorLegalizer::ExpandSETCC(SDNode *Node,
1449 SmallVectorImpl<SDValue> &Results) {
1450 bool NeedInvert = false;
1451 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
1453 MVT OpVT = Node->getOperand(0).getSimpleValueType();
1454 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
1456 if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
1457 Results.push_back(UnrollVSETCC(Node));
1462 SDValue LHS = Node->getOperand(0);
1463 SDValue RHS = Node->getOperand(1);
1464 SDValue CC = Node->getOperand(2);
1467 Mask = Node->getOperand(3);
1468 EVL = Node->getOperand(4);
1472 TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), LHS, RHS, CC, Mask,
1473 EVL, NeedInvert, dl, Chain);
1476 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
1477 // condition code, create a new SETCC node.
1480 LHS = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), LHS, RHS, CC,
1483 LHS = DAG.getNode(ISD::VP_SETCC, dl, Node->getValueType(0),
1484 {LHS, RHS, CC, Mask, EVL}, Node->getFlags());
1487 // If we expanded the SETCC by inverting the condition code, then wrap
1488 // the existing SETCC in a NOT to restore the intended condition.
1491 LHS = DAG.getLogicalNOT(dl, LHS, LHS->getValueType(0));
1493 LHS = DAG.getVPLogicalNOT(dl, LHS, Mask, EVL, LHS->getValueType(0));
1496 // Otherwise, SETCC for the given comparison type must be completely
1497 // illegal; expand it into a SELECT_CC.
1498 EVT VT = Node->getValueType(0);
1500 DAG.getNode(ISD::SELECT_CC, dl, VT, LHS, RHS,
1501 DAG.getBoolConstant(true, dl, VT, LHS.getValueType()),
1502 DAG.getBoolConstant(false, dl, VT, LHS.getValueType()), CC);
1503 LHS->setFlags(Node->getFlags());
1506 Results.push_back(LHS);
1509 void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
1510 SmallVectorImpl<SDValue> &Results) {
1511 SDValue Result, Overflow;
1512 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
1513 Results.push_back(Result);
1514 Results.push_back(Overflow);
1517 void VectorLegalizer::ExpandSADDSUBO(SDNode *Node,
1518 SmallVectorImpl<SDValue> &Results) {
1519 SDValue Result, Overflow;
1520 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
1521 Results.push_back(Result);
1522 Results.push_back(Overflow);
1525 void VectorLegalizer::ExpandMULO(SDNode *Node,
1526 SmallVectorImpl<SDValue> &Results) {
1527 SDValue Result, Overflow;
1528 if (!TLI.expandMULO(Node, Result, Overflow, DAG))
1529 std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(Node);
1531 Results.push_back(Result);
1532 Results.push_back(Overflow);
1535 void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node,
1536 SmallVectorImpl<SDValue> &Results) {
1538 if (SDValue Expanded = TLI.expandFixedPointDiv(N->getOpcode(), SDLoc(N),
1539 N->getOperand(0), N->getOperand(1), N->getConstantOperandVal(2), DAG))
1540 Results.push_back(Expanded);
1543 void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
1544 SmallVectorImpl<SDValue> &Results) {
1545 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) {
1546 ExpandUINT_TO_FLOAT(Node, Results);
1549 if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) {
1550 ExpandFP_TO_UINT(Node, Results);
1554 UnrollStrictFPOp(Node, Results);
1557 void VectorLegalizer::ExpandREM(SDNode *Node,
1558 SmallVectorImpl<SDValue> &Results) {
1559 assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) &&
1560 "Expected REM node");
1563 if (!TLI.expandREM(Node, Result, DAG))
1564 Result = DAG.UnrollVectorOp(Node);
1565 Results.push_back(Result);
1568 void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,
1569 SmallVectorImpl<SDValue> &Results) {
1570 EVT VT = Node->getValueType(0);
1571 EVT EltVT = VT.getVectorElementType();
1572 unsigned NumElems = VT.getVectorNumElements();
1573 unsigned NumOpers = Node->getNumOperands();
1574 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1576 EVT TmpEltVT = EltVT;
1577 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1578 Node->getOpcode() == ISD::STRICT_FSETCCS)
1579 TmpEltVT = TLI.getSetCCResultType(DAG.getDataLayout(),
1580 *DAG.getContext(), TmpEltVT);
1582 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
1583 SDValue Chain = Node->getOperand(0);
1586 SmallVector<SDValue, 32> OpValues;
1587 SmallVector<SDValue, 32> OpChains;
1588 for (unsigned i = 0; i < NumElems; ++i) {
1589 SmallVector<SDValue, 4> Opers;
1590 SDValue Idx = DAG.getVectorIdxConstant(i, dl);
1592 // The Chain is the first operand.
1593 Opers.push_back(Chain);
1595 // Now process the remaining operands.
1596 for (unsigned j = 1; j < NumOpers; ++j) {
1597 SDValue Oper = Node->getOperand(j);
1598 EVT OperVT = Oper.getValueType();
1600 if (OperVT.isVector())
1601 Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
1602 OperVT.getVectorElementType(), Oper, Idx);
1604 Opers.push_back(Oper);
1607 SDValue ScalarOp = DAG.getNode(Node->getOpcode(), dl, ValueVTs, Opers);
1608 SDValue ScalarResult = ScalarOp.getValue(0);
1609 SDValue ScalarChain = ScalarOp.getValue(1);
1611 if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1612 Node->getOpcode() == ISD::STRICT_FSETCCS)
1613 ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
1614 DAG.getAllOnesConstant(dl, EltVT),
1615 DAG.getConstant(0, dl, EltVT));
1617 OpValues.push_back(ScalarResult);
1618 OpChains.push_back(ScalarChain);
1621 SDValue Result = DAG.getBuildVector(VT, dl, OpValues);
1622 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
1624 Results.push_back(Result);
1625 Results.push_back(NewChain);
1628 SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) {
1629 EVT VT = Node->getValueType(0);
1630 unsigned NumElems = VT.getVectorNumElements();
1631 EVT EltVT = VT.getVectorElementType();
1632 SDValue LHS = Node->getOperand(0);
1633 SDValue RHS = Node->getOperand(1);
1634 SDValue CC = Node->getOperand(2);
1635 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
1637 SmallVector<SDValue, 8> Ops(NumElems);
1638 for (unsigned i = 0; i < NumElems; ++i) {
1639 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
1640 DAG.getVectorIdxConstant(i, dl));
1641 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
1642 DAG.getVectorIdxConstant(i, dl));
1643 Ops[i] = DAG.getNode(ISD::SETCC, dl,
1644 TLI.getSetCCResultType(DAG.getDataLayout(),
1645 *DAG.getContext(), TmpEltVT),
1646 LHSElem, RHSElem, CC);
1647 Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], DAG.getAllOnesConstant(dl, EltVT),
1648 DAG.getConstant(0, dl, EltVT));
1650 return DAG.getBuildVector(VT, dl, Ops);
1653 bool SelectionDAG::LegalizeVectors() {
1654 return VectorLegalizer(*this).Run();