return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
}
+// Gets the two common "VL" operands: an all-ones mask and the vector length.
+// VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
+// the vector type that it is contained in.
+static std::pair<SDValue, SDValue>
+getDefaultVLOps(MVT VecVT, MVT ContainerVT, SDLoc DL, SelectionDAG &DAG,
+ const RISCVSubtarget &Subtarget) {
+ assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
+ MVT XLenVT = Subtarget.getXLenVT();
+ SDValue VL = VecVT.isFixedLengthVector()
+ ? DAG.getConstant(VecVT.getVectorNumElements(), DL, XLenVT)
+ : DAG.getRegister(RISCV::X0, XLenVT);
+ MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
+ SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
+ return {Mask, VL};
+}
+
+// As above but assuming the given type is a scalable vector type.
+static std::pair<SDValue, SDValue>
+getDefaultScalableVLOps(MVT VecVT, SDLoc DL, SelectionDAG &DAG,
+ const RISCVSubtarget &Subtarget) {
+ assert(VecVT.isScalableVector() && "Expecting a scalable vector");
+ return getDefaultVLOps(VecVT, VecVT, DL, DAG, Subtarget);
+}
+
static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
const RISCVSubtarget &Subtarget) {
MVT VT = Op.getSimpleValueType();
MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
SDLoc DL(Op);
- SDValue VL =
- DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
+ SDValue Mask, VL;
+ std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
if (VT.getVectorElementType() == MVT::i1) {
if (ISD::isBuildVectorAllZeros(Op.getNode())) {
Op.getConstantOperandVal(i) == i);
if (IsVID) {
- MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
- SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, ContainerVT, Mask, VL);
return convertFromScalableVector(VT, VID, DAG, Subtarget);
}
V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
assert(Lane < (int)VT.getVectorNumElements() && "Unexpected lane!");
+ SDValue Mask, VL;
+ std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
MVT XLenVT = Subtarget.getXLenVT();
- SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
- MVT MaskVT =
- MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
- SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
SDValue Gather =
DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, V1,
DAG.getConstant(Lane, DL, XLenVT), Mask, VL);
SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
- EVT VecVT = Op.getValueType();
+ MVT VecVT = Op.getSimpleValueType();
SDValue Vec = Op.getOperand(0);
SDValue Val = Op.getOperand(1);
SDValue Idx = Op.getOperand(2);
if (Subtarget.is64Bit() || VecVT.getVectorElementType() != MVT::i64) {
if (isNullConstant(Idx))
return Op;
- SDValue Slidedown = DAG.getNode(RISCVISD::VSLIDEDOWN, DL, VecVT,
- DAG.getUNDEF(VecVT), Vec, Idx);
+ SDValue Mask, VL;
+ std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
+ SDValue Slidedown = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VecVT,
+ DAG.getUNDEF(VecVT), Vec, Idx, Mask, VL);
SDValue InsertElt0 =
DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecVT, Slidedown, Val,
DAG.getConstant(0, DL, Subtarget.getXLenVT()));
- return DAG.getNode(RISCVISD::VSLIDEUP, DL, VecVT, Vec, InsertElt0, Idx);
+ return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, VecVT, Vec, InsertElt0, Idx,
+ Mask, VL);
}
// Custom-legalize INSERT_VECTOR_ELT where XLEN<SEW, as the SEW element type
SDValue SplattedVal = DAG.getSplatVector(VecVT, DL, Val);
SDValue SplattedIdx = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, Idx);
- SDValue VL = DAG.getRegister(RISCV::X0, Subtarget.getXLenVT());
- MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
- SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
+ SDValue Mask, VL;
+ std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VecVT, Mask, VL);
auto SetCCVT =
getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VecVT);
SDValue Idx = Op.getOperand(1);
SDValue Vec = Op.getOperand(0);
EVT EltVT = Op.getValueType();
- EVT VecVT = Vec.getValueType();
+ MVT VecVT = Vec.getSimpleValueType();
MVT XLenVT = Subtarget.getXLenVT();
// If the index is 0, the vector is already in the right position.
if (!isNullConstant(Idx)) {
- Vec = DAG.getNode(RISCVISD::VSLIDEDOWN, DL, VecVT, DAG.getUNDEF(VecVT), Vec,
- Idx);
+ SDValue Mask, VL;
+ std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
+ Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VecVT, DAG.getUNDEF(VecVT),
+ Vec, Idx, Mask, VL);
}
if (!EltVT.isInteger()) {
}
SDLoc DL(Op);
- SDValue VL =
- DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
- MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
- SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
+ SDValue Mask, VL;
+ std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
Ops.push_back(Mask);
Ops.push_back(VL);
SDLoc DL(N);
SDValue Vec = N->getOperand(0);
SDValue Idx = N->getOperand(1);
- EVT VecVT = Vec.getValueType();
+ MVT VecVT = Vec.getSimpleValueType();
assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 &&
VecVT.getVectorElementType() == MVT::i64 &&
"Unexpected EXTRACT_VECTOR_ELT legalization");
SDValue Slidedown = Vec;
+ MVT XLenVT = Subtarget.getXLenVT();
// Unless the index is known to be 0, we must slide the vector down to get
// the desired element into index 0.
- if (!isNullConstant(Idx))
- Slidedown = DAG.getNode(RISCVISD::VSLIDEDOWN, DL, VecVT,
- DAG.getUNDEF(VecVT), Vec, Idx);
+ if (!isNullConstant(Idx)) {
+ SDValue Mask, VL;
+ std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
+ Slidedown = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VecVT,
+ DAG.getUNDEF(VecVT), Vec, Idx, Mask, VL);
+ }
- MVT XLenVT = Subtarget.getXLenVT();
// Extract the lower XLEN bits of the correct vector element.
SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Slidedown, Idx);
NODE_NAME_CASE(TRUNCATE_VECTOR)
NODE_NAME_CASE(VLEFF)
NODE_NAME_CASE(VLEFF_MASK)
- NODE_NAME_CASE(VSLIDEUP)
- NODE_NAME_CASE(VSLIDEDOWN)
+ NODE_NAME_CASE(VSLIDEUP_VL)
+ NODE_NAME_CASE(VSLIDEDOWN_VL)
NODE_NAME_CASE(VID_VL)
NODE_NAME_CASE(VFNCVT_ROD)
NODE_NAME_CASE(VECREDUCE_ADD)
(vti.Scalar vti.ScalarRegClass:$rs1),
vti.AVL, vti.SEW)>;
}
-
-def SDTRVVSlide : SDTypeProfile<1, 3, [
- SDTCisVec<0>, SDTCisSameAs<1, 0>, SDTCisSameAs<2, 0>, SDTCisVT<3, XLenVT>
-]>;
-
-def riscv_slideup : SDNode<"RISCVISD::VSLIDEUP", SDTRVVSlide, []>;
-def riscv_slidedown : SDNode<"RISCVISD::VSLIDEDOWN", SDTRVVSlide, []>;
-
-let Predicates = [HasStdExtV] in {
-
-foreach vti = !listconcat(AllIntegerVectors, AllFloatVectors) in {
- def : Pat<(vti.Vector (riscv_slideup (vti.Vector vti.RegClass:$rs3),
- (vti.Vector vti.RegClass:$rs1),
- uimm5:$rs2)),
- (!cast<Instruction>("PseudoVSLIDEUP_VI_"#vti.LMul.MX)
- vti.RegClass:$rs3, vti.RegClass:$rs1, uimm5:$rs2,
- vti.AVL, vti.SEW)>;
-
- def : Pat<(vti.Vector (riscv_slideup (vti.Vector vti.RegClass:$rs3),
- (vti.Vector vti.RegClass:$rs1),
- GPR:$rs2)),
- (!cast<Instruction>("PseudoVSLIDEUP_VX_"#vti.LMul.MX)
- vti.RegClass:$rs3, vti.RegClass:$rs1, GPR:$rs2,
- vti.AVL, vti.SEW)>;
-
- def : Pat<(vti.Vector (riscv_slidedown (vti.Vector vti.RegClass:$rs3),
- (vti.Vector vti.RegClass:$rs1),
- uimm5:$rs2)),
- (!cast<Instruction>("PseudoVSLIDEDOWN_VI_"#vti.LMul.MX)
- vti.RegClass:$rs3, vti.RegClass:$rs1, uimm5:$rs2,
- vti.AVL, vti.SEW)>;
-
- def : Pat<(vti.Vector (riscv_slidedown (vti.Vector vti.RegClass:$rs3),
- (vti.Vector vti.RegClass:$rs1),
- GPR:$rs2)),
- (!cast<Instruction>("PseudoVSLIDEDOWN_VX_"#vti.LMul.MX)
- vti.RegClass:$rs3, vti.RegClass:$rs1, GPR:$rs2,
- vti.AVL, vti.SEW)>;
-}
-} // Predicates = [HasStdExtV]
[SDTCisVec<0>, SDTCisVec<1>, SDTCVecEltisVT<1, i1>,
SDTCisSameNumEltsAs<0, 1>, SDTCisVT<2, XLenVT>]>, []>;
+def SDTRVVSlide : SDTypeProfile<1, 5, [
+ SDTCisVec<0>, SDTCisSameAs<1, 0>, SDTCisSameAs<2, 0>, SDTCisVT<3, XLenVT>,
+ SDTCVecEltisVT<4, i1>, SDTCisSameNumEltsAs<0, 4>, SDTCisVT<5, XLenVT>
+]>;
+
+def riscv_slideup_vl : SDNode<"RISCVISD::VSLIDEUP_VL", SDTRVVSlide, []>;
+def riscv_slidedown_vl : SDNode<"RISCVISD::VSLIDEDOWN_VL", SDTRVVSlide, []>;
+
let Predicates = [HasStdExtV] in {
foreach vti = AllIntegerVectors in
(XLenVT (VLOp GPR:$vl)))),
(!cast<Instruction>("PseudoVID_V_"#vti.LMul.MX) GPR:$vl, vti.SEW)>;
+foreach vti = !listconcat(AllIntegerVectors, AllFloatVectors) in {
+ def : Pat<(vti.Vector (riscv_slideup_vl (vti.Vector vti.RegClass:$rs3),
+ (vti.Vector vti.RegClass:$rs1),
+ uimm5:$rs2, (vti.Mask true_mask),
+ (XLenVT (VLOp GPR:$vl)))),
+ (!cast<Instruction>("PseudoVSLIDEUP_VI_"#vti.LMul.MX)
+ vti.RegClass:$rs3, vti.RegClass:$rs1, uimm5:$rs2,
+ GPR:$vl, vti.SEW)>;
+
+ def : Pat<(vti.Vector (riscv_slideup_vl (vti.Vector vti.RegClass:$rs3),
+ (vti.Vector vti.RegClass:$rs1),
+ GPR:$rs2, (vti.Mask true_mask),
+ (XLenVT (VLOp GPR:$vl)))),
+ (!cast<Instruction>("PseudoVSLIDEUP_VX_"#vti.LMul.MX)
+ vti.RegClass:$rs3, vti.RegClass:$rs1, GPR:$rs2,
+ GPR:$vl, vti.SEW)>;
+
+ def : Pat<(vti.Vector (riscv_slidedown_vl (vti.Vector vti.RegClass:$rs3),
+ (vti.Vector vti.RegClass:$rs1),
+ uimm5:$rs2, (vti.Mask true_mask),
+ (XLenVT (VLOp GPR:$vl)))),
+ (!cast<Instruction>("PseudoVSLIDEDOWN_VI_"#vti.LMul.MX)
+ vti.RegClass:$rs3, vti.RegClass:$rs1, uimm5:$rs2,
+ GPR:$vl, vti.SEW)>;
+
+ def : Pat<(vti.Vector (riscv_slidedown_vl (vti.Vector vti.RegClass:$rs3),
+ (vti.Vector vti.RegClass:$rs1),
+ GPR:$rs2, (vti.Mask true_mask),
+ (XLenVT (VLOp GPR:$vl)))),
+ (!cast<Instruction>("PseudoVSLIDEDOWN_VX_"#vti.LMul.MX)
+ vti.RegClass:$rs3, vti.RegClass:$rs1, GPR:$rs2,
+ GPR:$vl, vti.SEW)>;
+}
+
} // Predicates = [HasStdExtV]