Define vloxseg/vluxseg intrinsics and pseudo instructions.
Lower vloxseg/vluxseg intrinsics to pseudo instructions in RISCVDAGToDAGISel.
Differential Revision: https://reviews.llvm.org/D94903
LLVMMatchType<1>]),
[NoCapture<ArgIndex<nf>>, IntrReadMem]>, RISCVVIntrinsic;
+ // For indexed segment load
+ // Input: (pointer, index, vl)
+ class RISCVISegLoad<int nf>
+ : Intrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>,
+ !add(nf, -1))),
+ [LLVMPointerToElt<0>, llvm_anyvector_ty, llvm_anyint_ty],
+ [NoCapture<ArgIndex<0>>, IntrReadMem]>, RISCVVIntrinsic;
+ // For indexed segment load with mask
+ // Input: (maskedoff, pointer, index, mask, vl)
+ class RISCVISegLoadMask<int nf>
+ : Intrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>,
+ !add(nf, -1))),
+ !listconcat(!listsplat(LLVMMatchType<0>, nf),
+ [LLVMPointerToElt<0>,
+ llvm_anyvector_ty,
+ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+ llvm_anyint_ty]),
+ [NoCapture<ArgIndex<nf>>, IntrReadMem]>, RISCVVIntrinsic;
+
// For unit stride segment store
// Input: (value, pointer, vl)
class RISCVUSSegStore<int nf>
def "int_riscv_" # NAME : RISCVSSegLoad<nf>;
def "int_riscv_" # NAME # "_mask" : RISCVSSegLoadMask<nf>;
}
+ multiclass RISCVISegLoad<int nf> {
+ def "int_riscv_" # NAME : RISCVISegLoad<nf>;
+ def "int_riscv_" # NAME # "_mask" : RISCVISegLoadMask<nf>;
+ }
multiclass RISCVUSSegStore<int nf> {
def "int_riscv_" # NAME : RISCVUSSegStore<nf>;
def "int_riscv_" # NAME # "_mask" : RISCVUSSegStoreMask<nf>;
foreach nf = [2, 3, 4, 5, 6, 7, 8] in {
defm vlseg # nf : RISCVUSSegLoad<nf>;
defm vlsseg # nf : RISCVSSegLoad<nf>;
+ defm vloxseg # nf : RISCVISegLoad<nf>;
+ defm vluxseg # nf : RISCVISegLoad<nf>;
defm vsseg # nf : RISCVUSSegStore<nf>;
defm vssseg # nf : RISCVSSegStore<nf>;
}
Operands.push_back(SEW);
Operands.push_back(Node->getOperand(0)); // Chain.
const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
- IntNo, ScalarSize, static_cast<unsigned>(LMUL));
+ IntNo, ScalarSize, static_cast<unsigned>(LMUL),
+ static_cast<unsigned>(RISCVVLMUL::LMUL_1));
SDNode *Load =
CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
SDValue SuperReg = SDValue(Load, 0);
Operands.push_back(SEW);
Operands.push_back(Node->getOperand(0)); /// Chain.
const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
- IntNo, ScalarSize, static_cast<unsigned>(LMUL));
+ IntNo, ScalarSize, static_cast<unsigned>(LMUL),
+ static_cast<unsigned>(RISCVVLMUL::LMUL_1));
+ SDNode *Load =
+ CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
+ SDValue SuperReg = SDValue(Load, 0);
+ for (unsigned I = 0; I < NF; ++I)
+ ReplaceUses(SDValue(Node, I),
+ CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
+ VT, SuperReg));
+
+ ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
+ CurDAG->RemoveDeadNode(Node);
+}
+
+void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, unsigned IntNo) {
+ SDLoc DL(Node);
+ unsigned NF = Node->getNumValues() - 1;
+ EVT VT = Node->getValueType(0);
+ unsigned ScalarSize = VT.getScalarSizeInBits();
+ MVT XLenVT = Subtarget->getXLenVT();
+ RISCVVLMUL LMUL = getLMUL(VT);
+ SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
+ SDValue Operands[] = {
+ Node->getOperand(2), // Base pointer.
+ Node->getOperand(3), // Index.
+ Node->getOperand(4), // VL.
+ SEW, Node->getOperand(0) // Chain.
+ };
+
+ EVT IndexVT = Node->getOperand(3)->getValueType(0);
+ RISCVVLMUL IndexLMUL = getLMUL(IndexVT);
+ unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
+ const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
+ IntNo, IndexScalarSize, static_cast<unsigned>(LMUL),
+ static_cast<unsigned>(IndexLMUL));
+ SDNode *Load =
+ CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
+ SDValue SuperReg = SDValue(Load, 0);
+ for (unsigned I = 0; I < NF; ++I)
+ ReplaceUses(SDValue(Node, I),
+ CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
+ VT, SuperReg));
+
+ ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
+ CurDAG->RemoveDeadNode(Node);
+}
+
+void RISCVDAGToDAGISel::selectVLXSEGMask(SDNode *Node, unsigned IntNo) {
+ SDLoc DL(Node);
+ unsigned NF = Node->getNumValues() - 1;
+ EVT VT = Node->getValueType(0);
+ unsigned ScalarSize = VT.getScalarSizeInBits();
+ MVT XLenVT = Subtarget->getXLenVT();
+ RISCVVLMUL LMUL = getLMUL(VT);
+ SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
+ SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
+ SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
+ SDValue Operands[] = {
+ MaskedOff,
+ Node->getOperand(NF + 2), // Base pointer.
+ Node->getOperand(NF + 3), // Index.
+ Node->getOperand(NF + 4), // Mask.
+ Node->getOperand(NF + 5), // VL.
+ SEW,
+ Node->getOperand(0) // Chain.
+ };
+
+ EVT IndexVT = Node->getOperand(NF + 3)->getValueType(0);
+ RISCVVLMUL IndexLMUL = getLMUL(IndexVT);
+ unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
+ const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
+ IntNo, IndexScalarSize, static_cast<unsigned>(LMUL),
+ static_cast<unsigned>(IndexLMUL));
SDNode *Load =
CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
SDValue SuperReg = SDValue(Load, 0);
Operands.push_back(SEW);
Operands.push_back(Node->getOperand(0)); // Chain.
const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
- IntNo, ScalarSize, static_cast<unsigned>(LMUL));
+ IntNo, ScalarSize, static_cast<unsigned>(LMUL),
+ static_cast<unsigned>(RISCVVLMUL::LMUL_1));
SDNode *Store =
CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
ReplaceNode(Node, Store);
Operands.push_back(SEW);
Operands.push_back(Node->getOperand(0)); // Chain.
const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
- IntNo, ScalarSize, static_cast<unsigned>(LMUL));
+ IntNo, ScalarSize, static_cast<unsigned>(LMUL),
+ static_cast<unsigned>(RISCVVLMUL::LMUL_1));
SDNode *Store =
CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
ReplaceNode(Node, Store);
selectVLSEGMask(Node, IntNo, /*IsStrided=*/true);
return;
}
+ case Intrinsic::riscv_vloxseg2:
+ case Intrinsic::riscv_vloxseg3:
+ case Intrinsic::riscv_vloxseg4:
+ case Intrinsic::riscv_vloxseg5:
+ case Intrinsic::riscv_vloxseg6:
+ case Intrinsic::riscv_vloxseg7:
+ case Intrinsic::riscv_vloxseg8:
+ case Intrinsic::riscv_vluxseg2:
+ case Intrinsic::riscv_vluxseg3:
+ case Intrinsic::riscv_vluxseg4:
+ case Intrinsic::riscv_vluxseg5:
+ case Intrinsic::riscv_vluxseg6:
+ case Intrinsic::riscv_vluxseg7:
+ case Intrinsic::riscv_vluxseg8: {
+ selectVLXSEG(Node, IntNo);
+ return;
+ }
+ case Intrinsic::riscv_vloxseg2_mask:
+ case Intrinsic::riscv_vloxseg3_mask:
+ case Intrinsic::riscv_vloxseg4_mask:
+ case Intrinsic::riscv_vloxseg5_mask:
+ case Intrinsic::riscv_vloxseg6_mask:
+ case Intrinsic::riscv_vloxseg7_mask:
+ case Intrinsic::riscv_vloxseg8_mask:
+ case Intrinsic::riscv_vluxseg2_mask:
+ case Intrinsic::riscv_vluxseg3_mask:
+ case Intrinsic::riscv_vluxseg4_mask:
+ case Intrinsic::riscv_vluxseg5_mask:
+ case Intrinsic::riscv_vluxseg6_mask:
+ case Intrinsic::riscv_vluxseg7_mask:
+ case Intrinsic::riscv_vluxseg8_mask: {
+ selectVLXSEGMask(Node, IntNo);
+ return;
+ }
}
break;
}
void selectVLSEG(SDNode *Node, unsigned IntNo, bool IsStrided);
void selectVLSEGMask(SDNode *Node, unsigned IntNo, bool IsStrided);
+ void selectVLXSEG(SDNode *Node, unsigned IntNo);
+ void selectVLXSEGMask(SDNode *Node, unsigned IntNo);
void selectVSSEG(SDNode *Node, unsigned IntNo, bool IsStrided);
void selectVSSEGMask(SDNode *Node, unsigned IntNo, bool IsStrided);
unsigned int IntrinsicID;
unsigned int SEW;
unsigned int LMUL;
+ unsigned int IndexLMUL;
unsigned int Pseudo;
};
let PrimaryKeyName = "getRISCVVIntrinsicInfo";
}
-class RISCVZvlsseg<string IntrName, bits<11> S, bits<3> L> {
+class RISCVZvlsseg<string IntrName, bits<11> S, bits<3> L, bits<3> IL = V_M1.value> {
Intrinsic IntrinsicID = !cast<Intrinsic>(IntrName);
bits<11> SEW = S;
bits<3> LMUL = L;
+ bits<3> IndexLMUL = IL;
Pseudo Pseudo = !cast<Pseudo>(NAME);
}
def RISCVZvlssegTable : GenericTable {
let FilterClass = "RISCVZvlsseg";
- let Fields = ["IntrinsicID", "SEW", "LMUL", "Pseudo"];
- let PrimaryKey = ["IntrinsicID", "SEW", "LMUL"];
+ let Fields = ["IntrinsicID", "SEW", "LMUL", "IndexLMUL", "Pseudo"];
+ let PrimaryKey = ["IntrinsicID", "SEW", "LMUL", "IndexLMUL"];
let PrimaryKeyName = "getPseudo";
}
string L = !subst("VLSEG", "vlseg",
!subst("VLSSEG", "vlsseg",
!subst("VSSEG", "vsseg",
- !subst("VSSSEG", "vssseg", Upper))));
+ !subst("VSSSEG", "vssseg",
+ !subst("VLOXSEG", "vloxseg",
+ !subst("VLUXSEG", "vluxseg", Upper))))));
}
// Example: PseudoVLSEG2E32_V_M2 -> int_riscv_vlseg2
!subst("E16", "",
!subst("E32", "",
!subst("E64", "",
- !subst("_V", "", PseudoToVInst<PseudoInst>.VInst)))))>.L,
+ !subst("EI8", "",
+ !subst("EI16", "",
+ !subst("EI32", "",
+ !subst("EI64", "",
+ !subst("_V", "", PseudoToVInst<PseudoInst>.VInst)))))))))>.L,
!if(IsMasked, "_mask", ""));
}
let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
}
+class VPseudoISegLoadNoMask<VReg RetClass, VReg IdxClass, bits<11> EEW, bits<3> LMUL>:
+ Pseudo<(outs RetClass:$rd),
+ (ins GPR:$rs1, IdxClass:$offset, GPR:$vl, ixlenimm:$sew),[]>,
+ RISCVVPseudo,
+ RISCVZvlsseg<PseudoToIntrinsic<NAME, false>.Intrinsic, EEW, VLMul, LMUL> {
+ let mayLoad = 1;
+ let mayStore = 0;
+ let hasSideEffects = 0;
+ let usesCustomInserter = 1;
+ // For vector indexed segment loads, the destination vector register groups
+ // cannot overlap the source vector register group
+ let Constraints = "@earlyclobber $rd";
+ let Uses = [VL, VTYPE];
+ let HasVLOp = 1;
+ let HasSEWOp = 1;
+ let HasDummyMask = 1;
+ let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
+}
+
+class VPseudoISegLoadMask<VReg RetClass, VReg IdxClass, bits<11> EEW, bits<3> LMUL>:
+ Pseudo<(outs GetVRegNoV0<RetClass>.R:$rd),
+ (ins GetVRegNoV0<RetClass>.R:$merge, GPR:$rs1,
+ IdxClass:$offset, VMaskOp:$vm, GPR:$vl, ixlenimm:$sew),[]>,
+ RISCVVPseudo,
+ RISCVZvlsseg<PseudoToIntrinsic<NAME, true>.Intrinsic, EEW, VLMul, LMUL> {
+ let mayLoad = 1;
+ let mayStore = 0;
+ let hasSideEffects = 0;
+ let usesCustomInserter = 1;
+ // For vector indexed segment loads, the destination vector register groups
+ // cannot overlap the source vector register group
+ let Constraints = "@earlyclobber $rd, $rd = $merge";
+ let Uses = [VL, VTYPE];
+ let HasVLOp = 1;
+ let HasSEWOp = 1;
+ let HasMergeOp = 1;
+ let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
+}
+
class VPseudoUSSegStoreNoMask<VReg ValClass, bits<11> EEW>:
Pseudo<(outs),
(ins ValClass:$rd, GPR:$rs1, GPR:$vl, ixlenimm:$sew),[]>,
}
}
+multiclass VPseudoISegLoad {
+ foreach idx_eew = EEWList in { // EEW for index argument.
+ foreach idx_lmul = MxSet<idx_eew>.m in { // LMUL for index argument.
+ foreach val_lmul = MxList.m in { // LMUL for the value.
+ defvar IdxLInfo = idx_lmul.MX;
+ defvar IdxVreg = idx_lmul.vrclass;
+ defvar ValLInfo = val_lmul.MX;
+ let VLMul = val_lmul.value in {
+ foreach nf = NFSet<val_lmul>.L in {
+ defvar ValVreg = SegRegClass<val_lmul, nf>.RC;
+ def nf # "EI" # idx_eew # "_V_" # IdxLInfo # "_" # ValLInfo :
+ VPseudoISegLoadNoMask<ValVreg, IdxVreg, idx_eew, idx_lmul.value>;
+ def nf # "EI" # idx_eew # "_V_" # IdxLInfo # "_" # ValLInfo # "_MASK" :
+ VPseudoISegLoadMask<ValVreg, IdxVreg, idx_eew, idx_lmul.value>;
+ }
+ }
+ }
+ }
+ }
+}
+
multiclass VPseudoUSSegStore {
foreach eew = EEWList in {
foreach lmul = MxSet<eew>.m in {
//===----------------------------------------------------------------------===//
defm PseudoVLSEG : VPseudoUSSegLoad;
defm PseudoVLSSEG : VPseudoSSegLoad;
+defm PseudoVLOXSEG : VPseudoISegLoad;
+defm PseudoVLUXSEG : VPseudoISegLoad;
defm PseudoVSSEG : VPseudoUSSegStore;
defm PseudoVSSSEG : VPseudoSSegStore;