-Remove the ISD opcode for READ_VL. Just emit the MachineSDNode directly.
-Move segmented fault first only load intrinsic handling completely to
RISCVISelDAGToDAG.cpp and emit the ReadVL MachineSDNode there
instead of lowering to ISD opcodes first.
void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node) {
SDLoc DL(Node);
unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
- unsigned NF = Node->getNumValues() - 2; // Do not count Chain and Glue.
+ unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain.
EVT VT = Node->getValueType(0);
unsigned ScalarSize = VT.getScalarSizeInBits();
MVT XLenVT = Subtarget->getXLenVT();
static_cast<unsigned>(RISCVVLMUL::LMUL_1));
SDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other,
MVT::Glue, Operands);
+ SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
+ /*Glue*/ SDValue(Load, 2));
+
SDValue SuperReg = SDValue(Load, 0);
for (unsigned I = 0; I < NF; ++I)
ReplaceUses(SDValue(Node, I),
CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
VT, SuperReg));
- ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); // Chain.
- ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Glue.
+ ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0)); // VL
+ ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain
CurDAG->RemoveDeadNode(Node);
}
void RISCVDAGToDAGISel::selectVLSEGFFMask(SDNode *Node) {
SDLoc DL(Node);
unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
- unsigned NF = Node->getNumValues() - 2; // Do not count Chain and Glue.
+ unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain.
EVT VT = Node->getValueType(0);
unsigned ScalarSize = VT.getScalarSizeInBits();
MVT XLenVT = Subtarget->getXLenVT();
static_cast<unsigned>(RISCVVLMUL::LMUL_1));
SDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other,
MVT::Glue, Operands);
+ SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
+ /*Glue*/ SDValue(Load, 2));
+
SDValue SuperReg = SDValue(Load, 0);
for (unsigned I = 0; I < NF; ++I)
ReplaceUses(SDValue(Node, I),
CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
VT, SuperReg));
- ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); // Chain.
- ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Glue.
+ ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0)); // VL
+ ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain
CurDAG->RemoveDeadNode(Node);
}
selectVLXSEGMask(Node, IntNo);
return;
}
+ case Intrinsic::riscv_vlseg8ff:
+ case Intrinsic::riscv_vlseg7ff:
+ case Intrinsic::riscv_vlseg6ff:
+ case Intrinsic::riscv_vlseg5ff:
+ case Intrinsic::riscv_vlseg4ff:
+ case Intrinsic::riscv_vlseg3ff:
+ case Intrinsic::riscv_vlseg2ff: {
+ selectVLSEGFF(Node);
+ return;
+ }
+ case Intrinsic::riscv_vlseg8ff_mask:
+ case Intrinsic::riscv_vlseg7ff_mask:
+ case Intrinsic::riscv_vlseg6ff_mask:
+ case Intrinsic::riscv_vlseg5ff_mask:
+ case Intrinsic::riscv_vlseg4ff_mask:
+ case Intrinsic::riscv_vlseg3ff_mask:
+ case Intrinsic::riscv_vlseg2ff_mask: {
+ selectVLSEGFFMask(Node);
+ return;
+ }
}
break;
}
}
break;
}
- case RISCVISD::VLSEGFF: {
- selectVLSEGFF(Node);
- return;
- }
- case RISCVISD::VLSEGFF_MASK: {
- selectVLSEGFFMask(Node);
- return;
- }
}
// Select the default instruction.
}
}
- unsigned NF = 1;
switch (IntNo) {
default:
return SDValue(); // Don't custom lower most intrinsics.
SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Other, MVT::Glue);
SDValue Load = DAG.getNode(RISCVISD::VLEFF, DL, VTs, Op.getOperand(0),
Op.getOperand(2), Op.getOperand(3));
- VTs = DAG.getVTList(Op->getValueType(1), MVT::Other);
- SDValue ReadVL = DAG.getNode(RISCVISD::READ_VL, DL, VTs, Load.getValue(2));
+ SDValue ReadVL =
+ SDValue(DAG.getMachineNode(RISCV::PseudoReadVL, DL, Op->getValueType(1),
+ Load.getValue(2)),
+ 0);
return DAG.getMergeValues({Load, ReadVL, Load.getValue(1)}, DL);
}
case Intrinsic::riscv_vleff_mask: {
SDValue Load = DAG.getNode(RISCVISD::VLEFF_MASK, DL, VTs, Op.getOperand(0),
Op.getOperand(2), Op.getOperand(3),
Op.getOperand(4), Op.getOperand(5));
- VTs = DAG.getVTList(Op->getValueType(1), MVT::Other);
- SDValue ReadVL = DAG.getNode(RISCVISD::READ_VL, DL, VTs, Load.getValue(2));
+ SDValue ReadVL =
+ SDValue(DAG.getMachineNode(RISCV::PseudoReadVL, DL, Op->getValueType(1),
+ Load.getValue(2)),
+ 0);
return DAG.getMergeValues({Load, ReadVL, Load.getValue(1)}, DL);
}
- case Intrinsic::riscv_vlseg8ff:
- NF++;
- LLVM_FALLTHROUGH;
- case Intrinsic::riscv_vlseg7ff:
- NF++;
- LLVM_FALLTHROUGH;
- case Intrinsic::riscv_vlseg6ff:
- NF++;
- LLVM_FALLTHROUGH;
- case Intrinsic::riscv_vlseg5ff:
- NF++;
- LLVM_FALLTHROUGH;
- case Intrinsic::riscv_vlseg4ff:
- NF++;
- LLVM_FALLTHROUGH;
- case Intrinsic::riscv_vlseg3ff:
- NF++;
- LLVM_FALLTHROUGH;
- case Intrinsic::riscv_vlseg2ff: {
- NF++;
- SDLoc DL(Op);
- SmallVector<EVT, 8> EVTs(NF, Op.getValueType());
- EVTs.push_back(MVT::Other);
- EVTs.push_back(MVT::Glue);
- SDVTList VTs = DAG.getVTList(EVTs);
- SDValue Load =
- DAG.getNode(RISCVISD::VLSEGFF, DL, VTs, Op.getOperand(0),
- Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
- VTs = DAG.getVTList(Op->getValueType(NF), MVT::Other);
- SDValue ReadVL = DAG.getNode(RISCVISD::READ_VL, DL, VTs,
- /*Glue*/ Load.getValue(NF + 1));
- SmallVector<SDValue, 8> Results;
- for (unsigned i = 0; i < NF; ++i)
- Results.push_back(Load.getValue(i));
- Results.push_back(ReadVL);
- Results.push_back(Load.getValue(NF)); // Chain.
- return DAG.getMergeValues(Results, DL);
- }
- case Intrinsic::riscv_vlseg8ff_mask:
- NF++;
- LLVM_FALLTHROUGH;
- case Intrinsic::riscv_vlseg7ff_mask:
- NF++;
- LLVM_FALLTHROUGH;
- case Intrinsic::riscv_vlseg6ff_mask:
- NF++;
- LLVM_FALLTHROUGH;
- case Intrinsic::riscv_vlseg5ff_mask:
- NF++;
- LLVM_FALLTHROUGH;
- case Intrinsic::riscv_vlseg4ff_mask:
- NF++;
- LLVM_FALLTHROUGH;
- case Intrinsic::riscv_vlseg3ff_mask:
- NF++;
- LLVM_FALLTHROUGH;
- case Intrinsic::riscv_vlseg2ff_mask: {
- NF++;
- SDLoc DL(Op);
- SmallVector<EVT, 8> EVTs(NF, Op.getValueType());
- EVTs.push_back(MVT::Other);
- EVTs.push_back(MVT::Glue);
- SDVTList VTs = DAG.getVTList(EVTs);
- SmallVector<SDValue, 13> LoadOps;
- LoadOps.push_back(Op.getOperand(0)); // Chain.
- LoadOps.push_back(Op.getOperand(1)); // Intrinsic ID.
- for (unsigned i = 0; i < NF; ++i)
- LoadOps.push_back(Op.getOperand(2 + i)); // MaskedOff.
- LoadOps.push_back(Op.getOperand(2 + NF)); // Base.
- LoadOps.push_back(Op.getOperand(3 + NF)); // Mask.
- LoadOps.push_back(Op.getOperand(4 + NF)); // VL.
- SDValue Load = DAG.getNode(RISCVISD::VLSEGFF_MASK, DL, VTs, LoadOps);
- VTs = DAG.getVTList(Op->getValueType(NF), MVT::Other);
- SDValue ReadVL = DAG.getNode(RISCVISD::READ_VL, DL, VTs,
- /*Glue*/ Load.getValue(NF + 1));
- SmallVector<SDValue, 8> Results;
- for (unsigned i = 0; i < NF; ++i)
- Results.push_back(Load.getValue(i));
- Results.push_back(ReadVL);
- Results.push_back(Load.getValue(NF)); // Chain.
- return DAG.getMergeValues(Results, DL);
- }
}
}
NODE_NAME_CASE(TRUNCATE_VECTOR)
NODE_NAME_CASE(VLEFF)
NODE_NAME_CASE(VLEFF_MASK)
- NODE_NAME_CASE(VLSEGFF)
- NODE_NAME_CASE(VLSEGFF_MASK)
- NODE_NAME_CASE(READ_VL)
NODE_NAME_CASE(VSLIDEUP)
NODE_NAME_CASE(VSLIDEDOWN)
NODE_NAME_CASE(VID)
// Unit-stride fault-only-first load
VLEFF,
VLEFF_MASK,
- // Unit-stride fault-only-first segment load
- VLSEGFF,
- VLSEGFF_MASK,
- // read vl CSR
- READ_VL,
// Matches the semantics of vslideup/vslidedown. The first operand is the
// pass-thru operand, the second is the source vector, and the third is the
// XLenVT index (either constant or non-constant).
SDTCisVT<4, XLenVT>]>,
[SDNPHasChain, SDNPOutGlue, SDNPMayLoad,
SDNPSideEffect]>;
-def riscv_read_vl : SDNode<"RISCVISD::READ_VL",
- SDTypeProfile<1, 0, [SDTCisVT<0, XLenVT>]>,
- [SDNPInGlue]>;
// X0 has special meaning for vsetvl/vsetvli.
// rd | rs1 | AVL value | Effect on vl
let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 1,
Uses = [VL] in
-def PseudoReadVL : Pseudo<(outs GPR:$rd), (ins),
- [(set GPR:$rd, (riscv_read_vl))]>;
+def PseudoReadVL : Pseudo<(outs GPR:$rd), (ins), []>;
//===----------------------------------------------------------------------===//
// 6. Configuration-Setting Instructions