From a40e01e4423ca13052f0fe0e5bc14abec1dd866e Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 27 Jan 2021 11:01:07 -0800 Subject: [PATCH] [RISCV] Rework fault first only load isel. -Remove the ISD opcode for READ_VL. Just emit the MachineSDNode directly. -Move segmented fault first only load intrinsic handling completely to RISCVISelDAGToDAG.cpp and emit the ReadVL MachineSDNode there instead of lowering to ISD opcodes first. --- llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp | 46 ++++++++---- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 98 ++----------------------- llvm/lib/Target/RISCV/RISCVISelLowering.h | 5 -- llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td | 6 +- 4 files changed, 41 insertions(+), 114 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp index 7b0f386..7292cfe 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -225,7 +225,7 @@ void RISCVDAGToDAGISel::selectVLSEGMask(SDNode *Node, unsigned IntNo, void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node) { SDLoc DL(Node); unsigned IntNo = cast(Node->getOperand(1))->getZExtValue(); - unsigned NF = Node->getNumValues() - 2; // Do not count Chain and Glue. + unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain. EVT VT = Node->getValueType(0); unsigned ScalarSize = VT.getScalarSizeInBits(); MVT XLenVT = Subtarget->getXLenVT(); @@ -241,21 +241,24 @@ void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node) { static_cast(RISCVVLMUL::LMUL_1)); SDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, MVT::Glue, Operands); + SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT, + /*Glue*/ SDValue(Load, 2)); + SDValue SuperReg = SDValue(Load, 0); for (unsigned I = 0; I < NF; ++I) ReplaceUses(SDValue(Node, I), CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL, VT, SuperReg)); - ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); // Chain. - ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Glue. + ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0)); // VL + ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain CurDAG->RemoveDeadNode(Node); } void RISCVDAGToDAGISel::selectVLSEGFFMask(SDNode *Node) { SDLoc DL(Node); unsigned IntNo = cast(Node->getOperand(1))->getZExtValue(); - unsigned NF = Node->getNumValues() - 2; // Do not count Chain and Glue. + unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain. EVT VT = Node->getValueType(0); unsigned ScalarSize = VT.getScalarSizeInBits(); MVT XLenVT = Subtarget->getXLenVT(); @@ -275,14 +278,17 @@ void RISCVDAGToDAGISel::selectVLSEGFFMask(SDNode *Node) { static_cast(RISCVVLMUL::LMUL_1)); SDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, MVT::Glue, Operands); + SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT, + /*Glue*/ SDValue(Load, 2)); + SDValue SuperReg = SDValue(Load, 0); for (unsigned I = 0; I < NF; ++I) ReplaceUses(SDValue(Node, I), CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL, VT, SuperReg)); - ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); // Chain. - ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Glue. + ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0)); // VL + ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain CurDAG->RemoveDeadNode(Node); } @@ -680,6 +686,26 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) { selectVLXSEGMask(Node, IntNo); return; } + case Intrinsic::riscv_vlseg8ff: + case Intrinsic::riscv_vlseg7ff: + case Intrinsic::riscv_vlseg6ff: + case Intrinsic::riscv_vlseg5ff: + case Intrinsic::riscv_vlseg4ff: + case Intrinsic::riscv_vlseg3ff: + case Intrinsic::riscv_vlseg2ff: { + selectVLSEGFF(Node); + return; + } + case Intrinsic::riscv_vlseg8ff_mask: + case Intrinsic::riscv_vlseg7ff_mask: + case Intrinsic::riscv_vlseg6ff_mask: + case Intrinsic::riscv_vlseg5ff_mask: + case Intrinsic::riscv_vlseg4ff_mask: + case Intrinsic::riscv_vlseg3ff_mask: + case Intrinsic::riscv_vlseg2ff_mask: { + selectVLSEGFFMask(Node); + return; + } } break; } @@ -763,14 +789,6 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) { } break; } - case RISCVISD::VLSEGFF: { - selectVLSEGFF(Node); - return; - } - case RISCVISD::VLSEGFF_MASK: { - selectVLSEGFFMask(Node); - return; - } } // Select the default instruction. diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 97862f9..23a677d 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -1455,7 +1455,6 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op, } } - unsigned NF = 1; switch (IntNo) { default: return SDValue(); // Don't custom lower most intrinsics. @@ -1464,8 +1463,10 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op, SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Other, MVT::Glue); SDValue Load = DAG.getNode(RISCVISD::VLEFF, DL, VTs, Op.getOperand(0), Op.getOperand(2), Op.getOperand(3)); - VTs = DAG.getVTList(Op->getValueType(1), MVT::Other); - SDValue ReadVL = DAG.getNode(RISCVISD::READ_VL, DL, VTs, Load.getValue(2)); + SDValue ReadVL = + SDValue(DAG.getMachineNode(RISCV::PseudoReadVL, DL, Op->getValueType(1), + Load.getValue(2)), + 0); return DAG.getMergeValues({Load, ReadVL, Load.getValue(1)}, DL); } case Intrinsic::riscv_vleff_mask: { @@ -1474,92 +1475,12 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op, SDValue Load = DAG.getNode(RISCVISD::VLEFF_MASK, DL, VTs, Op.getOperand(0), Op.getOperand(2), Op.getOperand(3), Op.getOperand(4), Op.getOperand(5)); - VTs = DAG.getVTList(Op->getValueType(1), MVT::Other); - SDValue ReadVL = DAG.getNode(RISCVISD::READ_VL, DL, VTs, Load.getValue(2)); + SDValue ReadVL = + SDValue(DAG.getMachineNode(RISCV::PseudoReadVL, DL, Op->getValueType(1), + Load.getValue(2)), + 0); return DAG.getMergeValues({Load, ReadVL, Load.getValue(1)}, DL); } - case Intrinsic::riscv_vlseg8ff: - NF++; - LLVM_FALLTHROUGH; - case Intrinsic::riscv_vlseg7ff: - NF++; - LLVM_FALLTHROUGH; - case Intrinsic::riscv_vlseg6ff: - NF++; - LLVM_FALLTHROUGH; - case Intrinsic::riscv_vlseg5ff: - NF++; - LLVM_FALLTHROUGH; - case Intrinsic::riscv_vlseg4ff: - NF++; - LLVM_FALLTHROUGH; - case Intrinsic::riscv_vlseg3ff: - NF++; - LLVM_FALLTHROUGH; - case Intrinsic::riscv_vlseg2ff: { - NF++; - SDLoc DL(Op); - SmallVector EVTs(NF, Op.getValueType()); - EVTs.push_back(MVT::Other); - EVTs.push_back(MVT::Glue); - SDVTList VTs = DAG.getVTList(EVTs); - SDValue Load = - DAG.getNode(RISCVISD::VLSEGFF, DL, VTs, Op.getOperand(0), - Op.getOperand(1), Op.getOperand(2), Op.getOperand(3)); - VTs = DAG.getVTList(Op->getValueType(NF), MVT::Other); - SDValue ReadVL = DAG.getNode(RISCVISD::READ_VL, DL, VTs, - /*Glue*/ Load.getValue(NF + 1)); - SmallVector Results; - for (unsigned i = 0; i < NF; ++i) - Results.push_back(Load.getValue(i)); - Results.push_back(ReadVL); - Results.push_back(Load.getValue(NF)); // Chain. - return DAG.getMergeValues(Results, DL); - } - case Intrinsic::riscv_vlseg8ff_mask: - NF++; - LLVM_FALLTHROUGH; - case Intrinsic::riscv_vlseg7ff_mask: - NF++; - LLVM_FALLTHROUGH; - case Intrinsic::riscv_vlseg6ff_mask: - NF++; - LLVM_FALLTHROUGH; - case Intrinsic::riscv_vlseg5ff_mask: - NF++; - LLVM_FALLTHROUGH; - case Intrinsic::riscv_vlseg4ff_mask: - NF++; - LLVM_FALLTHROUGH; - case Intrinsic::riscv_vlseg3ff_mask: - NF++; - LLVM_FALLTHROUGH; - case Intrinsic::riscv_vlseg2ff_mask: { - NF++; - SDLoc DL(Op); - SmallVector EVTs(NF, Op.getValueType()); - EVTs.push_back(MVT::Other); - EVTs.push_back(MVT::Glue); - SDVTList VTs = DAG.getVTList(EVTs); - SmallVector LoadOps; - LoadOps.push_back(Op.getOperand(0)); // Chain. - LoadOps.push_back(Op.getOperand(1)); // Intrinsic ID. - for (unsigned i = 0; i < NF; ++i) - LoadOps.push_back(Op.getOperand(2 + i)); // MaskedOff. - LoadOps.push_back(Op.getOperand(2 + NF)); // Base. - LoadOps.push_back(Op.getOperand(3 + NF)); // Mask. - LoadOps.push_back(Op.getOperand(4 + NF)); // VL. - SDValue Load = DAG.getNode(RISCVISD::VLSEGFF_MASK, DL, VTs, LoadOps); - VTs = DAG.getVTList(Op->getValueType(NF), MVT::Other); - SDValue ReadVL = DAG.getNode(RISCVISD::READ_VL, DL, VTs, - /*Glue*/ Load.getValue(NF + 1)); - SmallVector Results; - for (unsigned i = 0; i < NF; ++i) - Results.push_back(Load.getValue(i)); - Results.push_back(ReadVL); - Results.push_back(Load.getValue(NF)); // Chain. - return DAG.getMergeValues(Results, DL); - } } } @@ -4088,9 +4009,6 @@ const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const { NODE_NAME_CASE(TRUNCATE_VECTOR) NODE_NAME_CASE(VLEFF) NODE_NAME_CASE(VLEFF_MASK) - NODE_NAME_CASE(VLSEGFF) - NODE_NAME_CASE(VLSEGFF_MASK) - NODE_NAME_CASE(READ_VL) NODE_NAME_CASE(VSLIDEUP) NODE_NAME_CASE(VSLIDEDOWN) NODE_NAME_CASE(VID) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h index 40b1a45..bff3f85 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.h +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h @@ -99,11 +99,6 @@ enum NodeType : unsigned { // Unit-stride fault-only-first load VLEFF, VLEFF_MASK, - // Unit-stride fault-only-first segment load - VLSEGFF, - VLSEGFF_MASK, - // read vl CSR - READ_VL, // Matches the semantics of vslideup/vslidedown. The first operand is the // pass-thru operand, the second is the source vector, and the third is the // XLenVT index (either constant or non-constant). diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index 06e4d05..9ff6b4e 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -33,9 +33,6 @@ def riscv_vleff_mask : SDNode<"RISCVISD::VLEFF_MASK", SDTCisVT<4, XLenVT>]>, [SDNPHasChain, SDNPOutGlue, SDNPMayLoad, SDNPSideEffect]>; -def riscv_read_vl : SDNode<"RISCVISD::READ_VL", - SDTypeProfile<1, 0, [SDTCisVT<0, XLenVT>]>, - [SDNPInGlue]>; // X0 has special meaning for vsetvl/vsetvli. // rd | rs1 | AVL value | Effect on vl @@ -3115,8 +3112,7 @@ let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 1 in { let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 1, Uses = [VL] in -def PseudoReadVL : Pseudo<(outs GPR:$rd), (ins), - [(set GPR:$rd, (riscv_read_vl))]>; +def PseudoReadVL : Pseudo<(outs GPR:$rd), (ins), []>; //===----------------------------------------------------------------------===// // 6. Configuration-Setting Instructions -- 2.7.4