From 801caff64d2f3eea7b97270855d765c34e102c19 Mon Sep 17 00:00:00 2001 From: Sanjay Patel Date: Tue, 5 May 2015 21:40:38 +0000 Subject: [PATCH] propagate IR-level fast-math-flags to DAG nodes (NFC) This patch adds the minimum plumbing necessary to use IR-level fast-math-flags (FMF) in the backend without actually using them for anything yet. This is a follow-on to: http://reviews.llvm.org/rL235997 ...which split the existing nsw / nuw / exact flags and FMF into their own struct. There are 2 structural changes here: 1. The main diff is that we're preparing to extend the optimization flags to affect more than just binary SDNodes. Eg, IR intrinsics ( https://llvm.org/bugs/show_bug.cgi?id=21290 ) or non-binop nodes that don't even exist in IR such as FMA, FNEG, etc. 2. The other change is that we're actually copying the FP fast-math-flags from the IR instructions to SDNodes. Differential Revision: http://reviews.llvm.org/D8900 llvm-svn: 236546 --- llvm/include/llvm/CodeGen/SelectionDAG.h | 10 +-- llvm/include/llvm/CodeGen/SelectionDAGNodes.h | 68 +++++++++-------- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 9 +-- llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 89 ++++++++++------------ .../CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 22 +++++- llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 5 +- llvm/lib/Target/X86/X86ISelLowering.cpp | 5 +- 7 files changed, 109 insertions(+), 99 deletions(-) diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 10aaecb..b7873c0 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -655,7 +655,7 @@ public: SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT); SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N); SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2, - bool nuw = false, bool nsw = false, bool exact = false); + const SDNodeFlags *Flags = nullptr); SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2, SDValue N3); SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2, @@ -978,8 +978,7 @@ public: /// Get the specified node if it's already available, or else return NULL. SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTs, ArrayRef Ops, - bool nuw = false, bool nsw = false, - bool exact = false); + const SDNodeFlags *Flags = nullptr); /// Creates a SDDbgValue node. SDDbgValue *getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N, unsigned R, @@ -1236,9 +1235,8 @@ private: void allnodes_clear(); - BinarySDNode *GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs, - SDValue N1, SDValue N2, bool nuw, bool nsw, - bool exact); + SDNode *GetSDNodeWithFlags(unsigned Opcode, SDLoc DL, SDVTList VTs, + ArrayRef Ops, const SDNodeFlags *Flags); /// List of non-single value types. FoldingSet VTListMap; diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index 179c3c03..56434bf 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -981,6 +981,44 @@ public: (NoSignedZeros << 6) | (AllowReciprocal << 7); } }; + +/// Returns true if the opcode is an operation with optional optimization flags. +static bool mayHaveOptimizationFlags(unsigned Opcode) { + switch (Opcode) { + case ISD::SDIV: + case ISD::UDIV: + case ISD::SRA: + case ISD::SRL: + case ISD::MUL: + case ISD::ADD: + case ISD::SUB: + case ISD::SHL: + case ISD::FADD: + case ISD::FDIV: + case ISD::FMUL: + case ISD::FREM: + case ISD::FSUB: + return true; + default: + return false; + } +} +/// This class is an extension of SDNode used from instructions that may have +/// associated extra flags. +class SDNodeWithFlags : public SDNode { +public: + SDNodeFlags Flags; + SDNodeWithFlags(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, + ArrayRef Ops, const SDNodeFlags &NodeFlags) + : SDNode(Opc, Order, dl, VTs, Ops) { + Flags = NodeFlags; + } + + // This is used to implement dyn_cast, isa, and other type queries. + static bool classof(const SDNode *N) { + return mayHaveOptimizationFlags(N->getOpcode()); + } +}; /// This class is used for single-operand SDNodes. This is solely /// to allow co-allocation of node operands with the node itself. @@ -1006,36 +1044,6 @@ public: } }; -/// Returns true if the opcode is a binary operation with flags. -static bool isBinOpWithFlags(unsigned Opcode) { - switch (Opcode) { - case ISD::SDIV: - case ISD::UDIV: - case ISD::SRA: - case ISD::SRL: - case ISD::MUL: - case ISD::ADD: - case ISD::SUB: - case ISD::SHL: - return true; - default: - return false; - } -} - -/// This class is an extension of BinarySDNode -/// used from those opcodes that have associated extra flags. -class BinaryWithFlagsSDNode : public BinarySDNode { -public: - SDNodeFlags Flags; - BinaryWithFlagsSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, - SDValue X, SDValue Y) - : BinarySDNode(Opc, Order, dl, VTs, X, Y), Flags() { } - static bool classof(const SDNode *N) { - return isBinOpWithFlags(N->getOpcode()); - } -}; - /// This class is used for three-operand SDNodes. This is solely /// to allow co-allocation of node operands with the node itself. class TernarySDNode : public SDNode { diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index bf42aab..c063d4f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -1452,12 +1452,9 @@ SDValue DAGCombiner::combine(SDNode *N) { if (isa(N0) || !isa(N1)) { SDValue Ops[] = {N1, N0}; SDNode *CSENode; - if (const BinaryWithFlagsSDNode *BinNode = - dyn_cast(N)) { - CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops, - BinNode->Flags.hasNoUnsignedWrap(), - BinNode->Flags.hasNoSignedWrap(), - BinNode->Flags.hasExact()); + if (auto *FlagsNode = dyn_cast(N)) { + CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), + Ops, &FlagsNode->Flags); } else { CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops); } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 2afde5f..3b22e97 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -400,18 +400,22 @@ static void AddNodeIDOperands(FoldingSetNodeID &ID, } } -static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, bool nuw, bool nsw, - bool exact) { - ID.AddBoolean(nuw); - ID.AddBoolean(nsw); - ID.AddBoolean(exact); +/// Add logical or fast math flag values to FoldingSetNodeID value. +static void AddNodeIDFlags(FoldingSetNodeID &ID, unsigned Opcode, + const SDNodeFlags *Flags) { + if (!Flags || !mayHaveOptimizationFlags(Opcode)) + return; + + unsigned RawFlags = Flags->getRawFlags(); + // If no flags are set, do not alter the ID. This saves time and allows + // a gradual increase in API usage of the optional optimization flags. + if (RawFlags != 0) + ID.AddInteger(RawFlags); } -/// AddBinaryNodeIDCustom - Add BinarySDNodes special infos -static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, unsigned Opcode, - bool nuw, bool nsw, bool exact) { - if (isBinOpWithFlags(Opcode)) - AddBinaryNodeIDCustom(ID, nuw, nsw, exact); +static void AddNodeIDFlags(FoldingSetNodeID &ID, const SDNode *N) { + if (auto *Node = dyn_cast(N)) + AddNodeIDFlags(ID, Node->getOpcode(), &Node->Flags); } static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC, @@ -506,21 +510,6 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { ID.AddInteger(ST->getPointerInfo().getAddrSpace()); break; } - case ISD::SDIV: - case ISD::UDIV: - case ISD::SRA: - case ISD::SRL: - case ISD::MUL: - case ISD::ADD: - case ISD::SUB: - case ISD::SHL: { - const BinaryWithFlagsSDNode *BinNode = cast(N); - AddBinaryNodeIDCustom(ID, N->getOpcode(), - BinNode->Flags.hasNoUnsignedWrap(), - BinNode->Flags.hasNoSignedWrap(), - BinNode->Flags.hasExact()); - break; - } case ISD::ATOMIC_CMP_SWAP: case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: case ISD::ATOMIC_SWAP: @@ -564,6 +553,8 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { } } // end switch (N->getOpcode()) + AddNodeIDFlags(ID, N); + // Target specific memory nodes could also have address spaces to check. if (N->isTargetMemoryOpcode()) ID.AddInteger(cast(N)->getPointerInfo().getAddrSpace()); @@ -958,22 +949,22 @@ void SelectionDAG::allnodes_clear() { DeallocateNode(AllNodes.begin()); } -BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL, - SDVTList VTs, SDValue N1, - SDValue N2, bool nuw, bool nsw, - bool exact) { - if (isBinOpWithFlags(Opcode)) { - BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode( - Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2); - FN->Flags.setNoUnsignedWrap(nuw); - FN->Flags.setNoSignedWrap(nsw); - FN->Flags.setExact(exact); +SDNode *SelectionDAG::GetSDNodeWithFlags(unsigned Opcode, SDLoc DL, + SDVTList VTs, ArrayRef Ops, + const SDNodeFlags *Flags) { + if (mayHaveOptimizationFlags(Opcode)) { + // If no flags were passed in, use a default flags object. + SDNodeFlags F; + if (Flags == nullptr) + Flags = &F; - return FN; + SDNodeWithFlags *NodeWithFlags = new (NodeAllocator) SDNodeWithFlags( + Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Ops, *Flags); + return NodeWithFlags; } - BinarySDNode *N = new (NodeAllocator) - BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2); + SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), + DL.getDebugLoc(), VTs, Ops); return N; } @@ -3201,7 +3192,7 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, SDLoc DL, EVT VT, } SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, - SDValue N2, bool nuw, bool nsw, bool exact) { + SDValue N2, const SDNodeFlags *Flags) { ConstantSDNode *N1C = dyn_cast(N1.getNode()); ConstantSDNode *N2C = dyn_cast(N2.getNode()); switch (Opcode) { @@ -3659,24 +3650,23 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, } // Memoize this node if possible. - BinarySDNode *N; + SDNode *N; SDVTList VTs = getVTList(VT); - const bool BinOpHasFlags = isBinOpWithFlags(Opcode); + SDValue Ops[] = { N1, N2 }; if (VT != MVT::Glue) { SDValue Ops[] = {N1, N2}; FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTs, Ops); - if (BinOpHasFlags) - AddBinaryNodeIDCustom(ID, Opcode, nuw, nsw, exact); + AddNodeIDFlags(ID, Opcode, Flags); void *IP = nullptr; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact); - + N = GetSDNodeWithFlags(Opcode, DL, VTs, Ops, Flags); + CSEMap.InsertNode(N, IP); } else { - N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact); + N = GetSDNodeWithFlags(Opcode, DL, VTs, Ops, Flags); } InsertNode(N); @@ -5980,13 +5970,12 @@ SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT, /// getNodeIfExists - Get the specified node if it's already available, or /// else return NULL. SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, - ArrayRef Ops, bool nuw, bool nsw, - bool exact) { + ArrayRef Ops, + const SDNodeFlags *Flags) { if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTList, Ops); - if (isBinOpWithFlags(Opcode)) - AddBinaryNodeIDCustom(ID, nuw, nsw, exact); + AddNodeIDFlags(ID, Opcode, Flags); void *IP = nullptr; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return E; diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 5b6efe0..edbc260 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -2208,6 +2208,8 @@ void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) { bool nuw = false; bool nsw = false; bool exact = false; + FastMathFlags FMF; + if (const OverflowingBinaryOperator *OFBinOp = dyn_cast(&I)) { nuw = OFBinOp->hasNoUnsignedWrap(); @@ -2217,8 +2219,20 @@ void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) { dyn_cast(&I)) exact = ExactOp->isExact(); + if (const FPMathOperator *FPOp = dyn_cast(&I)) + FMF = FPOp->getFastMathFlags(); + + SDNodeFlags Flags; + Flags.setAllowReciprocal(FMF.allowReciprocal()); + Flags.setExact(exact); + Flags.setNoInfs(FMF.noInfs()); + Flags.setNoNaNs(FMF.noNaNs()); + Flags.setNoSignedWrap(nsw); + Flags.setNoSignedZeros(FMF.noSignedZeros()); + Flags.setNoUnsignedWrap(nuw); + Flags.setUnsafeAlgebra(FMF.unsafeAlgebra()); SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(), - Op1, Op2, nuw, nsw, exact); + Op1, Op2, &Flags); setValue(&I, BinNodeValue); } @@ -2267,8 +2281,12 @@ void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) { exact = ExactOp->isExact(); } + SDNodeFlags Flags; + Flags.setExact(exact); + Flags.setNoSignedWrap(nsw); + Flags.setNoUnsignedWrap(nuw); SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2, - nuw, nsw, exact); + &Flags); setValue(&I, Res); } diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 34ddeb7..6f22f42 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -2660,8 +2660,9 @@ SDValue TargetLowering::BuildExactSDIV(SDValue Op1, SDValue Op2, SDLoc dl, // TODO: For UDIV use SRL instead of SRA. SDValue Amt = DAG.getConstant(ShAmt, dl, getShiftAmountTy(Op1.getValueType())); - Op1 = DAG.getNode(ISD::SRA, dl, Op1.getValueType(), Op1, Amt, false, false, - true); + SDNodeFlags Flags; + Flags.setExact(true); + Op1 = DAG.getNode(ISD::SRA, dl, Op1.getValueType(), Op1, Amt, &Flags); d = d.ashr(ShAmt); } diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 49f0a8a..e2c427d 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -12561,9 +12561,8 @@ SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC, SDLoc dl, case ISD::SUB: case ISD::MUL: case ISD::SHL: { - const BinaryWithFlagsSDNode *BinNode = - cast(Op.getNode()); - if (BinNode->Flags.hasNoSignedWrap()) + const SDNodeWithFlags *Node = cast(Op.getNode()); + if (Node->Flags.hasNoSignedWrap()) break; } default: -- 2.7.4