return false;
}
-TargetLowering::NegatibleCost
-TargetLowering::getNegatibleCost(SDValue Op, SelectionDAG &DAG,
- bool LegalOperations, bool ForCodeSize,
- unsigned Depth) const {
+SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
+ bool LegalOps, bool OptForSize,
+ NegatibleCost &Cost,
+ unsigned Depth) const {
// fneg is removable even if it has multiple uses.
- if (Op.getOpcode() == ISD::FNEG)
- return NegatibleCost::Cheaper;
+ if (Op.getOpcode() == ISD::FNEG) {
+ Cost = NegatibleCost::Cheaper;
+ return Op.getOperand(0);
+ }
- // Don't allow anything with multiple uses unless we know it is free.
- EVT VT = Op.getValueType();
+ // Don't recurse exponentially.
+ if (Depth > SelectionDAG::MaxRecursionDepth)
+ return SDValue();
+
+ // Pre-increment recursion depth for use in recursive calls.
+ ++Depth;
const SDNodeFlags Flags = Op->getFlags();
const TargetOptions &Options = DAG.getTarget().Options;
- if (!Op.hasOneUse()) {
- bool IsFreeExtend = Op.getOpcode() == ISD::FP_EXTEND &&
- isFPExtFree(VT, Op.getOperand(0).getValueType());
-
- // If we already have the use of the negated floating constant, it is free
- // to negate it even it has multiple uses.
- bool IsFreeConstant =
- Op.getOpcode() == ISD::ConstantFP &&
- !negateExpression(Op, DAG, LegalOperations, ForCodeSize).use_empty();
+ EVT VT = Op.getValueType();
+ unsigned Opcode = Op.getOpcode();
- if (!IsFreeExtend && !IsFreeConstant)
- return NegatibleCost::Expensive;
+ // Don't allow anything with multiple uses unless we know it is free.
+ if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) {
+ bool IsFreeExtend = Opcode == ISD::FP_EXTEND &&
+ isFPExtFree(VT, Op.getOperand(0).getValueType());
+ if (!IsFreeExtend)
+ return SDValue();
}
- // Don't recurse exponentially.
- if (Depth > SelectionDAG::MaxRecursionDepth)
- return NegatibleCost::Expensive;
+ SDLoc DL(Op);
- switch (Op.getOpcode()) {
+ switch (Opcode) {
case ISD::ConstantFP: {
- if (!LegalOperations)
- return NegatibleCost::Neutral;
-
// Don't invert constant FP values after legalization unless the target says
// the negated constant is legal.
- if (isOperationLegal(ISD::ConstantFP, VT) ||
+ bool IsOpLegal =
+ isOperationLegal(ISD::ConstantFP, VT) ||
isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT,
- ForCodeSize))
- return NegatibleCost::Neutral;
- break;
+ OptForSize);
+
+ if (LegalOps && !IsOpLegal)
+ break;
+
+ APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
+ V.changeSign();
+ SDValue CFP = DAG.getConstantFP(V, DL, VT);
+
+ // If we already have the use of the negated floating constant, it is free
+ // to negate it even it has multiple uses.
+ if (!Op.hasOneUse() && CFP.use_empty())
+ break;
+ Cost = NegatibleCost::Neutral;
+ return CFP;
}
case ISD::BUILD_VECTOR: {
// Only permit BUILD_VECTOR of constants.
if (llvm::any_of(Op->op_values(), [&](SDValue N) {
return !N.isUndef() && !isa<ConstantFPSDNode>(N);
}))
- return NegatibleCost::Expensive;
- if (!LegalOperations)
- return NegatibleCost::Neutral;
- if (isOperationLegal(ISD::ConstantFP, VT) &&
- isOperationLegal(ISD::BUILD_VECTOR, VT))
- return NegatibleCost::Neutral;
- if (llvm::all_of(Op->op_values(), [&](SDValue N) {
+ break;
+
+ bool IsOpLegal =
+ (isOperationLegal(ISD::ConstantFP, VT) &&
+ isOperationLegal(ISD::BUILD_VECTOR, VT)) ||
+ llvm::all_of(Op->op_values(), [&](SDValue N) {
return N.isUndef() ||
isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT,
- ForCodeSize);
- }))
- return NegatibleCost::Neutral;
- break;
- }
- case ISD::FADD: {
- if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
- return NegatibleCost::Expensive;
-
- // After operation legalization, it might not be legal to create new FSUBs.
- if (LegalOperations && !isOperationLegalOrCustom(ISD::FSUB, VT))
- return NegatibleCost::Expensive;
-
- // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
- NegatibleCost V0 = getNegatibleCost(Op.getOperand(0), DAG, LegalOperations,
- ForCodeSize, Depth + 1);
- if (V0 != NegatibleCost::Expensive)
- return V0;
- // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
- return getNegatibleCost(Op.getOperand(1), DAG, LegalOperations, ForCodeSize,
- Depth + 1);
- }
- case ISD::FSUB:
- // We can't turn -(A-B) into B-A when we honor signed zeros.
- if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
- return NegatibleCost::Expensive;
-
- // fold (fneg (fsub A, B)) -> (fsub B, A)
- return NegatibleCost::Neutral;
- case ISD::FMUL:
- case ISD::FDIV: {
- // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
- NegatibleCost V0 = getNegatibleCost(Op.getOperand(0), DAG, LegalOperations,
- ForCodeSize, Depth + 1);
- if (V0 != NegatibleCost::Expensive)
- return V0;
-
- // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
- if (auto *C = isConstOrConstSplatFP(Op.getOperand(1)))
- if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL)
- return NegatibleCost::Expensive;
-
- return getNegatibleCost(Op.getOperand(1), DAG, LegalOperations, ForCodeSize,
- Depth + 1);
- }
- case ISD::FMA:
- case ISD::FMAD: {
- if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
- return NegatibleCost::Expensive;
-
- // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
- // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
- NegatibleCost V2 = getNegatibleCost(Op.getOperand(2), DAG, LegalOperations,
- ForCodeSize, Depth + 1);
- if (NegatibleCost::Expensive == V2)
- return NegatibleCost::Expensive;
-
- // One of Op0/Op1 must be cheaply negatible, then select the cheapest.
- NegatibleCost V0 = getNegatibleCost(Op.getOperand(0), DAG, LegalOperations,
- ForCodeSize, Depth + 1);
- NegatibleCost V1 = getNegatibleCost(Op.getOperand(1), DAG, LegalOperations,
- ForCodeSize, Depth + 1);
- NegatibleCost V01 = std::min(V0, V1);
- if (V01 == NegatibleCost::Expensive)
- return NegatibleCost::Expensive;
- return std::min(V01, V2);
- }
+ OptForSize);
+ });
- case ISD::FP_EXTEND:
- case ISD::FP_ROUND:
- case ISD::FSIN:
- return getNegatibleCost(Op.getOperand(0), DAG, LegalOperations, ForCodeSize,
- Depth + 1);
- }
-
- return NegatibleCost::Expensive;
-}
-
-SDValue TargetLowering::negateExpression(SDValue Op, SelectionDAG &DAG,
- bool LegalOps, bool OptForSize,
- unsigned Depth) const {
- // fneg is removable even if it has multiple uses.
- if (Op.getOpcode() == ISD::FNEG)
- return Op.getOperand(0);
-
- assert(Depth <= SelectionDAG::MaxRecursionDepth &&
- "negateExpression doesn't match getNegatibleCost");
-
- // Pre-increment recursion depth for use in recursive calls.
- ++Depth;
- const SDNodeFlags Flags = Op->getFlags();
- EVT VT = Op.getValueType();
- unsigned Opcode = Op.getOpcode();
- SDLoc DL(Op);
+ if (LegalOps && !IsOpLegal)
+ break;
- switch (Opcode) {
- case ISD::ConstantFP: {
- APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
- V.changeSign();
- return DAG.getConstantFP(V, DL, VT);
- }
- case ISD::BUILD_VECTOR: {
SmallVector<SDValue, 4> Ops;
for (SDValue C : Op->op_values()) {
if (C.isUndef()) {
V.changeSign();
Ops.push_back(DAG.getConstantFP(V, DL, C.getValueType()));
}
+ Cost = NegatibleCost::Neutral;
return DAG.getBuildVector(VT, DL, Ops);
}
case ISD::FADD: {
+ if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
+ break;
+
+ // After operation legalization, it might not be legal to create new FSUBs.
+ if (LegalOps && !isOperationLegalOrCustom(ISD::FSUB, VT))
+ break;
SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
- assert((DAG.getTarget().Options.NoSignedZerosFPMath ||
- Flags.hasNoSignedZeros()) &&
- "Expected NSZ fp-flag");
// fold (fneg (fadd X, Y)) -> (fsub (fneg X), Y)
- NegatibleCost CostX = getNegatibleCost(X, DAG, LegalOps, OptForSize, Depth);
- if (CostX != NegatibleCost::Expensive)
- return DAG.getNode(ISD::FSUB, DL, VT,
- negateExpression(X, DAG, LegalOps, OptForSize, Depth),
- Y, Flags);
-
+ NegatibleCost CostX = NegatibleCost::Expensive;
+ SDValue NegX =
+ getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
// fold (fneg (fadd X, Y)) -> (fsub (fneg Y), X)
- return DAG.getNode(ISD::FSUB, DL, VT,
- negateExpression(Y, DAG, LegalOps, OptForSize, Depth), X,
- Flags);
+ NegatibleCost CostY = NegatibleCost::Expensive;
+ SDValue NegY =
+ getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
+
+ // Negate the X if its cost is less or equal than Y.
+ if (NegX && (CostX <= CostY)) {
+ Cost = CostX;
+ return DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags);
+ }
+
+ // Negate the Y if it is not expensive.
+ if (NegY) {
+ Cost = CostY;
+ return DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags);
+ }
+ break;
}
case ISD::FSUB: {
+ // We can't turn -(A-B) into B-A when we honor signed zeros.
+ if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
+ break;
+
SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
// fold (fneg (fsub 0, Y)) -> Y
if (ConstantFPSDNode *C = isConstOrConstSplatFP(X, /*AllowUndefs*/ true))
- if (C->isZero())
+ if (C->isZero()) {
+ Cost = NegatibleCost::Cheaper;
return Y;
+ }
// fold (fneg (fsub X, Y)) -> (fsub Y, X)
+ Cost = NegatibleCost::Neutral;
return DAG.getNode(ISD::FSUB, DL, VT, Y, X, Flags);
}
case ISD::FMUL:
case ISD::FDIV: {
SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
- // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
- NegatibleCost CostX = getNegatibleCost(X, DAG, LegalOps, OptForSize, Depth);
- if (CostX != NegatibleCost::Expensive)
- return DAG.getNode(Opcode, DL, VT,
- negateExpression(X, DAG, LegalOps, OptForSize, Depth),
- Y, Flags);
+ // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
+ NegatibleCost CostX = NegatibleCost::Expensive;
+ SDValue NegX =
+ getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
// fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
- return DAG.getNode(Opcode, DL, VT, X,
- negateExpression(Y, DAG, LegalOps, OptForSize, Depth),
- Flags);
+ NegatibleCost CostY = NegatibleCost::Expensive;
+ SDValue NegY =
+ getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
+
+ // Negate the X if its cost is less or equal than Y.
+ if (NegX && (CostX <= CostY)) {
+ Cost = CostX;
+ return DAG.getNode(Opcode, DL, VT, NegX, Y, Flags);
+ }
+
+ // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
+ if (auto *C = isConstOrConstSplatFP(Op.getOperand(1)))
+ if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL)
+ break;
+
+ // Negate the Y if it is not expensive.
+ if (NegY) {
+ Cost = CostY;
+ return DAG.getNode(Opcode, DL, VT, X, NegY, Flags);
+ }
+ break;
}
case ISD::FMA:
case ISD::FMAD: {
- assert((DAG.getTarget().Options.NoSignedZerosFPMath ||
- Flags.hasNoSignedZeros()) &&
- "Expected NSZ fp-flag");
+ if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
+ break;
SDValue X = Op.getOperand(0), Y = Op.getOperand(1), Z = Op.getOperand(2);
- SDValue NegZ = negateExpression(Z, DAG, LegalOps, OptForSize, Depth);
- NegatibleCost CostX = getNegatibleCost(X, DAG, LegalOps, OptForSize, Depth);
- NegatibleCost CostY = getNegatibleCost(Y, DAG, LegalOps, OptForSize, Depth);
- if (CostX <= CostY) {
- // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
- SDValue NegX = negateExpression(X, DAG, LegalOps, OptForSize, Depth);
+ NegatibleCost CostZ = NegatibleCost::Expensive;
+ SDValue NegZ =
+ getNegatedExpression(Z, DAG, LegalOps, OptForSize, CostZ, Depth);
+ // Give up if fail to negate the Z.
+ if (!NegZ)
+ break;
+
+ // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
+ NegatibleCost CostX = NegatibleCost::Expensive;
+ SDValue NegX =
+ getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
+ // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
+ NegatibleCost CostY = NegatibleCost::Expensive;
+ SDValue NegY =
+ getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
+
+ // Negate the X if its cost is less or equal than Y.
+ if (NegX && (CostX <= CostY)) {
+ Cost = std::min(CostX, CostZ);
return DAG.getNode(Opcode, DL, VT, NegX, Y, NegZ, Flags);
}
- // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
- SDValue NegY = negateExpression(Y, DAG, LegalOps, OptForSize, Depth);
- return DAG.getNode(Opcode, DL, VT, X, NegY, NegZ, Flags);
+ // Negate the Y if it is not expensive.
+ if (NegY) {
+ Cost = std::min(CostY, CostZ);
+ return DAG.getNode(Opcode, DL, VT, X, NegY, NegZ, Flags);
+ }
+ break;
}
case ISD::FP_EXTEND:
case ISD::FSIN:
- return DAG.getNode(
- Opcode, DL, VT,
- negateExpression(Op.getOperand(0), DAG, LegalOps, OptForSize, Depth));
+ if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
+ OptForSize, Cost, Depth))
+ return DAG.getNode(Opcode, DL, VT, NegV);
+ break;
case ISD::FP_ROUND:
- return DAG.getNode(
- ISD::FP_ROUND, DL, VT,
- negateExpression(Op.getOperand(0), DAG, LegalOps, OptForSize, Depth),
- Op.getOperand(1));
+ if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
+ OptForSize, Cost, Depth))
+ return DAG.getNode(ISD::FP_ROUND, DL, VT, NegV, Op.getOperand(1));
+ break;
}
- llvm_unreachable("Unknown code");
+ return SDValue();
}
//===----------------------------------------------------------------------===//
return SDValue();
}
-TargetLowering::NegatibleCost
-X86TargetLowering::getNegatibleCost(SDValue Op, SelectionDAG &DAG,
- bool LegalOperations, bool ForCodeSize,
- unsigned Depth) const {
+SDValue X86TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
+ bool LegalOperations,
+ bool ForCodeSize,
+ NegatibleCost &Cost,
+ unsigned Depth) const {
// fneg patterns are removable even if they have multiple uses.
- if (isFNEG(DAG, Op.getNode(), Depth))
- return NegatibleCost::Cheaper;
-
- // Don't recurse exponentially.
- if (Depth > SelectionDAG::MaxRecursionDepth)
- return NegatibleCost::Expensive;
-
- EVT VT = Op.getValueType();
- EVT SVT = VT.getScalarType();
- switch (Op.getOpcode()) {
- case ISD::FMA:
- case X86ISD::FMSUB:
- case X86ISD::FNMADD:
- case X86ISD::FNMSUB:
- case X86ISD::FMADD_RND:
- case X86ISD::FMSUB_RND:
- case X86ISD::FNMADD_RND:
- case X86ISD::FNMSUB_RND: {
- if (!Op.hasOneUse() || !Subtarget.hasAnyFMA() || !isTypeLegal(VT) ||
- !(SVT == MVT::f32 || SVT == MVT::f64) ||
- !isOperationLegal(ISD::FMA, VT))
- break;
-
- // This is always negatible for free but we might be able to remove some
- // extra operand negations as well.
- for (int i = 0; i != 3; ++i) {
- NegatibleCost V = getNegatibleCost(Op.getOperand(i), DAG, LegalOperations,
- ForCodeSize, Depth + 1);
- if (V == NegatibleCost::Cheaper)
- return V;
- }
- return NegatibleCost::Neutral;
- }
- case X86ISD::FRCP:
- return getNegatibleCost(Op.getOperand(0), DAG, LegalOperations, ForCodeSize,
- Depth + 1);
- }
-
- return TargetLowering::getNegatibleCost(Op, DAG, LegalOperations, ForCodeSize,
- Depth);
-}
-
-SDValue X86TargetLowering::negateExpression(SDValue Op, SelectionDAG &DAG,
- bool LegalOperations,
- bool ForCodeSize,
- unsigned Depth) const {
- // fneg patterns are removable even if they have multiple uses.
- if (SDValue Arg = isFNEG(DAG, Op.getNode(), Depth))
+ if (SDValue Arg = isFNEG(DAG, Op.getNode(), Depth)) {
+ Cost = NegatibleCost::Cheaper;
return DAG.getBitcast(Op.getValueType(), Arg);
+ }
EVT VT = Op.getValueType();
EVT SVT = VT.getScalarType();
bool NegC = !!NewOps[2];
unsigned NewOpc = negateFMAOpcode(Opc, NegA != NegB, NegC, true);
+ Cost = (NegA || NegB || NegC) ? NegatibleCost::Cheaper
+ : NegatibleCost::Neutral;
+
// Fill in the non-negated ops with the original values.
for (int i = 0, e = Op.getNumOperands(); i != e; ++i)
if (!NewOps[i])
return DAG.getNode(NewOpc, SDLoc(Op), VT, NewOps);
}
case X86ISD::FRCP:
- return DAG.getNode(Opc, SDLoc(Op), VT,
- negateExpression(Op.getOperand(0), DAG, LegalOperations,
- ForCodeSize, Depth + 1));
+ if (SDValue NegOp0 =
+ getNegatedExpression(Op.getOperand(0), DAG, LegalOperations,
+ ForCodeSize, Cost, Depth + 1))
+ return DAG.getNode(Opc, SDLoc(Op), VT, NegOp0);
break;
}
- return TargetLowering::negateExpression(Op, DAG, LegalOperations, ForCodeSize,
- Depth);
+ return TargetLowering::getNegatedExpression(Op, DAG, LegalOperations,
+ ForCodeSize, Cost, Depth);
}
static SDValue lowerX86FPLogicOp(SDNode *N, SelectionDAG &DAG,