case ISD::FNEARBYINT:
case AMDGPUISD::RCP:
case AMDGPUISD::RCP_LEGACY:
+ case AMDGPUISD::RCP_IFLAG:
case AMDGPUISD::SIN_HW:
case AMDGPUISD::FMUL_LEGACY:
case AMDGPUISD::FMIN_LEGACY:
case ISD::FSIN:
case AMDGPUISD::RCP:
case AMDGPUISD::RCP_LEGACY:
+ case AMDGPUISD::RCP_IFLAG:
case AMDGPUISD::SIN_HW: {
SDValue CvtSrc = N0.getOperand(0);
if (CvtSrc.getOpcode() == ISD::FNEG) {
}
}
+SDValue AMDGPUTargetLowering::performRcpCombine(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ const auto *CFP = dyn_cast<ConstantFPSDNode>(N->getOperand(0));
+ if (!CFP)
+ return SDValue();
+
+ // XXX - Should this flush denormals?
+ const APFloat &Val = CFP->getValueAPF();
+ APFloat One(Val.getSemantics(), "1.0");
+ return DCI.DAG.getConstantFP(One / Val, SDLoc(N), N->getValueType(0));
+}
+
SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
return performLoadCombine(N, DCI);
case ISD::STORE:
return performStoreCombine(N, DCI);
- case AMDGPUISD::RCP: {
- if (const auto *CFP = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) {
- // XXX - Should this flush denormals?
- const APFloat &Val = CFP->getValueAPF();
- APFloat One(Val.getSemantics(), "1.0");
- return DAG.getConstantFP(One / Val, SDLoc(N), N->getValueType(0));
- }
-
- break;
- }
+ case AMDGPUISD::RCP:
+ case AMDGPUISD::RCP_IFLAG:
+ return performRcpCombine(N, DCI);
case ISD::AssertZext:
case ISD::AssertSext:
return performAssertSZExtCombine(N, DCI);
NODE_NAME_CASE(RSQ)
NODE_NAME_CASE(RCP_LEGACY)
NODE_NAME_CASE(RSQ_LEGACY)
+ NODE_NAME_CASE(RCP_IFLAG)
NODE_NAME_CASE(FMUL_LEGACY)
NODE_NAME_CASE(RSQ_CLAMP)
NODE_NAME_CASE(LDEXP)
SDValue performSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performFNegCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performFAbsCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+ SDValue performRcpCombine(SDNode *N, DAGCombinerInfo &DCI) const;
static EVT getEquivalentMemType(LLVMContext &Context, EVT VT);
RSQ,
RCP_LEGACY,
RSQ_LEGACY,
+ RCP_IFLAG,
FMUL_LEGACY,
RSQ_CLAMP,
LDEXP,
def AMDGPUrcp_legacy : SDNode<"AMDGPUISD::RCP_LEGACY", SDTFPUnaryOp>;
def AMDGPUrsq_legacy : SDNode<"AMDGPUISD::RSQ_LEGACY", SDTFPUnaryOp>;
+def AMDGPUrcp_iflag : SDNode<"AMDGPUISD::RCP_IFLAG", SDTFPUnaryOp>;
+
// out = 1.0 / sqrt(a) result clamped to +/- max_float.
def AMDGPUrsq_clamp : SDNode<"AMDGPUISD::RSQ_CLAMP", SDTFPUnaryOp>;
case AMDGPUISD::FMAD_FTZ:
case AMDGPUISD::RCP:
case AMDGPUISD::RSQ:
+ case AMDGPUISD::RCP_IFLAG:
case AMDGPUISD::LDEXP:
return true;
default:
return SDValue();
}
+SDValue SITargetLowering::performRcpCombine(SDNode *N,
+ DAGCombinerInfo &DCI) const {
+ EVT VT = N->getValueType(0);
+ SDValue N0 = N->getOperand(0);
+
+ if (N0.isUndef())
+ return N0;
+
+ if (VT == MVT::f32 && (N0.getOpcode() == ISD::UINT_TO_FP ||
+ N0.getOpcode() == ISD::SINT_TO_FP)) {
+ return DCI.DAG.getNode(AMDGPUISD::RCP_IFLAG, SDLoc(N), VT, N0,
+ N->getFlags());
+ }
+
+ return AMDGPUTargetLowering::performRcpCombine(N, DCI);
+}
+
static bool isKnownNeverSNan(SelectionDAG &DAG, SDValue Op) {
if (!DAG.getTargetLoweringInfo().hasFloatingPointExceptions())
return true;
return performClassCombine(N, DCI);
case ISD::FCANONICALIZE:
return performFCanonicalizeCombine(N, DCI);
- case AMDGPUISD::FRACT:
case AMDGPUISD::RCP:
+ return performRcpCombine(N, DCI);
+ case AMDGPUISD::FRACT:
case AMDGPUISD::RSQ:
case AMDGPUISD::RCP_LEGACY:
case AMDGPUISD::RSQ_LEGACY:
+ case AMDGPUISD::RCP_IFLAG:
case AMDGPUISD::RSQ_CLAMP:
case AMDGPUISD::LDEXP: {
SDValue Src = N->getOperand(0);
SDValue performSetCCCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performCvtF32UByteNCombine(SDNode *N, DAGCombinerInfo &DCI) const;
SDValue performClampCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+ SDValue performRcpCombine(SDNode *N, DAGCombinerInfo &DCI) const;
bool isLegalFlatAddressingMode(const AddrMode &AM) const;
bool isLegalGlobalAddressingMode(const AddrMode &AM) const;
defm V_EXP_F32 : VOP1Inst <"v_exp_f32", VOP_F32_F32, fexp2>;
defm V_LOG_F32 : VOP1Inst <"v_log_f32", VOP_F32_F32, flog2>;
defm V_RCP_F32 : VOP1Inst <"v_rcp_f32", VOP_F32_F32, AMDGPUrcp>;
-defm V_RCP_IFLAG_F32 : VOP1Inst <"v_rcp_iflag_f32", VOP_F32_F32>;
+defm V_RCP_IFLAG_F32 : VOP1Inst <"v_rcp_iflag_f32", VOP_F32_F32, AMDGPUrcp_iflag>;
defm V_RSQ_F32 : VOP1Inst <"v_rsq_f32", VOP_F32_F32, AMDGPUrsq>;
defm V_SQRT_F32 : VOP1Inst <"v_sqrt_f32", VOP_F32_F32, fsqrt>;
} // End SchedRW = [WriteQuarterRate32]
--- /dev/null
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck --check-prefix=GCN %s
+
+; GCN-LABEL: {{^}}rcp_uint:
+; GCN: v_rcp_iflag_f32_e32
+define amdgpu_kernel void @rcp_uint(i32 addrspace(1)* %in, float addrspace(1)* %out) {
+ %load = load i32, i32 addrspace(1)* %in, align 4
+ %cvt = uitofp i32 %load to float
+ %div = fdiv float 1.000000e+00, %cvt
+ store float %div, float addrspace(1)* %out, align 4
+ ret void
+}
+
+; GCN-LABEL: {{^}}rcp_sint:
+; GCN: v_rcp_iflag_f32_e32
+define amdgpu_kernel void @rcp_sint(i32 addrspace(1)* %in, float addrspace(1)* %out) {
+ %load = load i32, i32 addrspace(1)* %in, align 4
+ %cvt = sitofp i32 %load to float
+ %div = fdiv float 1.000000e+00, %cvt
+ store float %div, float addrspace(1)* %out, align 4
+ ret void
+}
}
; FUNC-LABEL: {{^}}v_sdiv_i8:
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_bfe_i32 [[BFE:v[0-9]+]], v{{[0-9]+}}, 0, 8
; SI: buffer_store_dword [[BFE]]
define amdgpu_kernel void @v_sdiv_i8(i32 addrspace(1)* %out, i8 addrspace(1)* %in) {
}
; FUNC-LABEL: {{^}}v_sdiv_i23:
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_bfe_i32 [[BFE:v[0-9]+]], v{{[0-9]+}}, 0, 23
; SI: buffer_store_dword [[BFE]]
define amdgpu_kernel void @v_sdiv_i23(i32 addrspace(1)* %out, i23 addrspace(1)* %in) {
}
; FUNC-LABEL: {{^}}v_sdiv_i24:
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_bfe_i32 [[BFE:v[0-9]+]], v{{[0-9]+}}, 0, 24
; SI: buffer_store_dword [[BFE]]
define amdgpu_kernel void @v_sdiv_i24(i32 addrspace(1)* %out, i24 addrspace(1)* %in) {
; FUNC-LABEL: {{^}}sdiv24_i8:
; SI: v_cvt_f32_i32
; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_cvt_i32_f32
; EG: INT_TO_FLT
; FUNC-LABEL: {{^}}sdiv24_i16:
; SI: v_cvt_f32_i32
; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_cvt_i32_f32
; EG: INT_TO_FLT
; FUNC-LABEL: {{^}}sdiv24_i32:
; SI: v_cvt_f32_i32
; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_cvt_i32_f32
; EG: INT_TO_FLT
; FUNC-LABEL: {{^}}srem24_i8:
; SI: v_cvt_f32_i32
; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_cvt_i32_f32
; EG: INT_TO_FLT
; FUNC-LABEL: {{^}}srem24_i16:
; SI: v_cvt_f32_i32
; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_cvt_i32_f32
; EG: INT_TO_FLT
; FUNC-LABEL: {{^}}srem24_i32:
; SI: v_cvt_f32_i32
; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_cvt_i32_f32
; EG: INT_TO_FLT
; FUNC-LABEL: {{^}}srem25_i24_i11_i32:
; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 24
; EG: INT_TO_FLT
; FUNC-LABEL: {{^}}srem25_i11_i24_i32:
; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 24
; EG: INT_TO_FLT
; FUNC-LABEL: {{^}}srem25_i17_i12_i32:
; SI: v_cvt_f32_i32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 17
; EG: INT_TO_FLT
}
; FUNC-LABEL: {{^}}v_udiv_i8:
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_and_b32_e32 [[TRUNC:v[0-9]+]], 0xff, v{{[0-9]+}}
; SI: buffer_store_dword [[TRUNC]]
define amdgpu_kernel void @v_udiv_i8(i32 addrspace(1)* %out, i8 addrspace(1)* %in) {
}
; FUNC-LABEL: {{^}}v_udiv_i16:
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_and_b32_e32 [[TRUNC:v[0-9]+]], 0xffff, v{{[0-9]+}}
; SI: buffer_store_dword [[TRUNC]]
define amdgpu_kernel void @v_udiv_i16(i32 addrspace(1)* %out, i16 addrspace(1)* %in) {
}
; FUNC-LABEL: {{^}}v_udiv_i23:
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_and_b32_e32 [[TRUNC:v[0-9]+]], 0x7fffff, v{{[0-9]+}}
; SI: buffer_store_dword [[TRUNC]]
define amdgpu_kernel void @v_udiv_i23(i32 addrspace(1)* %out, i23 addrspace(1)* %in) {
; FUNC-LABEL: {{^}}udiv24_i8:
; SI: v_cvt_f32_ubyte
; SI: v_cvt_f32_ubyte
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_cvt_u32_f32
; EG: UINT_TO_FLT
; FUNC-LABEL: {{^}}udiv24_i16:
; SI: v_cvt_f32_u32
; SI: v_cvt_f32_u32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_cvt_u32_f32
; EG: UINT_TO_FLT
; FUNC-LABEL: {{^}}udiv23_i32:
; SI: v_cvt_f32_u32
; SI-DAG: v_cvt_f32_u32
-; SI-DAG: v_rcp_f32
+; SI-DAG: v_rcp_iflag_f32
; SI: v_cvt_u32_f32
; EG: UINT_TO_FLT
; FUNC-LABEL: {{^}}urem24_i8:
; SI: v_cvt_f32_ubyte
; SI: v_cvt_f32_ubyte
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_cvt_u32_f32
; EG: UINT_TO_FLT
; FUNC-LABEL: {{^}}urem24_i16:
; SI: v_cvt_f32_u32
; SI: v_cvt_f32_u32
-; SI: v_rcp_f32
+; SI: v_rcp_iflag_f32
; SI: v_cvt_u32_f32
; EG: UINT_TO_FLT
}
; FUNC-LABEL: {{^}}test_udiv24_u16_u23_i32:
-; SI-DAG: v_rcp_f32
+; SI-DAG: v_rcp_iflag_f32
; SI-DAG: s_mov_b32 [[MASK:s[0-9]+]], 0x7fffff{{$}}
; SI: v_and_b32_e32 v{{[0-9]+}}, [[MASK]],
}
; FUNC-LABEL: {{^}}test_udiv24_u23_u16_i32:
-; SI-DAG: v_rcp_f32
+; SI-DAG: v_rcp_iflag_f32
; SI-DAG: s_mov_b32 [[MASK:s[0-9]+]], 0x7fffff{{$}}
; SI: v_and_b32_e32 v{{[0-9]+}}, [[MASK]],