From e9f2bac9a07d4104d49b63408014d4e036c614ac Mon Sep 17 00:00:00 2001 From: Michael Maitland Date: Thu, 3 Nov 2022 10:27:32 -0700 Subject: [PATCH] [RISCV][Codegen] Account for LMUL in Vector floating-point instructions It is likley that subtargets act differently for vector floating-point instructions based on the LMUL. This patch creates seperate SchedRead, SchedWrite, WriteRes, ReadAdvance for each relevant LMUL. Differential Revision: https://reviews.llvm.org/D137426 --- llvm/lib/Target/RISCV/RISCVInstrInfoV.td | 101 +++-- llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td | 545 ++++++++++++++++++------ llvm/lib/Target/RISCV/RISCVScheduleV.td | 283 ++++++------ 3 files changed, 627 insertions(+), 302 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td index 76a15e7..57f5fe3 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td @@ -523,135 +523,167 @@ multiclass VALUNoVm_IV_V_X funct6> { multiclass VALU_FV_V_F funct6, string vw = "v"> { def V : VALUVV, - Sched<[WriteVFALUV, ReadVFALUV, ReadVFALUV, ReadVMask]>; + Sched<[WriteVFALUV_UpperBound, ReadVFALUV_UpperBound, + ReadVFALUV_UpperBound, ReadVMask]>; def F : VALUVF, - Sched<[WriteVFALUF, ReadVFALUV, ReadVFALUF, ReadVMask]>; + Sched<[WriteVFALUF_UpperBound, ReadVFALUV_UpperBound, + ReadVFALUF_UpperBound, ReadVMask]>; } multiclass VALU_FV_F funct6, string vw = "v"> { def F : VALUVF, - Sched<[WriteVFALUF, ReadVFALUV, ReadVFALUF, ReadVMask]>; + Sched<[WriteVFALUF_UpperBound, ReadVFALUV_UpperBound, + ReadVFALUF_UpperBound, ReadVMask]>; } multiclass VWALU_FV_V_F funct6, string vw = "v"> { def V : VALUVV, - Sched<[WriteVFWALUV, ReadVFWALUV, ReadVFWALUV, ReadVMask]>; + Sched<[WriteVFWALUV_UpperBound, ReadVFWALUV_UpperBound, + ReadVFWALUV_UpperBound, ReadVMask]>; def F : VALUVF, - Sched<[WriteVFWALUF, ReadVFWALUV, ReadVFWALUF, ReadVMask]>; + Sched<[WriteVFWALUF_UpperBound, ReadVFWALUV_UpperBound, + ReadVFWALUF_UpperBound, ReadVMask]>; } multiclass VMUL_FV_V_F funct6, string vw = "v"> { def V : VALUVV, - Sched<[WriteVFMulV, ReadVFMulV, ReadVFMulV, ReadVMask]>; + Sched<[WriteVFMulV_UpperBound, ReadVFMulV_UpperBound, + ReadVFMulV_UpperBound, ReadVMask]>; def F : VALUVF, - Sched<[WriteVFMulF, ReadVFMulV, ReadVFMulF, ReadVMask]>; + Sched<[WriteVFMulF_UpperBound, ReadVFMulV_UpperBound, + ReadVFMulF_UpperBound, ReadVMask]>; } multiclass VDIV_FV_V_F funct6, string vw = "v"> { def V : VALUVV, - Sched<[WriteVFDivV, ReadVFDivV, ReadVFDivV, ReadVMask]>; + Sched<[WriteVFDivV_UpperBound, ReadVFDivV_UpperBound, + ReadVFDivV_UpperBound, ReadVMask]>; def F : VALUVF, - Sched<[WriteVFDivF, ReadVFDivV, ReadVFDivF, ReadVMask]>; + Sched<[WriteVFDivF_UpperBound, ReadVFDivV_UpperBound, + ReadVFDivF_UpperBound, ReadVMask]>; } multiclass VRDIV_FV_F funct6, string vw = "v"> { def F : VALUVF, - Sched<[WriteVFDivF, ReadVFDivV, ReadVFDivF, ReadVMask]>; + Sched<[WriteVFDivF_UpperBound, ReadVFDivV_UpperBound, + ReadVFDivF_UpperBound, ReadVMask]>; } multiclass VWMUL_FV_V_F funct6, string vw = "v"> { def V : VALUVV, - Sched<[WriteVFWMulV, ReadVFWMulV, ReadVFWMulV, ReadVMask]>; + Sched<[WriteVFWMulV_UpperBound, ReadVFWMulV_UpperBound, + ReadVFWMulV_UpperBound, ReadVMask]>; def F : VALUVF, - Sched<[WriteVFWMulF, ReadVFWMulV, ReadVFWMulF, ReadVMask]>; + Sched<[WriteVFWMulF_UpperBound, ReadVFWMulV_UpperBound, + ReadVFWMulF_UpperBound, ReadVMask]>; } multiclass VMAC_FV_V_F funct6, string vw = "v"> { def V : VALUrVV, - Sched<[WriteVFMulAddV, ReadVFMulAddV, ReadVFMulAddV, ReadVMask]>; + Sched<[WriteVFMulAddV_UpperBound, ReadVFMulAddV_UpperBound, + ReadVFMulAddV_UpperBound, ReadVMask]>; def F : VALUrVF, - Sched<[WriteVFMulAddF, ReadVFMulAddV, ReadVFMulAddF, ReadVMask]>; + Sched<[WriteVFMulAddF_UpperBound, ReadVFMulAddV_UpperBound, + ReadVFMulAddF_UpperBound, ReadVMask]>; } multiclass VWMAC_FV_V_F funct6, string vw = "v"> { def V : VALUrVV, - Sched<[WriteVFWMulAddV, ReadVFWMulAddV, ReadVFWMulAddV, ReadVMask]>; + Sched<[WriteVFWMulAddV_UpperBound, ReadVFWMulAddV_UpperBound, + ReadVFWMulAddV_UpperBound, ReadVMask]>; def F : VALUrVF, - Sched<[WriteVFWMulAddF, ReadVFWMulAddV, ReadVFWMulAddF, ReadVMask]>; + Sched<[WriteVFWMulAddF_UpperBound, ReadVFWMulAddV_UpperBound, + ReadVFWMulAddF_UpperBound, ReadVMask]>; } multiclass VSQR_FV_VS2 funct6, bits<5> vs1> { def "" : VALUVs2, - Sched<[WriteVFSqrtV, ReadVFSqrtV, ReadVMask]>; + Sched<[WriteVFSqrtV_UpperBound, ReadVFSqrtV_UpperBound, + ReadVMask]>; } multiclass VRCP_FV_VS2 funct6, bits<5> vs1> { def "" : VALUVs2, - Sched<[WriteVFRecpV, ReadVFRecpV, ReadVMask]>; + Sched<[WriteVFRecpV_UpperBound, ReadVFRecpV_UpperBound, + ReadVMask]>; } multiclass VCMP_FV_V_F funct6, string vw = "v"> { def V : VALUVV, - Sched<[WriteVFCmpV, ReadVFCmpV, ReadVFCmpV, ReadVMask]>; + Sched<[WriteVFCmpV_UpperBound, ReadVFCmpV_UpperBound, + ReadVFCmpV_UpperBound, ReadVMask]>; def F : VALUVF, - Sched<[WriteVFCmpF, ReadVFCmpV, ReadVFCmpF, ReadVMask]>; + Sched<[WriteVFCmpF_UpperBound, ReadVFCmpV_UpperBound, + ReadVFCmpF_UpperBound, ReadVMask]>; } multiclass VCMP_FV_F funct6, string vw = "v"> { def F : VALUVF, - Sched<[WriteVFCmpF, ReadVFCmpV, ReadVFCmpF, ReadVMask]>; + Sched<[WriteVFCmpF_UpperBound, ReadVFCmpV_UpperBound, + ReadVFCmpF_UpperBound, ReadVMask]>; } multiclass VSGNJ_FV_V_F funct6, string vw = "v"> { def V : VALUVV, - Sched<[WriteVFSgnjV, ReadVFSgnjV, ReadVFSgnjV, ReadVMask]>; + Sched<[WriteVFSgnjV_UpperBound, ReadVFSgnjV_UpperBound, + ReadVFSgnjV_UpperBound, ReadVMask]>; def F : VALUVF, - Sched<[WriteVFSgnjF, ReadVFSgnjV, ReadVFSgnjF, ReadVMask]>; + Sched<[WriteVFSgnjF_UpperBound, ReadVFSgnjV_UpperBound, + ReadVFSgnjF_UpperBound, ReadVMask]>; } multiclass VCLS_FV_VS2 funct6, bits<5> vs1> { def "" : VALUVs2, - Sched<[WriteVFClassV, ReadVFClassV, ReadVMask]>; + Sched<[WriteVFClassV_UpperBound, ReadVFClassV_UpperBound, + ReadVMask]>; } multiclass VCVTF_IV_VS2 funct6, bits<5> vs1> { def "" : VALUVs2, - Sched<[WriteVFCvtIToFV, ReadVFCvtIToFV, ReadVMask]>; + Sched<[WriteVFCvtIToFV_UpperBound, ReadVFCvtIToFV_UpperBound, + ReadVMask]>; } multiclass VCVTI_FV_VS2 funct6, bits<5> vs1> { def "" : VALUVs2, - Sched<[WriteVFCvtFToIV, ReadVFCvtFToIV, ReadVMask]>; + Sched<[WriteVFCvtFToIV_UpperBound, ReadVFCvtFToIV_UpperBound, + ReadVMask]>; } multiclass VWCVTF_IV_VS2 funct6, bits<5> vs1> { def "" : VALUVs2, - Sched<[WriteVFWCvtIToFV, ReadVFWCvtIToFV, ReadVMask]>; + Sched<[WriteVFWCvtIToFV_UpperBound, ReadVFWCvtIToFV_UpperBound, + ReadVMask]>; } multiclass VWCVTI_FV_VS2 funct6, bits<5> vs1> { def "" : VALUVs2, - Sched<[WriteVFWCvtFToIV, ReadVFWCvtFToIV, ReadVMask]>; + Sched<[WriteVFWCvtFToIV_UpperBound, ReadVFWCvtFToIV_UpperBound, + ReadVMask]>; } multiclass VWCVTF_FV_VS2 funct6, bits<5> vs1> { def "" : VALUVs2, - Sched<[WriteVFWCvtFToFV, ReadVFWCvtFToFV, ReadVMask]>; + Sched<[WriteVFWCvtFToFV_UpperBound, ReadVFWCvtFToFV_UpperBound, + ReadVMask]>; } multiclass VNCVTF_IV_VS2 funct6, bits<5> vs1> { def "" : VALUVs2, - Sched<[WriteVFNCvtIToFV, ReadVFNCvtIToFV, ReadVMask]>; + Sched<[WriteVFNCvtIToFV_UpperBound, ReadVFNCvtIToFV_UpperBound, + ReadVMask]>; } multiclass VNCVTI_FV_VS2 funct6, bits<5> vs1> { def "" : VALUVs2, - Sched<[WriteVFNCvtFToIV, ReadVFNCvtFToIV, ReadVMask]>; + Sched<[WriteVFNCvtFToIV_UpperBound, ReadVFNCvtFToIV_UpperBound, + ReadVMask]>; } multiclass VNCVTF_FV_VS2 funct6, bits<5> vs1> { def "" : VALUVs2, - Sched<[WriteVFNCvtFToFV, ReadVFNCvtFToFV, ReadVMask]>; + Sched<[WriteVFNCvtFToFV_UpperBound, ReadVFNCvtFToFV_UpperBound, + ReadVMask]>; } multiclass VRED_MV_V funct6> { @@ -1337,14 +1369,15 @@ let vm = 0 in def VFMERGE_VFM : RVInstVX<0b010111, OPFVF, (outs VR:$vd), (ins VR:$vs2, FPR32:$rs1, VMV0:$v0), "vfmerge.vfm", "$vd, $vs2, $rs1, v0">, - Sched<[WriteVFMergeV, ReadVFMergeV, ReadVFMergeF, ReadVMask]>; + Sched<[WriteVFMergeV_UpperBound, ReadVFMergeV_UpperBound, + ReadVFMergeF_UpperBound, ReadVMask]>; // Vector Floating-Point Move Instruction let RVVConstraint = NoConstraint in let vm = 1, vs2 = 0 in def VFMV_V_F : RVInstVX<0b010111, OPFVF, (outs VR:$vd), (ins FPR32:$rs1), "vfmv.v.f", "$vd, $rs1">, - Sched<[WriteVFMovV, ReadVFMovF]>; + Sched<[WriteVFMovV_UpperBound, ReadVFMovF_UpperBound]>; } // hasSideEffects = 0, mayLoad = 0, mayStore = 0 diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index 39dff11..aae4db8 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -1944,9 +1944,8 @@ multiclass VPseudoBinaryV_VV_LMUL { } // Similar to VPseudoBinaryV_VV, but uses MxListF. -multiclass VPseudoBinaryFV_VV { - foreach m = MxListF in - defm _VV : VPseudoBinary; +multiclass VPseudoBinaryFV_VV { + defm _VV : VPseudoBinary; } multiclass VPseudoVGTR_VV_EEW { @@ -1980,11 +1979,9 @@ multiclass VPseudoVSLD1_VX { Sched<[WriteVISlide1X, ReadVISlideV, ReadVISlideX, ReadVMask]>; } -multiclass VPseudoBinaryV_VF { - foreach f = FPList in - foreach m = f.MxList in - defm "_V" # f.FX : VPseudoBinary; +multiclass VPseudoBinaryV_VF { + defm "_V" # f.FX : VPseudoBinary; } multiclass VPseudoVSLD1_VF { @@ -2035,12 +2032,10 @@ multiclass VPseudoBinaryW_VX_LMUL { "@earlyclobber $rd">; } -multiclass VPseudoBinaryW_VF { - foreach f = FPListW in - foreach m = f.MxListFW in - defm "_V" # f.FX : VPseudoBinary; +multiclass VPseudoBinaryW_VF { + defm "_V" # f.FX : VPseudoBinary; } multiclass VPseudoBinaryW_WV mxlist = MxListW> { @@ -2063,11 +2058,9 @@ multiclass VPseudoBinaryW_WX_LMUL { defm "_WX" : VPseudoBinary; } -multiclass VPseudoBinaryW_WF { - foreach f = FPListW in - foreach m = f.MxListFW in - defm "_W" # f.FX : VPseudoBinary; +multiclass VPseudoBinaryW_WF { + defm "_W" # f.FX : VPseudoBinary; } // Narrowing instructions like vnsrl/vnsra/vnclip(u) don't need @earlyclobber @@ -2148,18 +2141,24 @@ multiclass VPseudoTiedBinaryV_XM("WriteVFMergeV_" # mx); + defvar ReadVFMergeV_MX = !cast("ReadVFMergeV_" # mx); + defvar ReadVFMergeF_MX = !cast("ReadVFMergeF_" # mx); + + def "_V" # f.FX # "M_" # mx : VPseudoBinaryCarryIn.R, m.vrclass, f.fprclass, m, /*CarryIn=*/1, "">, - Sched<[WriteVFMergeV, ReadVFMergeV, ReadVFMergeF, ReadVMask]>; + Sched<[WriteVFMergeV_MX, ReadVFMergeV_MX, ReadVFMergeF_MX, ReadVMask]>; // Tied version to allow codegen control over the tail elements - def "_V" # f.FX # "M_" # m.MX # "_TU": + def "_V" # f.FX # "M_" # mx # "_TU": VPseudoTiedBinaryCarryIn.R, m.vrclass, f.fprclass, m, /*CarryIn=*/1, "">, - Sched<[WriteVFMergeV, ReadVFMergeV, ReadVFMergeF, ReadVMask]>; + Sched<[WriteVFMergeV_MX, ReadVFMergeV_MX, ReadVFMergeF_MX, ReadVMask]>; } + } } multiclass VPseudoBinaryV_IM("WriteVFMovV_" # mx); + defvar ReadVFMovF_MX = !cast("ReadVFMovF_" # mx); + let VLMul = m.value in { - def "_" # f.FX # "_" # m.MX : + def "_" # f.FX # "_" # mx : VPseudoUnaryNoDummyMask, - Sched<[WriteVFMovV, ReadVFMovF]>; - def "_" # f.FX # "_" # m.MX # "_TU": + Sched<[WriteVFMovV_MX, ReadVFMovF_MX]>; + def "_" # f.FX # "_" # mx # "_TU": VPseudoUnaryNoDummyMaskTU, - Sched<[WriteVFMovV, ReadVFMovF]>; + Sched<[WriteVFMovV_MX, ReadVFMovF_MX]>; } } } @@ -2225,42 +2228,54 @@ multiclass VPseudoVMV_F { multiclass VPseudoVCLS_V { foreach m = MxListF in { + defvar mx = m.MX; + defvar WriteVFClassV_MX = !cast("WriteVFClassV_" # mx); + defvar ReadVFClassV_MX = !cast("ReadVFClassV_" # mx); + let VLMul = m.value in { - def "_V_" # m.MX : VPseudoUnaryNoMask, - Sched<[WriteVFClassV, ReadVFClassV, ReadVMask]>; - def "_V_" # m.MX # "_TU": VPseudoUnaryNoMaskTU, - Sched<[WriteVFClassV, ReadVFClassV, ReadVMask]>; - def "_V_" # m.MX # "_MASK" : VPseudoUnaryMaskTA, - RISCVMaskedPseudo, - Sched<[WriteVFClassV, ReadVFClassV, ReadVMask]>; + def "_V_" # mx : VPseudoUnaryNoMask, + Sched<[WriteVFClassV_MX, ReadVFClassV_MX, ReadVMask]>; + def "_V_" # mx # "_TU": VPseudoUnaryNoMaskTU, + Sched<[WriteVFClassV_MX, ReadVFClassV_MX, ReadVMask]>; + def "_V_" # mx # "_MASK" : VPseudoUnaryMaskTA, + RISCVMaskedPseudo, + Sched<[WriteVFClassV_MX, ReadVFClassV_MX, ReadVMask]>; } } } multiclass VPseudoVSQR_V { foreach m = MxListF in { + defvar mx = m.MX; + defvar WriteVFSqrtV_MX = !cast("WriteVFSqrtV_" # mx); + defvar ReadVFSqrtV_MX = !cast("ReadVFSqrtV_" # mx); + let VLMul = m.value in { - def "_V_" # m.MX : VPseudoUnaryNoMask, - Sched<[WriteVFSqrtV, ReadVFSqrtV, ReadVMask]>; - def "_V_" # m.MX # "_TU": VPseudoUnaryNoMaskTU, - Sched<[WriteVFSqrtV, ReadVFSqrtV, ReadVMask]>; - def "_V_" # m.MX # "_MASK" : VPseudoUnaryMaskTA, - RISCVMaskedPseudo, - Sched<[WriteVFSqrtV, ReadVFSqrtV, ReadVMask]>; + def "_V_" # mx : VPseudoUnaryNoMask, + Sched<[WriteVFSqrtV_MX, ReadVFSqrtV_MX, ReadVMask]>; + def "_V_" # mx # "_TU": VPseudoUnaryNoMaskTU, + Sched<[WriteVFSqrtV_MX, ReadVFSqrtV_MX, ReadVMask]>; + def "_V_" # mx # "_MASK" : VPseudoUnaryMaskTA, + RISCVMaskedPseudo, + Sched<[WriteVFSqrtV_MX, ReadVFSqrtV_MX, ReadVMask]>; } } } multiclass VPseudoVRCP_V { foreach m = MxListF in { + defvar mx = m.MX; + defvar WriteVFRecpV_MX = !cast("WriteVFRecpV_" # mx); + defvar ReadVFRecpV_MX = !cast("ReadVFRecpV_" # mx); + let VLMul = m.value in { - def "_V_" # m.MX : VPseudoUnaryNoMask, - Sched<[WriteVFRecpV, ReadVFRecpV, ReadVMask]>; - def "_V_" # m.MX # "_TU": VPseudoUnaryNoMaskTU, - Sched<[WriteVFRecpV, ReadVFRecpV, ReadVMask]>; - def "_V_" # m.MX # "_MASK" : VPseudoUnaryMaskTA, - RISCVMaskedPseudo, - Sched<[WriteVFRecpV, ReadVFRecpV, ReadVMask]>; + def "_V_" # mx : VPseudoUnaryNoMask, + Sched<[WriteVFRecpV_MX, ReadVFRecpV_MX, ReadVMask]>; + def "_V_" # mx # "_TU": VPseudoUnaryNoMaskTU, + Sched<[WriteVFRecpV_MX, ReadVFRecpV_MX, ReadVMask]>; + def "_V_" # mx # "_MASK" : VPseudoUnaryMaskTA, + RISCVMaskedPseudo, + Sched<[WriteVFRecpV_MX, ReadVFRecpV_MX, ReadVMask]>; } } } @@ -2275,9 +2290,9 @@ multiclass PseudoVEXT_VF2 { let VLMul = m.value in { def "_" # mx : VPseudoUnaryNoMask, - Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; + Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; def "_" # mx # "_TU": VPseudoUnaryNoMaskTU, - Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; + Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; def "_" # mx # "_MASK" : VPseudoUnaryMaskTA, RISCVMaskedPseudo, @@ -2296,9 +2311,9 @@ multiclass PseudoVEXT_VF4 { let VLMul = m.value in { def "_" # mx : VPseudoUnaryNoMask, - Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; + Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; def "_" # mx # "_TU": VPseudoUnaryNoMaskTU, - Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; + Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; def "_" # mx # "_MASK" : VPseudoUnaryMaskTA, RISCVMaskedPseudo, @@ -2317,9 +2332,9 @@ multiclass PseudoVEXT_VF8 { let VLMul = m.value in { def "_" # mx : VPseudoUnaryNoMask, - Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; + Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; def "_" # mx # "_TU": VPseudoUnaryNoMaskTU, - Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; + Sched<[WriteVExtV_MX, ReadVExtV_MX, ReadVMask]>; def "_" # mx # "_MASK" : VPseudoUnaryMaskTA, RISCVMaskedPseudo, @@ -2356,12 +2371,10 @@ multiclass VPseudoBinaryM_VX { !if(!ge(m.octuple, 16), "@earlyclobber $rd", "")>; } -multiclass VPseudoBinaryM_VF { - foreach f = FPList in - foreach m = f.MxList in - defm "_V" # f.FX : - VPseudoBinaryM; +multiclass VPseudoBinaryM_VF { + defm "_V" # f.FX : + VPseudoBinaryM; } multiclass VPseudoBinaryM_VI { @@ -2542,22 +2555,62 @@ multiclass VPseudoVDIV_VV_VX { } multiclass VPseudoVFMUL_VV_VF { - defm "" : VPseudoBinaryFV_VV, - Sched<[WriteVFMulV, ReadVFMulV, ReadVFMulV, ReadVMask]>; - defm "" : VPseudoBinaryV_VF, - Sched<[WriteVFMulF, ReadVFMulV, ReadVFMulF, ReadVMask]>; + foreach m = MxListF in { + defvar mx = m.MX; + defvar WriteVFMulV_MX = !cast("WriteVFMulV_" # mx); + defvar ReadVFMulV_MX = !cast("ReadVFMulV_" # mx); + + defm "" : VPseudoBinaryFV_VV, + Sched<[WriteVFMulV_MX, ReadVFMulV_MX, ReadVFMulV_MX, ReadVMask]>; + } + + foreach f = FPList in { + foreach m = f.MxList in { + defvar mx = m.MX; + defvar WriteVFMulF_MX = !cast("WriteVFMulF_" # mx); + defvar ReadVFMulV_MX = !cast("ReadVFMulV_" # mx); + defvar ReadVFMulF_MX = !cast("ReadVFMulF_" # mx); + + defm "" : VPseudoBinaryV_VF, + Sched<[WriteVFMulF_MX, ReadVFMulV_MX, ReadVFMulF_MX, ReadVMask]>; + } + } } multiclass VPseudoVFDIV_VV_VF { - defm "" : VPseudoBinaryFV_VV, - Sched<[WriteVFDivV, ReadVFDivV, ReadVFDivV, ReadVMask]>; - defm "" : VPseudoBinaryV_VF, - Sched<[WriteVFDivF, ReadVFDivV, ReadVFDivF, ReadVMask]>; + foreach m = MxListF in { + defvar mx = m.MX; + defvar WriteVFDivV_MX = !cast("WriteVFDivV_" # mx); + defvar ReadVFDivV_MX = !cast("ReadVFDivV_" # mx); + + defm "" : VPseudoBinaryFV_VV, + Sched<[WriteVFDivV_MX, ReadVFDivV_MX, ReadVFDivV_MX, ReadVMask]>; + } + + foreach f = FPList in { + foreach m = f.MxList in { + defvar mx = m.MX; + defvar WriteVFDivF_MX = !cast("WriteVFDivF_" # mx); + defvar ReadVFDivV_MX = !cast("ReadVFDivV_" # mx); + defvar ReadVFDivF_MX = !cast("ReadVFDivF_" # mx); + + defm "" : VPseudoBinaryV_VF, + Sched<[WriteVFDivF_MX, ReadVFDivV_MX, ReadVFDivF_MX, ReadVMask]>; + } + } } multiclass VPseudoVFRDIV_VF { - defm "" : VPseudoBinaryV_VF, - Sched<[WriteVFDivF, ReadVFDivV, ReadVFDivF, ReadVMask]>; + foreach f = FPList in { + foreach m = f.MxList in { + defvar mx = m.MX; + defvar WriteVFDivF_MX = !cast("WriteVFDivF_" # mx); + defvar ReadVFDivV_MX = !cast("ReadVFDivV_" # mx); + defvar ReadVFDivF_MX = !cast("ReadVFDivF_" # mx); + defm "" : VPseudoBinaryV_VF, + Sched<[WriteVFDivF_MX, ReadVFDivV_MX, ReadVFDivF_MX, ReadVMask]>; + } + } } multiclass VPseudoVALU_VV_VX { @@ -2576,29 +2629,85 @@ multiclass VPseudoVALU_VV_VX { } multiclass VPseudoVSGNJ_VV_VF { - defm "" : VPseudoBinaryFV_VV, - Sched<[WriteVFSgnjV, ReadVFSgnjV, ReadVFSgnjV, ReadVMask]>; - defm "" : VPseudoBinaryV_VF, - Sched<[WriteVFSgnjF, ReadVFSgnjV, ReadVFSgnjF, ReadVMask]>; + foreach m = MxListF in { + defvar mx = m.MX; + defvar WriteVFSgnjV_MX = !cast("WriteVFSgnjV_" # mx); + defvar ReadVFSgnjV_MX = !cast("ReadVFSgnjV_" # mx); + + defm "" : VPseudoBinaryFV_VV, + Sched<[WriteVFSgnjV_MX, ReadVFSgnjV_MX, ReadVFSgnjV_MX, ReadVMask]>; + } + + foreach f = FPList in { + foreach m = f.MxList in { + defvar mx = m.MX; + defvar WriteVFSgnjF_MX = !cast("WriteVFSgnjF_" # mx); + defvar ReadVFSgnjV_MX = !cast("ReadVFSgnjV_" # mx); + defvar ReadVFSgnjF_MX = !cast("ReadVFSgnjF_" # mx); + + defm "" : VPseudoBinaryV_VF, + Sched<[WriteVFSgnjF_MX, ReadVFSgnjV_MX, ReadVFSgnjF_MX, ReadVMask]>; + } + } } multiclass VPseudoVMAX_VV_VF { - defm "" : VPseudoBinaryFV_VV, - Sched<[WriteVFCmpV, ReadVFCmpV, ReadVFCmpV, ReadVMask]>; - defm "" : VPseudoBinaryV_VF, - Sched<[WriteVFCmpF, ReadVFCmpV, ReadVFCmpF, ReadVMask]>; + foreach m = MxListF in { + defvar mx = m.MX; + defvar WriteVFCmpV_MX = !cast("WriteVFCmpV_" # mx); + defvar ReadVFCmpV_MX = !cast("ReadVFCmpV_" # mx); + + defm "" : VPseudoBinaryFV_VV, + Sched<[WriteVFCmpV_MX, ReadVFCmpV_MX, ReadVFCmpV_MX, ReadVMask]>; + } + + foreach f = FPList in { + foreach m = f.MxList in { + defvar mx = m.MX; + defvar WriteVFCmpF_MX = !cast("WriteVFCmpF_" # mx); + defvar ReadVFCmpV_MX = !cast("ReadVFCmpV_" # mx); + defvar ReadVFCmpF_MX = !cast("ReadVFCmpF_" # mx); + + defm "" : VPseudoBinaryV_VF, + Sched<[WriteVFCmpF_MX, ReadVFCmpV_MX, ReadVFCmpF_MX, ReadVMask]>; + } + } } multiclass VPseudoVALU_VV_VF { - defm "" : VPseudoBinaryFV_VV, - Sched<[WriteVFALUV, ReadVFALUV, ReadVFALUV, ReadVMask]>; - defm "" : VPseudoBinaryV_VF, - Sched<[WriteVFALUF, ReadVFALUV, ReadVFALUF, ReadVMask]>; + foreach m = MxListF in { + defvar mx = m.MX; + defvar WriteVFALUV_MX = !cast("WriteVFALUV_" # mx); + defvar ReadVFALUV_MX = !cast("ReadVFALUV_" # mx); + + defm "" : VPseudoBinaryFV_VV, + Sched<[WriteVFALUV_MX, ReadVFALUV_MX, ReadVFALUV_MX, ReadVMask]>; + } + + foreach f = FPList in { + foreach m = f.MxList in { + defvar mx = m.MX; + defvar WriteVFALUF_MX = !cast("WriteVFALUF_" # mx); + defvar ReadVFALUV_MX = !cast("ReadVFALUV_" # mx); + defvar ReadVFALUF_MX = !cast("ReadVFALUF_" # mx); + defm "" : VPseudoBinaryV_VF, + Sched<[WriteVFALUF_MX, ReadVFALUV_MX, ReadVFALUF_MX, ReadVMask]>; + } + } } multiclass VPseudoVALU_VF { - defm "" : VPseudoBinaryV_VF, - Sched<[WriteVFALUF, ReadVFALUV, ReadVFALUF, ReadVMask]>; + foreach f = FPList in { + foreach m = f.MxList in { + defvar mx = m.MX; + defvar WriteVFALUF_MX = !cast("WriteVFALUF_" # mx); + defvar ReadVFALUV_MX = !cast("ReadVFALUV_" # mx); + defvar ReadVFALUF_MX = !cast("ReadVFALUF_" # mx); + + defm "" : VPseudoBinaryV_VF, + Sched<[WriteVFALUF_MX, ReadVFALUV_MX, ReadVFALUF_MX, ReadVMask]>; + } + } } multiclass VPseudoVALU_VX_VI { @@ -2647,10 +2756,26 @@ multiclass VPseudoVWMUL_VV_VX { } multiclass VPseudoVWMUL_VV_VF { - defm "" : VPseudoBinaryW_VV, - Sched<[WriteVFWMulV, ReadVFWMulV, ReadVFWMulV, ReadVMask]>; - defm "" : VPseudoBinaryW_VF, - Sched<[WriteVFWMulF, ReadVFWMulV, ReadVFWMulF, ReadVMask]>; + foreach m = MxListFW in { + defvar mx = m.MX; + defvar WriteVFWMulV_MX = !cast("WriteVFWMulV_" # mx); + defvar ReadVFWMulV_MX = !cast("ReadVFWMulV_" # mx); + + defm "" : VPseudoBinaryW_VV_LMUL, + Sched<[WriteVFWMulV_MX, ReadVFWMulV_MX, ReadVFWMulV_MX, ReadVMask]>; + } + + foreach f = FPListW in { + foreach m = f.MxListFW in { + defvar mx = m.MX; + defvar WriteVFWMulF_MX = !cast("WriteVFWMulF_" # mx); + defvar ReadVFWMulV_MX = !cast("ReadVFWMulV_" # mx); + defvar ReadVFWMulF_MX = !cast("ReadVFWMulF_" # mx); + + defm "" : VPseudoBinaryW_VF, + Sched<[WriteVFWMulF_MX, ReadVFWMulV_MX, ReadVFWMulF_MX, ReadVMask]>; + } + } } multiclass VPseudoVWALU_WV_WX { @@ -2669,17 +2794,48 @@ multiclass VPseudoVWALU_WV_WX { } multiclass VPseudoVFWALU_VV_VF { - defm "" : VPseudoBinaryW_VV, - Sched<[WriteVFWALUV, ReadVFWALUV, ReadVFWALUV, ReadVMask]>; - defm "" : VPseudoBinaryW_VF, - Sched<[WriteVFWALUF, ReadVFWALUV, ReadVFWALUF, ReadVMask]>; + foreach m = MxListFW in { + defvar mx = m.MX; + defvar WriteVFWALUV_MX = !cast("WriteVFWALUV_" # mx); + defvar ReadVFWALUV_MX = !cast("ReadVFWALUV_" # mx); + + defm "" : VPseudoBinaryW_VV_LMUL, + Sched<[WriteVFWALUV_MX, ReadVFWALUV_MX, ReadVFWALUV_MX, ReadVMask]>; + } + + foreach f = FPListW in { + foreach m = f.MxListFW in { + defvar mx = m.MX; + defvar WriteVFWALUF_MX = !cast("WriteVFWALUF_" # mx); + defvar ReadVFWALUV_MX = !cast("ReadVFWALUV_" # mx); + defvar ReadVFWALUF_MX = !cast("ReadVFWALUF_" # mx); + + defm "" : VPseudoBinaryW_VF, + Sched<[WriteVFWALUF_MX, ReadVFWALUV_MX, ReadVFWALUF_MX, ReadVMask]>; + } + } } multiclass VPseudoVFWALU_WV_WF { - defm "" : VPseudoBinaryW_WV, - Sched<[WriteVFWALUV, ReadVFWALUV, ReadVFWALUV, ReadVMask]>; - defm "" : VPseudoBinaryW_WF, - Sched<[WriteVFWALUF, ReadVFWALUV, ReadVFWALUF, ReadVMask]>; + foreach m = MxListFW in { + defvar mx = m.MX; + defvar WriteVFWALUV_MX = !cast("WriteVFWALUV_" # mx); + defvar ReadVFWALUV_MX = !cast("ReadVFWALUV_" # mx); + + defm "" : VPseudoBinaryW_WV_LMUL, + Sched<[WriteVFWALUV_MX, ReadVFWALUV_MX, ReadVFWALUV_MX, ReadVMask]>; + } + foreach f = FPListW in { + foreach m = f.MxListFW in { + defvar mx = m.MX; + defvar WriteVFWALUF_MX = !cast("WriteVFWALUF_" # mx); + defvar ReadVFWALUV_MX = !cast("ReadVFWALUV_" # mx); + defvar ReadVFWALUF_MX = !cast("ReadVFWALUF_" # mx); + + defm "" : VPseudoBinaryW_WF, + Sched<[WriteVFWALUF_MX, ReadVFWALUV_MX, ReadVFWALUF_MX, ReadVMask]>; + } + } } multiclass VPseudoVMRG_VM_XM_IM { @@ -2914,12 +3070,10 @@ multiclass VPseudoTernaryV_VX_AAXA { Constraint, /*Commutable*/1>; } -multiclass VPseudoTernaryV_VF_AAXA { - foreach f = FPList in - foreach m = f.MxList in - defm "_V" # f.FX : VPseudoTernaryWithPolicy; +multiclass VPseudoTernaryV_VF_AAXA { + defm "_V" # f.FX : VPseudoTernaryWithPolicy; } multiclass VPseudoTernaryW_VV mxlist = MxListW> { @@ -2941,12 +3095,10 @@ multiclass VPseudoTernaryW_VX { constraint>; } -multiclass VPseudoTernaryW_VF { +multiclass VPseudoTernaryW_VF { defvar constraint = "@earlyclobber $rd"; - foreach f = FPListW in - foreach m = f.MxListFW in - defm "_V" # f.FX : VPseudoTernaryWithPolicy; + defm "_V" # f.FX : VPseudoTernaryWithPolicy; } multiclass VPseudoVSLDV_VI { @@ -2972,10 +3124,26 @@ multiclass VPseudoVMAC_VV_VX_AAXA { } multiclass VPseudoVMAC_VV_VF_AAXA { - defm "" : VPseudoTernaryV_VV_AAXA, - Sched<[WriteVFMulAddV, ReadVFMulAddV, ReadVFMulAddV, ReadVFMulAddV, ReadVMask]>; - defm "" : VPseudoTernaryV_VF_AAXA, - Sched<[WriteVFMulAddF, ReadVFMulAddV, ReadVFMulAddV, ReadVFMulAddF, ReadVMask]>; + foreach m = MxListF in { + defvar mx = m.MX; + defvar WriteVFMulAddV_MX = !cast("WriteVFMulAddV_" # mx); + defvar ReadVFMulAddV_MX = !cast("ReadVFMulAddV_" # mx); + + defm "" : VPseudoTernaryV_VV_AAXA_LMUL, + Sched<[WriteVFMulAddV_MX, ReadVFMulAddV_MX, ReadVFMulAddV_MX, ReadVFMulAddV_MX, ReadVMask]>; + } + + foreach f = FPList in { + foreach m = f.MxList in { + defvar mx = m.MX; + defvar WriteVFMulAddF_MX = !cast("WriteVFMulAddF_" # mx); + defvar ReadVFMulAddV_MX = !cast("ReadVFMulAddV_" # mx); + defvar ReadVFMulAddF_MX = !cast("ReadVFMulAddF_" # mx); + + defm "" : VPseudoTernaryV_VF_AAXA, + Sched<[WriteVFMulAddF_MX, ReadVFMulAddV_MX, ReadVFMulAddV_MX, ReadVFMulAddF_MX, ReadVMask]>; + } + } } multiclass VPseudoVSLD_VX_VI { @@ -3016,10 +3184,28 @@ multiclass VPseudoVWMAC_VX { } multiclass VPseudoVWMAC_VV_VF { - defm "" : VPseudoTernaryW_VV, - Sched<[WriteVFWMulAddV, ReadVFWMulAddV, ReadVFWMulAddV, ReadVFWMulAddV, ReadVMask]>; - defm "" : VPseudoTernaryW_VF, - Sched<[WriteVFWMulAddF, ReadVFWMulAddV, ReadVFWMulAddV, ReadVFWMulAddF, ReadVMask]>; + foreach m = MxListFW in { + defvar mx = m.MX; + defvar WriteVFWMulAddV_MX = !cast("WriteVFWMulAddV_" # mx); + defvar ReadVFWMulAddV_MX = !cast("ReadVFWMulAddV_" # mx); + + defm "" : VPseudoTernaryW_VV_LMUL, + Sched<[WriteVFWMulAddV_MX, ReadVFWMulAddV_MX, + ReadVFWMulAddV_MX, ReadVFWMulAddV_MX, ReadVMask]>; + } + + foreach f = FPListW in { + foreach m = f.MxListFW in { + defvar mx = m.MX; + defvar WriteVFWMulAddF_MX = !cast("WriteVFWMulAddF_" # mx); + defvar ReadVFWMulAddV_MX = !cast("ReadVFWMulAddV_" # mx); + defvar ReadVFWMulAddF_MX = !cast("ReadVFWMulAddF_" # mx); + + defm "" : VPseudoTernaryW_VF, + Sched<[WriteVFWMulAddF_MX, ReadVFWMulAddV_MX, + ReadVFWMulAddV_MX, ReadVFWMulAddF_MX, ReadVMask]>; + } + } } multiclass VPseudoVCMPM_VV_VX_VI { @@ -3056,15 +3242,40 @@ multiclass VPseudoVCMPM_VV_VX { } multiclass VPseudoVCMPM_VV_VF { - defm "" : VPseudoBinaryM_VV, - Sched<[WriteVFCmpV, ReadVFCmpV, ReadVFCmpV, ReadVMask]>; - defm "" : VPseudoBinaryM_VF, - Sched<[WriteVFCmpF, ReadVFCmpV, ReadVFCmpF, ReadVMask]>; + foreach m = MxListF in { + defvar mx = m.MX; + defvar WriteVFCmpV_MX = !cast("WriteVFCmpV_" # mx); + defvar ReadVFCmpV_MX = !cast("ReadVFCmpV_" # mx); + + defm "" : VPseudoBinaryM_VV_LMUL, + Sched<[WriteVFCmpV_MX, ReadVFCmpV_MX, ReadVFCmpV_MX, ReadVMask]>; + } + + foreach f = FPList in { + foreach m = f.MxList in { + defvar mx = m.MX; + defvar WriteVFCmpF_MX = !cast("WriteVFCmpF_" # mx); + defvar ReadVFCmpV_MX = !cast("ReadVFCmpV_" # mx); + defvar ReadVFCmpF_MX = !cast("ReadVFCmpF_" # mx); + + defm "" : VPseudoBinaryM_VF, + Sched<[WriteVFCmpF_MX, ReadVFCmpV_MX, ReadVFCmpF_MX, ReadVMask]>; + } + } } multiclass VPseudoVCMPM_VF { - defm "" : VPseudoBinaryM_VF, - Sched<[WriteVFCmpF, ReadVFCmpV, ReadVFCmpF, ReadVMask]>; + foreach f = FPList in { + foreach m = f.MxList in { + defvar mx = m.MX; + defvar WriteVFCmpF_MX = !cast("WriteVFCmpF_" # mx); + defvar ReadVFCmpV_MX = !cast("ReadVFCmpV_" # mx); + defvar ReadVFCmpF_MX = !cast("ReadVFCmpF_" # mx); + + defm "" : VPseudoBinaryM_VF, + Sched<[WriteVFCmpF_MX, ReadVFCmpV_MX, ReadVFCmpF_MX, ReadVMask]>; + } + } } multiclass VPseudoVCMPM_VX_VI { @@ -3150,27 +3361,47 @@ multiclass VPseudoConversionNoExcept("WriteVFCvtFToIV_" # mx); + defvar ReadVFCvtFToIV_MX = !cast("ReadVFCvtFToIV_" # mx); + defm _V : VPseudoConversion, - Sched<[WriteVFCvtFToIV, ReadVFCvtFToIV, ReadVMask]>; + Sched<[WriteVFCvtFToIV_MX, ReadVFCvtFToIV_MX, ReadVMask]>; + } } multiclass VPseudoVCVTI_RM_V { - foreach m = MxListF in + foreach m = MxListF in { + defvar mx = m.MX; + defvar WriteVFCvtFToIV_MX = !cast("WriteVFCvtFToIV_" # mx); + defvar ReadVFCvtFToIV_MX = !cast("ReadVFCvtFToIV_" # mx); + defm _V : VPseudoConversionRM, - Sched<[WriteVFCvtFToIV, ReadVFCvtFToIV, ReadVMask]>; + Sched<[WriteVFCvtFToIV_MX, ReadVFCvtFToIV_MX, ReadVMask]>; + } } multiclass VPseudoVFROUND_NOEXCEPT_V { - foreach m = MxListF in + foreach m = MxListF in { + defvar mx = m.MX; + defvar WriteVFCvtFToIV_MX = !cast("WriteVFCvtFToIV_" # mx); + defvar ReadVFCvtFToIV_MX = !cast("ReadVFCvtFToIV_" # mx); + defm _V : VPseudoConversionNoExcept, - Sched<[WriteVFCvtFToIV, ReadVFCvtFToIV, ReadVMask]>; + Sched<[WriteVFCvtFToIV_MX, ReadVFCvtFToIV_MX, ReadVMask]>; + } } multiclass VPseudoVCVTF_V { - foreach m = MxListF in + foreach m = MxListF in { + defvar mx = m.MX; + defvar WriteVFCvtIToFV_MX = !cast("WriteVFCvtIToFV_" # mx); + defvar ReadVFCvtIToFV_MX = !cast("ReadVFCvtIToFV_" # mx); + defm _V : VPseudoConversion, - Sched<[WriteVFCvtIToFV, ReadVFCvtIToFV, ReadVMask]>; + Sched<[WriteVFCvtIToFV_MX, ReadVFCvtIToFV_MX, ReadVMask]>; + } } multiclass VPseudoConversionW_V { @@ -3181,44 +3412,74 @@ multiclass VPseudoConversionW_V { multiclass VPseudoVWCVTI_V { defvar constraint = "@earlyclobber $rd"; - foreach m = MxListFW in + foreach m = MxListFW in { + defvar mx = m.MX; + defvar WriteVFWCvtFToIV_MX = !cast("WriteVFWCvtFToIV_" # mx); + defvar ReadVFWCvtFToIV_MX = !cast("ReadVFWCvtFToIV_" # mx); + defm _V : VPseudoConversion, - Sched<[WriteVFWCvtFToIV, ReadVFWCvtFToIV, ReadVMask]>; + Sched<[WriteVFWCvtFToIV_MX, ReadVFWCvtFToIV_MX, ReadVMask]>; + } } multiclass VPseudoVWCVTF_V { defvar constraint = "@earlyclobber $rd"; - foreach m = MxListW in + foreach m = MxListW in { + defvar mx = m.MX; + defvar WriteVFWCvtIToFV_MX = !cast("WriteVFWCvtIToFV_" # mx); + defvar ReadVFWCvtIToFV_MX = !cast("ReadVFWCvtIToFV_" # mx); + defm _V : VPseudoConversion, - Sched<[WriteVFWCvtIToFV, ReadVFWCvtIToFV, ReadVMask]>; + Sched<[WriteVFWCvtIToFV_MX, ReadVFWCvtIToFV_MX, ReadVMask]>; + } } multiclass VPseudoVWCVTD_V { defvar constraint = "@earlyclobber $rd"; - foreach m = MxListFW in + foreach m = MxListFW in { + defvar mx = m.MX; + defvar WriteVFWCvtFToFV_MX = !cast("WriteVFWCvtFToFV_" # mx); + defvar ReadVFWCvtFToFV_MX = !cast("ReadVFWCvtFToFV_" # mx); + defm _V : VPseudoConversion, - Sched<[WriteVFWCvtFToFV, ReadVFWCvtFToFV, ReadVMask]>; + Sched<[WriteVFWCvtFToFV_MX, ReadVFWCvtFToFV_MX, ReadVMask]>; + } } multiclass VPseudoVNCVTI_W { defvar constraint = "@earlyclobber $rd"; - foreach m = MxListW in + foreach m = MxListW in { + defvar mx = m.MX; + defvar WriteVFNCvtFToIV_MX = !cast("WriteVFNCvtFToIV_" # mx); + defvar ReadVFNCvtFToIV_MX = !cast("ReadVFNCvtFToIV_" # mx); + defm _W : VPseudoConversion, - Sched<[WriteVFNCvtFToIV, ReadVFNCvtFToIV, ReadVMask]>; + Sched<[WriteVFNCvtFToIV_MX, ReadVFNCvtFToIV_MX, ReadVMask]>; + } } multiclass VPseudoVNCVTF_W { defvar constraint = "@earlyclobber $rd"; - foreach m = MxListFW in + foreach m = MxListFW in { + defvar mx = m.MX; + defvar WriteVFNCvtIToFV_MX = !cast("WriteVFNCvtIToFV_" # mx); + defvar ReadVFNCvtIToFV_MX = !cast("ReadVFNCvtIToFV_" # mx); + defm _W : VPseudoConversion, - Sched<[WriteVFNCvtIToFV, ReadVFNCvtIToFV, ReadVMask]>; + Sched<[WriteVFNCvtIToFV_MX, ReadVFNCvtIToFV_MX, ReadVMask]>; + } } multiclass VPseudoVNCVTD_W { defvar constraint = "@earlyclobber $rd"; - foreach m = MxListFW in + foreach m = MxListFW in { + defvar mx = m.MX; + defvar WriteVFNCvtFToFV_MX = !cast("WriteVFNCvtFToFV_" # mx); + defvar ReadVFNCvtFToFV_MX = !cast("ReadVFNCvtFToFV_" # mx); + defm _W : VPseudoConversion, - Sched<[WriteVFNCvtFToFV, ReadVFNCvtFToFV, ReadVMask]>; + Sched<[WriteVFNCvtFToFV_MX, ReadVFNCvtFToFV_MX, ReadVMask]>; + } } multiclass VPseudoUSSegLoad { diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td index b4bcd3a..aab5cf2 100644 --- a/llvm/lib/Target/RISCV/RISCVScheduleV.td +++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td @@ -12,6 +12,7 @@ defvar SchedMxList = ["UpperBound", "M1", "M2", "M4", "M8", "MF2", "MF4", "MF8"]; // Used for widening and narrowing instructions as it doesn't contain M8. defvar SchedMxListW = ["UpperBound", "MF8", "MF4", "MF2", "M1", "M2", "M4"]; +defvar SchedMxListFW = ["UpperBound", "MF4", "MF2", "M1", "M2", "M4"]; // Creates SchedWrite for each (name, LMUL) pair for LMUL in SchedMxList multiclass LMULSchedWrites { @@ -27,6 +28,13 @@ multiclass LMULSchedWritesW { } } +// Creates SchedWrite for each (name, LMUL) pair for LMUL in SchedMxListFW +multiclass LMULSchedWritesFW { + foreach mx = SchedMxListFW in { + def name # "_" # mx : SchedWrite; + } +} + // Creates SchedRead for each (name, LMUL) pair for LMUL in SchedMxList multiclass LMULSchedReads { foreach mx = SchedMxList in { @@ -41,6 +49,13 @@ multiclass LMULSchedReadsW { } } +// Creates SchedRead for each (name, LMUL) pair for LMUL in SchedMxListFW +multiclass LMULSchedReadsFW { + foreach mx = SchedMxListFW in { + def name # "_" # mx : SchedRead; + } +} + // Creates WriteRes for each (name, LMUL, resources) tuple for LMUL // in SchedMxList multiclass LMULWriteRes resources> { @@ -57,6 +72,14 @@ multiclass LMULWriteResW resources> { } } +// Creates WriteRes for each (name, LMUL, resources) tuple for LMUL +// in SchedMxListFW +multiclass LMULWriteResFW resources> { + foreach mx = SchedMxListFW in { + def : WriteRes(name # "_" # mx), resources>; + } +} + // Creates ReadAdvance for each (name, LMUL, val) tuple for LMUL // in SchedMxList multiclass LMULReadAdvance writes = []> { @@ -73,6 +96,14 @@ multiclass LMULReadAdvanceW writes = []> } } +// Creates ReadAdvance for each (name, LMUL, val) tuple for LMUL +// in SchedMxListFW +multiclass LMULReadAdvanceFW writes = []> { + foreach mx = SchedMxListFW in { + def : ReadAdvance(name # "_" # mx), val, writes>; + } +} + // 3.6 Vector Byte Length vlenb def WriteRdVLENB : SchedWrite; @@ -215,54 +246,54 @@ defm "" : LMULSchedWritesW<"WriteVNClipI">; // 13. Vector Floating-Point Instructions // 13.2. Vector Single-Width Floating-Point Add/Subtract Instructions -def WriteVFALUV : SchedWrite; -def WriteVFALUF : SchedWrite; +defm "" : LMULSchedWrites<"WriteVFALUV">; +defm "" : LMULSchedWrites<"WriteVFALUF">; // 13.3. Vector Widening Floating-Point Add/Subtract Instructions -def WriteVFWALUV : SchedWrite; -def WriteVFWALUF : SchedWrite; +defm "" : LMULSchedWritesFW<"WriteVFWALUV">; +defm "" : LMULSchedWritesFW<"WriteVFWALUF">; // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions -def WriteVFMulV : SchedWrite; -def WriteVFMulF : SchedWrite; -def WriteVFDivV : SchedWrite; -def WriteVFDivF : SchedWrite; +defm "" : LMULSchedWrites<"WriteVFMulV">; +defm "" : LMULSchedWrites<"WriteVFMulF">; +defm "" : LMULSchedWrites<"WriteVFDivV">; +defm "" : LMULSchedWrites<"WriteVFDivF">; // 13.5. Vector Widening Floating-Point Multiply -def WriteVFWMulV : SchedWrite; -def WriteVFWMulF : SchedWrite; +defm "" : LMULSchedWritesFW<"WriteVFWMulV">; +defm "" : LMULSchedWritesFW<"WriteVFWMulF">; // 13.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions -def WriteVFMulAddV : SchedWrite; -def WriteVFMulAddF : SchedWrite; +defm "" : LMULSchedWrites<"WriteVFMulAddV">; +defm "" : LMULSchedWrites<"WriteVFMulAddF">; // 13.7. Vector Widening Floating-Point Fused Multiply-Add Instructions -def WriteVFWMulAddV : SchedWrite; -def WriteVFWMulAddF : SchedWrite; +defm "" : LMULSchedWritesFW<"WriteVFWMulAddV">; +defm "" : LMULSchedWritesFW<"WriteVFWMulAddF">; // 13.8. Vector Floating-Point Square-Root Instruction -def WriteVFSqrtV : SchedWrite; +defm "" : LMULSchedWrites<"WriteVFSqrtV">; // 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction // 13.10. Vector Floating-Point Reciprocal Estimate Instruction -def WriteVFRecpV : SchedWrite; +defm "" : LMULSchedWrites<"WriteVFRecpV">; // 13.11. Vector Floating-Point MIN/MAX Instructions // 13.13. Vector Floating-Point Compare Instructions -def WriteVFCmpV : SchedWrite; -def WriteVFCmpF : SchedWrite; +defm "" : LMULSchedWrites<"WriteVFCmpV">; +defm "" : LMULSchedWrites<"WriteVFCmpF">; // 13.12. Vector Floating-Point Sign-Injection Instructions -def WriteVFSgnjV : SchedWrite; -def WriteVFSgnjF : SchedWrite; +defm "" : LMULSchedWrites<"WriteVFSgnjV">; +defm "" : LMULSchedWrites<"WriteVFSgnjF">; // 13.14. Vector Floating-Point Classify Instruction -def WriteVFClassV : SchedWrite; +defm "" : LMULSchedWrites<"WriteVFClassV">; // 13.15. Vector Floating-Point Merge Instruction -def WriteVFMergeV : SchedWrite; +defm "" : LMULSchedWrites<"WriteVFMergeV">; // 13.16. Vector Floating-Point Move Instruction -def WriteVFMovV : SchedWrite; +defm "" : LMULSchedWrites<"WriteVFMovV">; // 13.17. Single-Width Floating-Point/Integer Type-Convert Instructions -def WriteVFCvtIToFV : SchedWrite; -def WriteVFCvtFToIV : SchedWrite; +defm "" : LMULSchedWrites<"WriteVFCvtIToFV">; +defm "" : LMULSchedWrites<"WriteVFCvtFToIV">; // 13.18. Widening Floating-Point/Integer Type-Convert Instructions -def WriteVFWCvtIToFV : SchedWrite; -def WriteVFWCvtFToIV : SchedWrite; -def WriteVFWCvtFToFV : SchedWrite; +defm "" : LMULSchedWritesW<"WriteVFWCvtIToFV">; +defm "" : LMULSchedWritesFW<"WriteVFWCvtFToIV">; +defm "" : LMULSchedWritesFW<"WriteVFWCvtFToFV">; // 13.19. Narrowing Floating-Point/Integer Type-Convert Instructions -def WriteVFNCvtIToFV : SchedWrite; -def WriteVFNCvtFToIV : SchedWrite; -def WriteVFNCvtFToFV : SchedWrite; +defm "" : LMULSchedWritesFW<"WriteVFNCvtIToFV">; +defm "" : LMULSchedWritesW<"WriteVFNCvtFToIV">; +defm "" : LMULSchedWritesFW<"WriteVFNCvtFToFV">; // 14. Vector Reduction Operations // 14.1. Vector Single-Width Integer Reduction Instructions @@ -428,55 +459,55 @@ defm "" : LMULSchedReadsW<"ReadVNClipX">; // 13. Vector Floating-Point Instructions // 13.2. Vector Single-Width Floating-Point Add/Subtract Instructions -def ReadVFALUV : SchedRead; -def ReadVFALUF : SchedRead; +defm "" : LMULSchedReads<"ReadVFALUV">; +defm "" : LMULSchedReads<"ReadVFALUF">; // 13.3. Vector Widening Floating-Point Add/Subtract Instructions -def ReadVFWALUV : SchedRead; -def ReadVFWALUF : SchedRead; +defm "" : LMULSchedReadsFW<"ReadVFWALUV">; +defm "" : LMULSchedReadsFW<"ReadVFWALUF">; // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions -def ReadVFMulV : SchedRead; -def ReadVFMulF : SchedRead; -def ReadVFDivV : SchedRead; -def ReadVFDivF : SchedRead; +defm "" : LMULSchedReads<"ReadVFMulV">; +defm "" : LMULSchedReads<"ReadVFMulF">; +defm "" : LMULSchedReads<"ReadVFDivV">; +defm "" : LMULSchedReads<"ReadVFDivF">; // 13.5. Vector Widening Floating-Point Multiply -def ReadVFWMulV : SchedRead; -def ReadVFWMulF : SchedRead; +defm "" : LMULSchedReadsFW<"ReadVFWMulV">; +defm "" : LMULSchedReadsFW<"ReadVFWMulF">; // 13.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions -def ReadVFMulAddV : SchedRead; -def ReadVFMulAddF : SchedRead; +defm "" : LMULSchedReads<"ReadVFMulAddV">; +defm "" : LMULSchedReads<"ReadVFMulAddF">; // 13.7. Vector Widening Floating-Point Fused Multiply-Add Instructions -def ReadVFWMulAddV : SchedRead; -def ReadVFWMulAddF : SchedRead; +defm "" : LMULSchedReadsFW<"ReadVFWMulAddV">; +defm "" : LMULSchedReadsFW<"ReadVFWMulAddF">; // 13.8. Vector Floating-Point Square-Root Instruction -def ReadVFSqrtV : SchedRead; +defm "" : LMULSchedReads<"ReadVFSqrtV">; // 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction // 13.10. Vector Floating-Point Reciprocal Estimate Instruction -def ReadVFRecpV : SchedRead; +defm "" : LMULSchedReads<"ReadVFRecpV">; // 13.11. Vector Floating-Point MIN/MAX Instructions // 13.13. Vector Floating-Point Compare Instructions -def ReadVFCmpV : SchedRead; -def ReadVFCmpF : SchedRead; +defm "" : LMULSchedReads<"ReadVFCmpV">; +defm "" : LMULSchedReads<"ReadVFCmpF">; // 13.12. Vector Floating-Point Sign-Injection Instructions -def ReadVFSgnjV : SchedRead; -def ReadVFSgnjF : SchedRead; +defm "" : LMULSchedReads<"ReadVFSgnjV">; +defm "" : LMULSchedReads<"ReadVFSgnjF">; // 13.14. Vector Floating-Point Classify Instruction -def ReadVFClassV : SchedRead; +defm "" : LMULSchedReads<"ReadVFClassV">; // 13.15. Vector Floating-Point Merge Instruction -def ReadVFMergeV : SchedRead; -def ReadVFMergeF : SchedRead; +defm "" : LMULSchedReads<"ReadVFMergeV">; +defm "" : LMULSchedReads<"ReadVFMergeF">; // 13.16. Vector Floating-Point Move Instruction -def ReadVFMovF : SchedRead; +defm "" : LMULSchedReads<"ReadVFMovF">; // 13.17. Single-Width Floating-Point/Integer Type-Convert Instructions -def ReadVFCvtIToFV : SchedRead; -def ReadVFCvtFToIV : SchedRead; +defm "" : LMULSchedReads<"ReadVFCvtIToFV">; +defm "" : LMULSchedReads<"ReadVFCvtFToIV">; // 13.18. Widening Floating-Point/Integer Type-Convert Instructions -def ReadVFWCvtIToFV : SchedRead; -def ReadVFWCvtFToIV : SchedRead; -def ReadVFWCvtFToFV : SchedRead; +defm "" : LMULSchedReadsW<"ReadVFWCvtIToFV">; +defm "" : LMULSchedReadsFW<"ReadVFWCvtFToIV">; +defm "" : LMULSchedReadsFW<"ReadVFWCvtFToFV">; // 13.19. Narrowing Floating-Point/Integer Type-Convert Instructions -def ReadVFNCvtIToFV : SchedRead; -def ReadVFNCvtFToIV : SchedRead; -def ReadVFNCvtFToFV : SchedRead; +defm "" : LMULSchedReadsFW<"ReadVFNCvtIToFV">; +defm "" : LMULSchedReadsW<"ReadVFNCvtFToIV">; +defm "" : LMULSchedReadsFW<"ReadVFNCvtFToFV">; // 14. Vector Reduction Operations // 14.1. Vector Single-Width Integer Reduction Instructions @@ -658,37 +689,37 @@ defm "" : LMULWriteResW<"WriteVNClipX", []>; defm "" : LMULWriteResW<"WriteVNClipI", []>; // 13. Vector Floating-Point Instructions -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; -def : WriteRes; +defm "" : LMULWriteRes<"WriteVFALUV", []>; +defm "" : LMULWriteRes<"WriteVFALUF", []>; +defm "" : LMULWriteResFW<"WriteVFWALUV", []>; +defm "" : LMULWriteResFW<"WriteVFWALUF", []>; +defm "" : LMULWriteRes<"WriteVFMulV", []>; +defm "" : LMULWriteRes<"WriteVFMulF", []>; +defm "" : LMULWriteRes<"WriteVFDivV", []>; +defm "" : LMULWriteRes<"WriteVFDivF", []>; +defm "" : LMULWriteResFW<"WriteVFWMulV", []>; +defm "" : LMULWriteResFW<"WriteVFWMulF", []>; +defm "" : LMULWriteRes<"WriteVFMulAddV", []>; +defm "" : LMULWriteRes<"WriteVFMulAddF", []>; +defm "" : LMULWriteResFW<"WriteVFWMulAddV", []>; +defm "" : LMULWriteResFW<"WriteVFWMulAddF", []>; +defm "" : LMULWriteRes<"WriteVFSqrtV", []>; +defm "" : LMULWriteRes<"WriteVFRecpV", []>; +defm "" : LMULWriteRes<"WriteVFCmpV", []>; +defm "" : LMULWriteRes<"WriteVFCmpF", []>; +defm "" : LMULWriteRes<"WriteVFSgnjV", []>; +defm "" : LMULWriteRes<"WriteVFSgnjF", []>; +defm "" : LMULWriteRes<"WriteVFClassV", []>; +defm "" : LMULWriteRes<"WriteVFMergeV", []>; +defm "" : LMULWriteRes<"WriteVFMovV", []>; +defm "" : LMULWriteRes<"WriteVFCvtIToFV", []>; +defm "" : LMULWriteRes<"WriteVFCvtFToIV", []>; +defm "" : LMULWriteResW<"WriteVFWCvtIToFV", []>; +defm "" : LMULWriteResFW<"WriteVFWCvtFToIV", []>; +defm "" : LMULWriteResFW<"WriteVFWCvtFToFV", []>; +defm "" : LMULWriteResFW<"WriteVFNCvtIToFV", []>; +defm "" : LMULWriteResW<"WriteVFNCvtFToIV", []>; +defm "" : LMULWriteResFW<"WriteVFNCvtFToFV", []>; // 14. Vector Reduction Operations def : WriteRes; @@ -806,38 +837,38 @@ defm "" : LMULReadAdvanceW<"ReadVNClipV", 0>; defm "" : LMULReadAdvanceW<"ReadVNClipX", 0>; // 13. Vector Floating-Point Instructions -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; -def : ReadAdvance; +defm "" : LMULReadAdvance<"ReadVFALUV", 0>; +defm "" : LMULReadAdvance<"ReadVFALUF", 0>; +defm "" : LMULReadAdvanceFW<"ReadVFWALUV", 0>; +defm "" : LMULReadAdvanceFW<"ReadVFWALUF", 0>; +defm "" : LMULReadAdvance<"ReadVFMulV", 0>; +defm "" : LMULReadAdvance<"ReadVFMulF", 0>; +defm "" : LMULReadAdvance<"ReadVFDivV", 0>; +defm "" : LMULReadAdvance<"ReadVFDivF", 0>; +defm "" : LMULReadAdvanceFW<"ReadVFWMulV", 0>; +defm "" : LMULReadAdvanceFW<"ReadVFWMulF", 0>; +defm "" : LMULReadAdvance<"ReadVFMulAddV", 0>; +defm "" : LMULReadAdvance<"ReadVFMulAddF", 0>; +defm "" : LMULReadAdvanceFW<"ReadVFWMulAddV", 0>; +defm "" : LMULReadAdvanceFW<"ReadVFWMulAddF", 0>; +defm "" : LMULReadAdvance<"ReadVFSqrtV", 0>; +defm "" : LMULReadAdvance<"ReadVFRecpV", 0>; +defm "" : LMULReadAdvance<"ReadVFCmpV", 0>; +defm "" : LMULReadAdvance<"ReadVFCmpF", 0>; +defm "" : LMULReadAdvance<"ReadVFSgnjV", 0>; +defm "" : LMULReadAdvance<"ReadVFSgnjF", 0>; +defm "" : LMULReadAdvance<"ReadVFClassV", 0>; +defm "" : LMULReadAdvance<"ReadVFMergeV", 0>; +defm "" : LMULReadAdvance<"ReadVFMergeF", 0>; +defm "" : LMULReadAdvance<"ReadVFMovF", 0>; +defm "" : LMULReadAdvance<"ReadVFCvtIToFV", 0>; +defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>; +defm "" : LMULReadAdvanceW<"ReadVFWCvtIToFV", 0>; +defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>; +defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToFV", 0>; +defm "" : LMULReadAdvanceFW<"ReadVFNCvtIToFV", 0>; +defm "" : LMULReadAdvanceW<"ReadVFNCvtFToIV", 0>; +defm "" : LMULReadAdvanceFW<"ReadVFNCvtFToFV", 0>; // 14. Vector Reduction Operations def : ReadAdvance; -- 2.7.4