From: Ju-Zhe Zhong Date: Tue, 14 Feb 2023 09:24:04 +0000 (+0800) Subject: RISC-V: Finish all integer C/C++ intrinsics X-Git-Tag: upstream/13.1.0~1213 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=272e119d97217562803b42384db546e86b511d11;p=platform%2Fupstream%2Fgcc.git RISC-V: Finish all integer C/C++ intrinsics gcc/ChangeLog: * config/riscv/predicates.md: Refine codes. * config/riscv/riscv-protos.h (RVV_VUNDEF): New macro. * config/riscv/riscv-v.cc: Refine codes. * config/riscv/riscv-vector-builtins-bases.cc (enum ternop_type): New enum. (class imac): New class. (enum widen_ternop_type): New enum. (class iwmac): New class. (BASE): New class. * config/riscv/riscv-vector-builtins-bases.h: Ditto. * config/riscv/riscv-vector-builtins-functions.def (vmacc): Ditto. (vnmsac): Ditto. (vmadd): Ditto. (vnmsub): Ditto. (vwmacc): Ditto. (vwmaccu): Ditto. (vwmaccsu): Ditto. (vwmaccus): Ditto. * config/riscv/riscv-vector-builtins.cc (function_builder::apply_predication): Adjust for multiply-add support. (function_expander::add_vundef_operand): Refine codes. (function_expander::use_ternop_insn): New function. (function_expander::use_widen_ternop_insn): Ditto. * config/riscv/riscv-vector-builtins.h: New function. * config/riscv/vector.md (@pred_mul_): New pattern. (pred_mul__undef_merge): Ditto. (*pred_): Ditto. (*pred_): Ditto. (*pred_mul_): Ditto. (@pred_mul__scalar): Ditto. (*pred_mul__undef_merge_scalar): Ditto. (*pred__scalar): Ditto. (*pred__scalar): Ditto. (*pred_mul__scalar): Ditto. (*pred_mul__undef_merge_extended_scalar): Ditto. (*pred__extended_scalar): Ditto. (*pred__extended_scalar): Ditto. (*pred_mul__extended_scalar): Ditto. (@pred_widen_mul_plus): Ditto. (@pred_widen_mul_plus_scalar): Ditto. (@pred_widen_mul_plussu): Ditto. (@pred_widen_mul_plussu_scalar): Ditto. (@pred_widen_mul_plusus_scalar): Ditto. --- diff --git a/gcc/config/riscv/predicates.md b/gcc/config/riscv/predicates.md index bbbf523..7bc7c0b 100644 --- a/gcc/config/riscv/predicates.md +++ b/gcc/config/riscv/predicates.md @@ -292,8 +292,7 @@ (match_operand 0 "vector_all_trues_mask_operand"))) (define_predicate "vector_undef_operand" - (match_test "GET_CODE (op) == UNSPEC - && (XINT (op, 1) == UNSPEC_VUNDEF)")) + (match_test "rtx_equal_p (op, RVV_VUNDEF (GET_MODE (op)))")) (define_predicate "vector_merge_operand" (ior (match_operand 0 "register_operand") diff --git a/gcc/config/riscv/riscv-protos.h b/gcc/config/riscv/riscv-protos.h index a4476e6..9d8b0b7 100644 --- a/gcc/config/riscv/riscv-protos.h +++ b/gcc/config/riscv/riscv-protos.h @@ -121,6 +121,8 @@ extern void riscv_run_selftests (void); namespace riscv_vector { #define RVV_VLMAX gen_rtx_REG (Pmode, X0_REGNUM) +#define RVV_VUNDEF(MODE) \ + gen_rtx_UNSPEC (MODE, gen_rtvec (1, const0_rtx), UNSPEC_VUNDEF) enum vlmul_type { LMUL_1 = 0, diff --git a/gcc/config/riscv/riscv-v.cc b/gcc/config/riscv/riscv-v.cc index cc26888..600b2e6 100644 --- a/gcc/config/riscv/riscv-v.cc +++ b/gcc/config/riscv/riscv-v.cc @@ -67,9 +67,7 @@ public: } void add_vundef_operand (machine_mode mode) { - add_input_operand (gen_rtx_UNSPEC (mode, gen_rtvec (1, const0_rtx), - UNSPEC_VUNDEF), - mode); + add_input_operand (RVV_VUNDEF (mode), mode); } void add_policy_operand (enum tail_policy vta, enum mask_policy vma) { diff --git a/gcc/config/riscv/riscv-vector-builtins-bases.cc b/gcc/config/riscv/riscv-vector-builtins-bases.cc index 4f3531d..ba70148 100644 --- a/gcc/config/riscv/riscv-vector-builtins-bases.cc +++ b/gcc/config/riscv/riscv-vector-builtins-bases.cc @@ -539,6 +539,132 @@ public: } }; +/* Enumerates types of ternary operations. + We have 2 types ternop: + - 1. accumulator is vd: + vmacc.vv vd,vs1,vs2 # vd = vs1 * vs2 + vd. + - 2. accumulator is vs2: + vmadd.vv vd,vs1,vs2 # vd = vs1 * vd + vs2. */ +enum ternop_type +{ + TERNOP_VMACC, + TERNOP_VNMSAC, + TERNOP_VMADD, + TERNOP_VNMSUB, +}; + +/* Implements vmacc/vnmsac/vmadd/vnmsub. */ +template +class imac : public function_base +{ +public: + bool has_merge_operand_p () const override { return false; } + + rtx expand (function_expander &e) const override + { + switch (TERNOP_TYPE) + { + case TERNOP_VMACC: + if (e.op_info->op == OP_TYPE_vx) + return e.use_ternop_insn ( + true, code_for_pred_mul_scalar (PLUS, e.vector_mode ())); + if (e.op_info->op == OP_TYPE_vv) + return e.use_ternop_insn (true, + code_for_pred_mul (PLUS, e.vector_mode ())); + break; + case TERNOP_VNMSAC: + if (e.op_info->op == OP_TYPE_vx) + return e.use_ternop_insn ( + true, code_for_pred_mul_scalar (MINUS, e.vector_mode ())); + if (e.op_info->op == OP_TYPE_vv) + return e.use_ternop_insn (true, code_for_pred_mul (MINUS, + e.vector_mode ())); + break; + case TERNOP_VMADD: + if (e.op_info->op == OP_TYPE_vx) + return e.use_ternop_insn ( + false, code_for_pred_mul_scalar (PLUS, e.vector_mode ())); + if (e.op_info->op == OP_TYPE_vv) + return e.use_ternop_insn (false, + code_for_pred_mul (PLUS, e.vector_mode ())); + break; + case TERNOP_VNMSUB: + if (e.op_info->op == OP_TYPE_vx) + return e.use_ternop_insn ( + false, code_for_pred_mul_scalar (MINUS, e.vector_mode ())); + if (e.op_info->op == OP_TYPE_vv) + return e.use_ternop_insn (false, + code_for_pred_mul (MINUS, + e.vector_mode ())); + break; + default: + break; + } + gcc_unreachable (); + } +}; + +/* Enumerates types of widen ternary operations. + We have 4 types ternop: + - 1. vwmacc. + - 2. vwmaccu. + - 3. vwmaccsu. + - 4. vwmaccus. */ +enum widen_ternop_type +{ + WIDEN_TERNOP_VWMACC, + WIDEN_TERNOP_VWMACCU, + WIDEN_TERNOP_VWMACCSU, + WIDEN_TERNOP_VWMACCUS, +}; + +/* Implements vwmacc. */ +template +class iwmac : public function_base +{ +public: + bool has_merge_operand_p () const override { return false; } + + rtx expand (function_expander &e) const override + { + switch (WIDEN_TERNOP_TYPE) + { + case WIDEN_TERNOP_VWMACC: + if (e.op_info->op == OP_TYPE_vx) + return e.use_widen_ternop_insn ( + code_for_pred_widen_mul_plus_scalar (SIGN_EXTEND, + e.vector_mode ())); + if (e.op_info->op == OP_TYPE_vv) + return e.use_widen_ternop_insn ( + code_for_pred_widen_mul_plus (SIGN_EXTEND, e.vector_mode ())); + break; + case WIDEN_TERNOP_VWMACCU: + if (e.op_info->op == OP_TYPE_vx) + return e.use_widen_ternop_insn ( + code_for_pred_widen_mul_plus_scalar (ZERO_EXTEND, + e.vector_mode ())); + if (e.op_info->op == OP_TYPE_vv) + return e.use_widen_ternop_insn ( + code_for_pred_widen_mul_plus (ZERO_EXTEND, e.vector_mode ())); + break; + case WIDEN_TERNOP_VWMACCSU: + if (e.op_info->op == OP_TYPE_vx) + return e.use_widen_ternop_insn ( + code_for_pred_widen_mul_plussu_scalar (e.vector_mode ())); + if (e.op_info->op == OP_TYPE_vv) + return e.use_widen_ternop_insn ( + code_for_pred_widen_mul_plussu (e.vector_mode ())); + break; + case WIDEN_TERNOP_VWMACCUS: + return e.use_widen_ternop_insn ( + code_for_pred_widen_mul_plusus_scalar (e.vector_mode ())); + default: + break; + } + gcc_unreachable (); + } +}; + static CONSTEXPR const vsetvl vsetvl_obj; static CONSTEXPR const vsetvl vsetvlmax_obj; static CONSTEXPR const loadstore vle_obj; @@ -616,6 +742,14 @@ static CONSTEXPR const icmp vmsltu_obj; static CONSTEXPR const icmp vmsgtu_obj; static CONSTEXPR const icmp vmsleu_obj; static CONSTEXPR const icmp vmsgeu_obj; +static CONSTEXPR const imac vmacc_obj; +static CONSTEXPR const imac vnmsac_obj; +static CONSTEXPR const imac vmadd_obj; +static CONSTEXPR const imac vnmsub_obj; +static CONSTEXPR const iwmac vwmacc_obj; +static CONSTEXPR const iwmac vwmaccu_obj; +static CONSTEXPR const iwmac vwmaccsu_obj; +static CONSTEXPR const iwmac vwmaccus_obj; static CONSTEXPR const binop vsadd_obj; static CONSTEXPR const binop vssub_obj; static CONSTEXPR const binop vsaddu_obj; @@ -712,6 +846,14 @@ BASE (vmsltu) BASE (vmsgtu) BASE (vmsleu) BASE (vmsgeu) +BASE (vmacc) +BASE (vnmsac) +BASE (vmadd) +BASE (vnmsub) +BASE (vwmacc) +BASE (vwmaccu) +BASE (vwmaccsu) +BASE (vwmaccus) BASE (vsadd) BASE (vssub) BASE (vsaddu) diff --git a/gcc/config/riscv/riscv-vector-builtins-bases.h b/gcc/config/riscv/riscv-vector-builtins-bases.h index 6233b41..cb36f1f 100644 --- a/gcc/config/riscv/riscv-vector-builtins-bases.h +++ b/gcc/config/riscv/riscv-vector-builtins-bases.h @@ -101,6 +101,14 @@ extern const function_base *const vmsltu; extern const function_base *const vmsgtu; extern const function_base *const vmsleu; extern const function_base *const vmsgeu; +extern const function_base *const vmacc; +extern const function_base *const vnmsac; +extern const function_base *const vmadd; +extern const function_base *const vnmsub; +extern const function_base *const vwmacc; +extern const function_base *const vwmaccu; +extern const function_base *const vwmaccsu; +extern const function_base *const vwmaccus; extern const function_base *const vsadd; extern const function_base *const vssub; extern const function_base *const vsaddu; diff --git a/gcc/config/riscv/riscv-vector-builtins-functions.def b/gcc/config/riscv/riscv-vector-builtins-functions.def index f61f48d..2227127 100644 --- a/gcc/config/riscv/riscv-vector-builtins-functions.def +++ b/gcc/config/riscv/riscv-vector-builtins-functions.def @@ -177,6 +177,21 @@ DEF_RVV_FUNCTION (vmsltu, return_mask, none_m_mu_preds, u_mvx_ops) DEF_RVV_FUNCTION (vmsgtu, return_mask, none_m_mu_preds, u_mvx_ops) DEF_RVV_FUNCTION (vmsleu, return_mask, none_m_mu_preds, u_mvx_ops) DEF_RVV_FUNCTION (vmsgeu, return_mask, none_m_mu_preds, u_mvx_ops) +DEF_RVV_FUNCTION (vmacc, alu, full_preds, iu_vvvv_ops) +DEF_RVV_FUNCTION (vnmsac, alu, full_preds, iu_vvvv_ops) +DEF_RVV_FUNCTION (vmadd, alu, full_preds, iu_vvvv_ops) +DEF_RVV_FUNCTION (vnmsub, alu, full_preds, iu_vvvv_ops) +DEF_RVV_FUNCTION (vmacc, alu, full_preds, iu_vvxv_ops) +DEF_RVV_FUNCTION (vnmsac, alu, full_preds, iu_vvxv_ops) +DEF_RVV_FUNCTION (vmadd, alu, full_preds, iu_vvxv_ops) +DEF_RVV_FUNCTION (vnmsub, alu, full_preds, iu_vvxv_ops) +DEF_RVV_FUNCTION (vwmacc, alu, full_preds, i_wwvv_ops) +DEF_RVV_FUNCTION (vwmacc, alu, full_preds, i_wwxv_ops) +DEF_RVV_FUNCTION (vwmaccu, alu, full_preds, u_wwvv_ops) +DEF_RVV_FUNCTION (vwmaccu, alu, full_preds, u_wwxv_ops) +DEF_RVV_FUNCTION (vwmaccsu, alu, full_preds, i_su_wwvv_ops) +DEF_RVV_FUNCTION (vwmaccsu, alu, full_preds, i_su_wwxv_ops) +DEF_RVV_FUNCTION (vwmaccus, alu, full_preds, i_us_wwxv_ops) /* 12. Vector Fixed-Point Arithmetic Instructions. */ DEF_RVV_FUNCTION (vsadd, alu, full_preds, i_vvv_ops) DEF_RVV_FUNCTION (vssub, alu, full_preds, i_vvv_ops) diff --git a/gcc/config/riscv/riscv-vector-builtins.cc b/gcc/config/riscv/riscv-vector-builtins.cc index c1f595c..1047bd2 100644 --- a/gcc/config/riscv/riscv-vector-builtins.cc +++ b/gcc/config/riscv/riscv-vector-builtins.cc @@ -314,6 +314,18 @@ static CONSTEXPR const rvv_arg_type_info vv_args[] = {rvv_arg_type_info (RVV_BASE_vector), rvv_arg_type_info (RVV_BASE_vector), rvv_arg_type_info_end}; +/* A list of args for vector_type func (vector_type, vector_type, vector_type) + * function. */ +static CONSTEXPR const rvv_arg_type_info vvv_args[] + = {rvv_arg_type_info (RVV_BASE_vector), rvv_arg_type_info (RVV_BASE_vector), + rvv_arg_type_info (RVV_BASE_vector), rvv_arg_type_info_end}; + +/* A list of args for vector_type func (vector_type, vector_type, vector_type) + * function. */ +static CONSTEXPR const rvv_arg_type_info vxv_args[] + = {rvv_arg_type_info (RVV_BASE_vector), rvv_arg_type_info (RVV_BASE_scalar), + rvv_arg_type_info (RVV_BASE_vector), rvv_arg_type_info_end}; + /* A list of args for vector_type func (vector_type, vector_type, mask_type) * function. */ static CONSTEXPR const rvv_arg_type_info vvm_args[] @@ -378,6 +390,44 @@ static CONSTEXPR const rvv_arg_type_info wvv_args[] = {rvv_arg_type_info (RVV_BASE_double_trunc_vector), rvv_arg_type_info (RVV_BASE_double_trunc_vector), rvv_arg_type_info_end}; +/* A list of args for vector_type func (vector_type, double demote type, double + * demote type) function. */ +static CONSTEXPR const rvv_arg_type_info wwvv_args[] + = {rvv_arg_type_info (RVV_BASE_vector), + rvv_arg_type_info (RVV_BASE_double_trunc_vector), + rvv_arg_type_info (RVV_BASE_double_trunc_vector), rvv_arg_type_info_end}; + +/* A list of args for vector_type func (vector_type, double demote type, double + * demote type) function. */ +static CONSTEXPR const rvv_arg_type_info wwxv_args[] + = {rvv_arg_type_info (RVV_BASE_vector), + rvv_arg_type_info (RVV_BASE_double_trunc_scalar), + rvv_arg_type_info (RVV_BASE_double_trunc_vector), rvv_arg_type_info_end}; + +/* A list of args for vector_type func (vector_type, double demote type, double + * demote type) function. */ +static CONSTEXPR const rvv_arg_type_info su_wwvv_args[] + = {rvv_arg_type_info (RVV_BASE_vector), + rvv_arg_type_info (RVV_BASE_double_trunc_vector), + rvv_arg_type_info (RVV_BASE_double_trunc_unsigned_vector), + rvv_arg_type_info_end}; + +/* A list of args for vector_type func (vector_type, double demote type, double + * demote type) function. */ +static CONSTEXPR const rvv_arg_type_info su_wwxv_args[] + = {rvv_arg_type_info (RVV_BASE_vector), + rvv_arg_type_info (RVV_BASE_double_trunc_scalar), + rvv_arg_type_info (RVV_BASE_double_trunc_unsigned_vector), + rvv_arg_type_info_end}; + +/* A list of args for vector_type func (vector_type, double demote type, double + * demote type) function. */ +static CONSTEXPR const rvv_arg_type_info us_wwxv_args[] + = {rvv_arg_type_info (RVV_BASE_vector), + rvv_arg_type_info (RVV_BASE_double_trunc_unsigned_scalar), + rvv_arg_type_info (RVV_BASE_double_trunc_vector), + rvv_arg_type_info_end}; + /* A list of args for vector_type func (signed double demote type, * unsigneddouble demote type) function. */ static CONSTEXPR const rvv_arg_type_info su_wvv_args[] @@ -578,6 +628,22 @@ static CONSTEXPR const rvv_op_info iu_vvv_ops vv_args /* Args */}; /* A static operand information for vector_type func (vector_type, vector_type, + * vector_type) function registration. */ +static CONSTEXPR const rvv_op_info iu_vvvv_ops + = {iu_ops, /* Types */ + OP_TYPE_vv, /* Suffix */ + rvv_arg_type_info (RVV_BASE_vector), /* Return type */ + vvv_args /* Args */}; + +/* A static operand information for vector_type func (vector_type, scalar_type, + * vector_type) function registration. */ +static CONSTEXPR const rvv_op_info iu_vvxv_ops + = {iu_ops, /* Types */ + OP_TYPE_vx, /* Suffix */ + rvv_arg_type_info (RVV_BASE_vector), /* Return type */ + vxv_args /* Args */}; + +/* A static operand information for vector_type func (vector_type, vector_type, * mask_type) function registration. */ static CONSTEXPR const rvv_op_info iu_vvvm_ops = {iu_ops, /* Types */ @@ -882,6 +948,62 @@ static CONSTEXPR const rvv_op_info i_wvv_ops rvv_arg_type_info (RVV_BASE_vector), /* Return type */ wvv_args /* Args */}; +/* A static operand information for vector_type func (vector_type, double demote + * type, double demote type) function registration. */ +static CONSTEXPR const rvv_op_info i_wwvv_ops + = {wexti_ops, /* Types */ + OP_TYPE_vv, /* Suffix */ + rvv_arg_type_info (RVV_BASE_vector), /* Return type */ + wwvv_args /* Args */}; + +/* A static operand information for vector_type func (vector_type, double demote + * scalar_type, double demote type) function registration. */ +static CONSTEXPR const rvv_op_info i_wwxv_ops + = {wexti_ops, /* Types */ + OP_TYPE_vx, /* Suffix */ + rvv_arg_type_info (RVV_BASE_vector), /* Return type */ + wwxv_args /* Args */}; + +/* A static operand information for vector_type func (vector_type, double demote + * type, double demote type) function registration. */ +static CONSTEXPR const rvv_op_info u_wwvv_ops + = {wextu_ops, /* Types */ + OP_TYPE_vv, /* Suffix */ + rvv_arg_type_info (RVV_BASE_vector), /* Return type */ + wwvv_args /* Args */}; + +/* A static operand information for vector_type func (vector_type, double demote + * scalar_type, double demote type) function registration. */ +static CONSTEXPR const rvv_op_info u_wwxv_ops + = {wextu_ops, /* Types */ + OP_TYPE_vx, /* Suffix */ + rvv_arg_type_info (RVV_BASE_vector), /* Return type */ + wwxv_args /* Args */}; + +/* A static operand information for vector_type func (vector_type, double demote + * type, double demote type) function registration. */ +static CONSTEXPR const rvv_op_info i_su_wwvv_ops + = {wexti_ops, /* Types */ + OP_TYPE_vv, /* Suffix */ + rvv_arg_type_info (RVV_BASE_vector), /* Return type */ + su_wwvv_args /* Args */}; + +/* A static operand information for vector_type func (vector_type, double demote + * scalar_type, double demote type) function registration. */ +static CONSTEXPR const rvv_op_info i_su_wwxv_ops + = {wexti_ops, /* Types */ + OP_TYPE_vx, /* Suffix */ + rvv_arg_type_info (RVV_BASE_vector), /* Return type */ + su_wwxv_args /* Args */}; + +/* A static operand information for vector_type func (vector_type, double demote + * scalar_type, double demote type) function registration. */ +static CONSTEXPR const rvv_op_info i_us_wwxv_ops + = {wexti_ops, /* Types */ + OP_TYPE_vx, /* Suffix */ + rvv_arg_type_info (RVV_BASE_vector), /* Return type */ + us_wwxv_args /* Args */}; + /* A static operand information for vector_type func (signed double demote type, * unsigned double demote type) function registration. */ static CONSTEXPR const rvv_op_info i_su_wvv_ops @@ -1617,9 +1739,10 @@ function_builder::apply_predication (const function_instance &instance, vec &argument_types) const { /* These predication types need to apply merge type. */ - if (instance.pred == PRED_TYPE_tu || instance.pred == PRED_TYPE_tum - || instance.pred == PRED_TYPE_tumu || instance.pred == PRED_TYPE_mu) - argument_types.quick_insert (0, return_type); + if (instance.base->has_merge_operand_p ()) + if (instance.pred == PRED_TYPE_tu || instance.pred == PRED_TYPE_tum + || instance.pred == PRED_TYPE_tumu || instance.pred == PRED_TYPE_mu) + argument_types.quick_insert (0, return_type); /* These predication types need to apply mask type. */ tree mask_type = builtin_types[mask_types[instance.type.index]].vector; @@ -1816,8 +1939,7 @@ function_expander::add_input_operand (unsigned argno) void function_expander::add_vundef_operand (machine_mode mode) { - rtx vundef = gen_rtx_UNSPEC (mode, gen_rtvec (1, const0_rtx), UNSPEC_VUNDEF); - add_input_operand (mode, vundef); + add_input_operand (mode, RVV_VUNDEF (mode)); } /* Add a memory operand with mode MODE and address ADDR. */ @@ -1979,6 +2101,117 @@ function_expander::use_compare_insn (rtx_code rcode, insn_code icode) return generate_insn (icode); } +/* Implement the call using instruction ICODE, with a 1:1 mapping between + arguments and input operands. */ +rtx +function_expander::use_ternop_insn (bool vd_accum_p, insn_code icode) +{ + machine_mode mode = TYPE_MODE (builtin_types[type.index].vector); + tree mask_type = builtin_types[mask_types[type.index]].vector; + machine_mode mask_mode = TYPE_MODE (mask_type); + + /* Record the offset to get the argument. */ + int arg_offset = 0; + + if (use_real_mask_p (pred)) + add_input_operand (arg_offset++); + else + add_all_one_mask_operand (mask_mode); + + rtx vd = expand_normal (CALL_EXPR_ARG (exp, arg_offset++)); + rtx vs1 = expand_normal (CALL_EXPR_ARG (exp, arg_offset++)); + rtx vs2 = expand_normal (CALL_EXPR_ARG (exp, arg_offset++)); + rtx merge = use_real_merge_p (pred) ? vd : RVV_VUNDEF (mode); + + if (VECTOR_MODE_P (GET_MODE (vs1))) + { + if (!vd_accum_p) + add_input_operand (mode, vd); + add_input_operand (mode, vs1); + add_input_operand (mode, vs2); + if (vd_accum_p) + add_input_operand (mode, vd); + add_input_operand (mode, merge); + } + else + { + add_input_operand (GET_MODE_INNER (mode), vs1); + if (vd_accum_p) + { + add_input_operand (mode, vs2); + add_input_operand (mode, vd); + } + else + { + add_input_operand (mode, vd); + add_input_operand (mode, vs2); + } + add_input_operand (mode, merge); + } + + for (int argno = arg_offset; argno < call_expr_nargs (exp); argno++) + add_input_operand (argno); + + add_input_operand (Pmode, get_tail_policy_for_pred (pred)); + add_input_operand (Pmode, get_mask_policy_for_pred (pred)); + add_input_operand (Pmode, get_avl_type_rtx (avl_type::NONVLMAX)); + + /* See optabs.cc, the maximum nops is 9 for using 'maybe_gen_insn'. + We temporarily use GCN directly. We will change it back it we + can support nops >= 10. */ + gcc_assert (maybe_legitimize_operands (icode, 0, opno, m_ops)); + rtx_insn *pat = GEN_FCN ( + icode) (m_ops[0].value, m_ops[1].value, m_ops[2].value, m_ops[3].value, + m_ops[4].value, m_ops[5].value, m_ops[6].value, m_ops[7].value, + m_ops[8].value, m_ops[9].value); + emit_insn (pat); + return m_ops[0].value; +} + +/* Implement the call using instruction ICODE, with a 1:1 mapping between + arguments and input operands. */ +rtx +function_expander::use_widen_ternop_insn (insn_code icode) +{ + machine_mode mode = TYPE_MODE (builtin_types[type.index].vector); + tree mask_type = builtin_types[mask_types[type.index]].vector; + machine_mode mask_mode = TYPE_MODE (mask_type); + + /* Record the offset to get the argument. */ + int arg_offset = 0; + + if (use_real_mask_p (pred)) + add_input_operand (arg_offset++); + else + add_all_one_mask_operand (mask_mode); + + rtx merge = RVV_VUNDEF (mode); + if (use_real_merge_p (pred)) + merge = expand_normal (CALL_EXPR_ARG (exp, arg_offset)); + + for (int argno = arg_offset; argno < call_expr_nargs (exp); argno++) + { + if (argno == call_expr_nargs (exp) - 1) + add_input_operand (mode, merge); + add_input_operand (argno); + } + + add_input_operand (Pmode, get_tail_policy_for_pred (pred)); + add_input_operand (Pmode, get_mask_policy_for_pred (pred)); + add_input_operand (Pmode, get_avl_type_rtx (avl_type::NONVLMAX)); + + /* See optabs.cc, the maximum nops is 9 for using 'maybe_gen_insn'. + We temporarily use GCN directly. We will change it back it we + can support nops >= 10. */ + gcc_assert (maybe_legitimize_operands (icode, 0, opno, m_ops)); + rtx_insn *pat = GEN_FCN ( + icode) (m_ops[0].value, m_ops[1].value, m_ops[2].value, m_ops[3].value, + m_ops[4].value, m_ops[5].value, m_ops[6].value, m_ops[7].value, + m_ops[8].value, m_ops[9].value); + emit_insn (pat); + return m_ops[0].value; +} + /* Generate instruction ICODE, given that its operands have already been added to M_OPS. Return the value of the first operand. */ rtx diff --git a/gcc/config/riscv/riscv-vector-builtins.h b/gcc/config/riscv/riscv-vector-builtins.h index 9807fdf..b9d7367 100644 --- a/gcc/config/riscv/riscv-vector-builtins.h +++ b/gcc/config/riscv/riscv-vector-builtins.h @@ -348,6 +348,8 @@ public: rtx use_contiguous_load_insn (insn_code); rtx use_contiguous_store_insn (insn_code); rtx use_compare_insn (rtx_code, insn_code); + rtx use_ternop_insn (bool, insn_code); + rtx use_widen_ternop_insn (insn_code); rtx generate_insn (insn_code); /* The function call expression. */ diff --git a/gcc/config/riscv/vector.md b/gcc/config/riscv/vector.md index 3443657..2d8179f 100644 --- a/gcc/config/riscv/vector.md +++ b/gcc/config/riscv/vector.md @@ -146,7 +146,7 @@ vialu,vshift,vicmp,vimul,vidiv,vsalu,\ vext,viwalu,viwmul,vicalu,vnshift,\ vimuladd,vimerge,vaalu,vsmul,vsshift,\ - vnclip,viminmax") + vnclip,viminmax,viwmuladd") (const_int INVALID_ATTRIBUTE) (eq_attr "mode" "VNx1QI,VNx1BI") (symbol_ref "riscv_vector::get_ratio(E_VNx1QImode)") @@ -198,11 +198,14 @@ (define_attr "merge_op_idx" "" (cond [(eq_attr "type" "vlde,vimov,vfmov,vldm,vlds,vmalu,vldux,vldox,vicmp,\ vialu,vshift,viminmax,vimul,vidiv,vsalu,vext,viwalu,\ - viwmul,vnshift,vimuladd,vaalu,vsmul,vsshift,vnclip") + viwmul,vnshift,vaalu,vsmul,vsshift,vnclip") (const_int 2) (eq_attr "type" "vimerge") - (const_int 1)] + (const_int 1) + + (eq_attr "type" "vimuladd,viwmuladd") + (const_int 5)] (const_int INVALID_ATTRIBUTE))) ;; The index of operand[] to get the avl op. @@ -219,11 +222,14 @@ (const_int 4)) (eq_attr "type" "vldux,vldox,vialu,vshift,viminmax,vimul,vidiv,vsalu,\ - viwalu,viwmul,vnshift,vimuladd,vimerge,vaalu,vsmul,\ + viwalu,viwmul,vnshift,vimerge,vaalu,vsmul,\ vsshift,vnclip") (const_int 5) (eq_attr "type" "vicmp") + (const_int 6) + + (eq_attr "type" "vimuladd,viwmuladd") (const_int 6)] (const_int INVALID_ATTRIBUTE))) @@ -240,9 +246,12 @@ (symbol_ref "riscv_vector::get_ta(operands[5])")) (eq_attr "type" "vldux,vldox,vialu,vshift,viminmax,vimul,vidiv,vsalu,\ - viwalu,viwmul,vnshift,vimuladd,vimerge,vaalu,vsmul,\ + viwalu,viwmul,vnshift,vimerge,vaalu,vsmul,\ vsshift,vnclip") - (symbol_ref "riscv_vector::get_ta(operands[6])")] + (symbol_ref "riscv_vector::get_ta(operands[6])") + + (eq_attr "type" "vimuladd,viwmuladd") + (symbol_ref "riscv_vector::get_ta(operands[7])")] (const_int INVALID_ATTRIBUTE))) ;; The mask policy op value. @@ -258,9 +267,12 @@ (symbol_ref "riscv_vector::get_ma(operands[6])")) (eq_attr "type" "vldux,vldox,vialu,vshift,viminmax,vimul,vidiv,vsalu,\ - viwalu,viwmul,vnshift,vimuladd,vaalu,vsmul,vsshift,\ + viwalu,viwmul,vnshift,vaalu,vsmul,vsshift,\ vnclip,vicmp") - (symbol_ref "riscv_vector::get_ma(operands[7])")] + (symbol_ref "riscv_vector::get_ma(operands[7])") + + (eq_attr "type" "vimuladd,viwmuladd") + (symbol_ref "riscv_vector::get_ma(operands[8])")] (const_int INVALID_ATTRIBUTE))) ;; The avl type value. @@ -282,7 +294,10 @@ vnclip,vicmp") (symbol_ref "INTVAL (operands[8])") (eq_attr "type" "vstux,vstox") - (symbol_ref "INTVAL (operands[5])")] + (symbol_ref "INTVAL (operands[5])") + + (eq_attr "type" "vimuladd,viwmuladd") + (symbol_ref "INTVAL (operands[9])")] (const_int INVALID_ATTRIBUTE))) ;; ----------------------------------------------------------------- @@ -3948,7 +3963,7 @@ "TARGET_VECTOR" { enum rtx_code code = GET_CODE (operands[3]); - rtx undef = gen_rtx_UNSPEC (mode, gen_rtvec (1, const0_rtx), UNSPEC_VUNDEF); + rtx undef = RVV_VUNDEF (mode); if (code == GEU && rtx_equal_p (operands[5], const0_rtx)) { /* If vmsgeu with 0 immediate, expand it to vmset. */ @@ -4036,6 +4051,651 @@ }) ;; ------------------------------------------------------------------------------- +;; ---- Predicated integer ternary operations +;; ------------------------------------------------------------------------------- +;; Includes: +;; - 11.13 Vector Single-Width Integer Multiply-Add Instructions +;; ------------------------------------------------------------------------------- + +(define_expand "@pred_mul_" + [(set (match_operand:VI 0 "register_operand") + (if_then_else:VI + (unspec: + [(match_operand: 1 "vector_mask_operand") + (match_operand 6 "vector_length_operand") + (match_operand 7 "const_int_operand") + (match_operand 8 "const_int_operand") + (match_operand 9 "const_int_operand") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus_minus:VI + (mult:VI + (match_operand:VI 2 "register_operand") + (match_operand:VI 3 "register_operand")) + (match_operand:VI 4 "register_operand")) + (match_operand:VI 5 "vector_merge_operand")))] + "TARGET_VECTOR" +{ + /* Swap the multiplication operands if the fallback value is the + second of the two. */ + if (rtx_equal_p (operands[3], operands[5])) + std::swap (operands[2], operands[3]); +}) + +(define_insn "pred_mul__undef_merge" + [(set (match_operand:VI 0 "register_operand" "=vd, vr, vd, vr, ?&vr") + (if_then_else:VI + (unspec: + [(match_operand: 1 "vector_mask_operand" " vm,Wc1, vm,Wc1,vmWc1") + (match_operand 6 "vector_length_operand" " rK, rK, rK, rK, rK") + (match_operand 7 "const_int_operand" " i, i, i, i, i") + (match_operand 8 "const_int_operand" " i, i, i, i, i") + (match_operand 9 "const_int_operand" " i, i, i, i, i") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus_minus:VI + (mult:VI + (match_operand:VI 2 "register_operand" " %0, 0, vr, vr, vr") + (match_operand:VI 3 "register_operand" " vr, vr, vr, vr, vr")) + (match_operand:VI 4 "register_operand" " vr, vr, 0, 0, vr")) + (match_operand:VI 5 "vector_undef_operand" " vu, vu, vu, vu, vu")))] + "TARGET_VECTOR" + "@ + v.vv\t%0,%3,%4%p1 + v.vv\t%0,%3,%4%p1 + v.vv\t%0,%2,%3%p1 + v.vv\t%0,%2,%3%p1 + vmv.v.v\t%0,%4\;v.vv\t%0,%2,%3%p1" + [(set_attr "type" "vimuladd") + (set_attr "mode" "")]) + +(define_insn "*pred_" + [(set (match_operand:VI 0 "register_operand" "=vd, vr, ?&vr") + (if_then_else:VI + (unspec: + [(match_operand: 1 "vector_mask_operand" " vm,Wc1,vmWc1") + (match_operand 5 "vector_length_operand" " rK, rK, rK") + (match_operand 6 "const_int_operand" " i, i, i") + (match_operand 7 "const_int_operand" " i, i, i") + (match_operand 8 "const_int_operand" " i, i, i") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus_minus:VI + (mult:VI + (match_operand:VI 2 "register_operand" " 0, 0, vr") + (match_operand:VI 3 "register_operand" " vr, vr, vr")) + (match_operand:VI 4 "register_operand" " vr, vr, vr")) + (match_dup 2)))] + "TARGET_VECTOR" + "@ + v.vv\t%0,%3,%4%p1 + v.vv\t%0,%3,%4%p1 + vmv.v.v\t%0,%2\;v.vv\t%0,%3,%4%p1" + [(set_attr "type" "vimuladd") + (set_attr "mode" "") + (set_attr "merge_op_idx" "4") + (set_attr "vl_op_idx" "5") + (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])")) + (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])")) + (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))]) + +(define_insn "*pred_" + [(set (match_operand:VI 0 "register_operand" "=vd, vr, ?&vr") + (if_then_else:VI + (unspec: + [(match_operand: 1 "vector_mask_operand" " vm,Wc1,vmWc1") + (match_operand 5 "vector_length_operand" " rK, rK, rK") + (match_operand 6 "const_int_operand" " i, i, i") + (match_operand 7 "const_int_operand" " i, i, i") + (match_operand 8 "const_int_operand" " i, i, i") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus_minus:VI + (mult:VI + (match_operand:VI 2 "register_operand" " vr, vr, vr") + (match_operand:VI 3 "register_operand" " vr, vr, vr")) + (match_operand:VI 4 "register_operand" " 0, 0, vr")) + (match_dup 4)))] + "TARGET_VECTOR" + "@ + v.vv\t%0,%2,%3%p1 + v.vv\t%0,%2,%3%p1 + vmv.v.v\t%0,%4\;v.vv\t%0,%2,%3%p1" + [(set_attr "type" "vimuladd") + (set_attr "mode" "") + (set_attr "merge_op_idx" "2") + (set_attr "vl_op_idx" "5") + (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])")) + (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])")) + (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))]) + +(define_insn_and_rewrite "*pred_mul_" + [(set (match_operand:VI 0 "register_operand" "=&vr,?&vr, ?&vr, ?&vr, ?&vr") + (if_then_else:VI + (unspec: + [(match_operand: 1 "vector_mask_operand" "vmWc1,vmWc1,vmWc1,vmWc1,vmWc1") + (match_operand 6 "vector_length_operand" " rK, rK, rK, rK, rK") + (match_operand 7 "const_int_operand" " i, i, i, i, i") + (match_operand 8 "const_int_operand" " i, i, i, i, i") + (match_operand 9 "const_int_operand" " i, i, i, i, i") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus_minus:VI + (mult:VI + (match_operand:VI 2 "register_operand" " vr, vr, vi, vr, vr") + (match_operand:VI 3 "register_operand" " vr, vr, vr, vi, vr")) + (match_operand:VI 4 "vector_arith_operand" " vr, vi, vr, vr, vr")) + (match_operand:VI 5 "register_operand" " 0, vr, vr, vr, vr")))] + "TARGET_VECTOR + && !rtx_equal_p (operands[2], operands[5]) + && !rtx_equal_p (operands[3], operands[5]) + && !rtx_equal_p (operands[4], operands[5])" + "@ + vmv.v.v\t%0,%4\;v.vv\t%0,%2,%3%p1 + # + # + # + #" + "&& reload_completed + && !rtx_equal_p (operands[0], operands[5])" + { + if (satisfies_constraint_vi (operands[3])) + std::swap (operands[2], operands[3]); + + if (satisfies_constraint_vi (operands[2])) + { + emit_insn (gen_pred_merge (operands[0], RVV_VUNDEF (mode), + operands[5], operands[2], operands[1], operands[6], + operands[7], operands[9])); + operands[5] = operands[2] = operands[0]; + } + else + { + emit_insn (gen_pred_merge (operands[0], RVV_VUNDEF (mode), + operands[5], operands[4], operands[1], operands[6], + operands[7], operands[9])); + operands[5] = operands[4] = operands[0]; + } + } + [(set_attr "type" "vimuladd") + (set_attr "mode" "")]) + +(define_expand "@pred_mul__scalar" + [(set (match_operand:VI_QHS 0 "register_operand") + (if_then_else:VI_QHS + (unspec: + [(match_operand: 1 "vector_mask_operand") + (match_operand 6 "vector_length_operand") + (match_operand 7 "const_int_operand") + (match_operand 8 "const_int_operand") + (match_operand 9 "const_int_operand") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus_minus:VI_QHS + (mult:VI_QHS + (vec_duplicate:VI_QHS + (match_operand: 2 "reg_or_int_operand")) + (match_operand:VI_QHS 3 "register_operand")) + (match_operand:VI_QHS 4 "register_operand")) + (match_operand:VI_QHS 5 "vector_merge_operand")))] + "TARGET_VECTOR" +{ + operands[2] = force_reg (mode, operands[2]); +}) + +(define_insn "*pred_mul__undef_merge_scalar" + [(set (match_operand:VI 0 "register_operand" "=vd, vr, vd, vr, ?&vr") + (if_then_else:VI + (unspec: + [(match_operand: 1 "vector_mask_operand" " vm,Wc1, vm,Wc1,vmWc1") + (match_operand 6 "vector_length_operand" " rK, rK, rK, rK, rK") + (match_operand 7 "const_int_operand" " i, i, i, i, i") + (match_operand 8 "const_int_operand" " i, i, i, i, i") + (match_operand 9 "const_int_operand" " i, i, i, i, i") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus_minus:VI + (mult:VI + (vec_duplicate:VI + (match_operand: 2 "register_operand" " r, r, r, r, r")) + (match_operand:VI 3 "register_operand" " 0, 0, vr, vr, vr")) + (match_operand:VI 4 "register_operand" " vr, vr, 0, 0, vr")) + (match_operand:VI 5 "vector_undef_operand" " vu, vu, vu, vu, vu")))] + "TARGET_VECTOR" + "@ + v.vx\t%0,%2,%4%p1 + v.vx\t%0,%2,%4%p1 + v.vx\t%0,%2,%3%p1 + v.vx\t%0,%2,%3%p1 + vmv.v.v\t%0,%4\;v.vx\t%0,%2,%3%p1" + [(set_attr "type" "vimuladd") + (set_attr "mode" "")]) + +(define_insn "*pred__scalar" + [(set (match_operand:VI 0 "register_operand" "=vd, vr, ?&vr") + (if_then_else:VI + (unspec: + [(match_operand: 1 "vector_mask_operand" " vm,Wc1,vmWc1") + (match_operand 5 "vector_length_operand" " rK, rK, rK") + (match_operand 6 "const_int_operand" " i, i, i") + (match_operand 7 "const_int_operand" " i, i, i") + (match_operand 8 "const_int_operand" " i, i, i") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus_minus:VI + (mult:VI + (vec_duplicate:VI + (match_operand: 2 "register_operand" " r, r, vr")) + (match_operand:VI 3 "register_operand" " 0, 0, vr")) + (match_operand:VI 4 "register_operand" " vr, vr, vr")) + (match_dup 3)))] + "TARGET_VECTOR" + "@ + v.vx\t%0,%2,%4%p1 + v.vx\t%0,%2,%4%p1 + vmv.v.v\t%0,%2\;v.vx\t%0,%2,%4%p1" + [(set_attr "type" "vimuladd") + (set_attr "mode" "") + (set_attr "merge_op_idx" "4") + (set_attr "vl_op_idx" "5") + (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])")) + (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])")) + (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))]) + +(define_insn "*pred__scalar" + [(set (match_operand:VI 0 "register_operand" "=vd, vr, ?&vr") + (if_then_else:VI + (unspec: + [(match_operand: 1 "vector_mask_operand" " vm,Wc1,vmWc1") + (match_operand 5 "vector_length_operand" " rK, rK, rK") + (match_operand 6 "const_int_operand" " i, i, i") + (match_operand 7 "const_int_operand" " i, i, i") + (match_operand 8 "const_int_operand" " i, i, i") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus_minus:VI + (mult:VI + (vec_duplicate:VI + (match_operand: 2 "register_operand" " r, r, vr")) + (match_operand:VI 3 "register_operand" " vr, vr, vr")) + (match_operand:VI 4 "register_operand" " 0, 0, vr")) + (match_dup 4)))] + "TARGET_VECTOR" + "@ + v.vx\t%0,%2,%3%p1 + v.vx\t%0,%2,%3%p1 + vmv.v.v\t%0,%4\;v.vx\t%0,%2,%3%p1" + [(set_attr "type" "vimuladd") + (set_attr "mode" "") + (set_attr "merge_op_idx" "2") + (set_attr "vl_op_idx" "5") + (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])")) + (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])")) + (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))]) + +(define_insn_and_rewrite "*pred_mul__scalar" + [(set (match_operand:VI 0 "register_operand" "=&vr, ?&vr, ?&vr, ?&vr") + (if_then_else:VI + (unspec: + [(match_operand: 1 "vector_mask_operand" "vmWc1,vmWc1,vmWc1,vmWc1") + (match_operand 6 "vector_length_operand" " rK, rK, rK, rK") + (match_operand 7 "const_int_operand" " i, i, i, i") + (match_operand 8 "const_int_operand" " i, i, i, i") + (match_operand 9 "const_int_operand" " i, i, i, i") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus_minus:VI + (mult:VI + (vec_duplicate:VI + (match_operand: 2 "register_operand" " r, r, r, r")) + (match_operand:VI 3 "register_operand" " vr, vr, vi, vr")) + (match_operand:VI 4 "vector_arith_operand" " vr, vi, vr, vr")) + (match_operand:VI 5 "register_operand" " 0, vr, vr, vr")))] + "TARGET_VECTOR + && !rtx_equal_p (operands[3], operands[5]) + && !rtx_equal_p (operands[4], operands[5])" + "@ + vmv.v.v\t%0,%4\;v.vx\t%0,%2,%3%p1 + # + # + #" + "&& reload_completed + && !rtx_equal_p (operands[0], operands[5])" + { + if (satisfies_constraint_vi (operands[3])) + { + emit_insn (gen_pred_merge (operands[0], RVV_VUNDEF (mode), + operands[5], operands[3], operands[1], operands[6], + operands[7], operands[9])); + operands[5] = operands[3] = operands[0]; + } + else + { + emit_insn (gen_pred_merge (operands[0], RVV_VUNDEF (mode), + operands[5], operands[4], operands[1], operands[6], + operands[7], operands[9])); + operands[5] = operands[4] = operands[0]; + } + } + [(set_attr "type" "vimuladd") + (set_attr "mode" "")]) + +(define_expand "@pred_mul__scalar" + [(set (match_operand:VI_D 0 "register_operand") + (if_then_else:VI_D + (unspec: + [(match_operand: 1 "vector_mask_operand") + (match_operand 6 "vector_length_operand") + (match_operand 7 "const_int_operand") + (match_operand 8 "const_int_operand") + (match_operand 9 "const_int_operand") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus_minus:VI_D + (mult:VI_D + (vec_duplicate:VI_D + (match_operand: 2 "reg_or_int_operand")) + (match_operand:VI_D 3 "register_operand")) + (match_operand:VI_D 4 "register_operand")) + (match_operand:VI_D 5 "vector_merge_operand")))] + "TARGET_VECTOR" +{ + if (!TARGET_64BIT) + { + rtx v = gen_reg_rtx (mode); + + if (riscv_vector::simm32_p (operands[2])) + operands[2] = gen_rtx_SIGN_EXTEND (mode, + force_reg (Pmode, operands[2])); + else + { + if (CONST_INT_P (operands[2])) + operands[2] = force_reg (mode, operands[2]); + + riscv_vector::emit_nonvlmax_op (code_for_pred_broadcast (mode), + v, operands[2], operands[6], mode); + emit_insn (gen_pred_mul_ (operands[0], operands[1], + v, operands[3], operands[4], operands[5], operands[6], + operands[7], operands[8], operands[9])); + DONE; + } + } + else + operands[2] = force_reg (mode, operands[2]); +}) + +(define_insn "*pred_mul__undef_merge_extended_scalar" + [(set (match_operand:VI_D 0 "register_operand" "=vd, vr, vd, vr, ?&vr") + (if_then_else:VI_D + (unspec: + [(match_operand: 1 "vector_mask_operand" " vm,Wc1, vm,Wc1,vmWc1") + (match_operand 6 "vector_length_operand" " rK, rK, rK, rK, rK") + (match_operand 7 "const_int_operand" " i, i, i, i, i") + (match_operand 8 "const_int_operand" " i, i, i, i, i") + (match_operand 9 "const_int_operand" " i, i, i, i, i") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus_minus:VI_D + (mult:VI_D + (vec_duplicate:VI_D + (sign_extend: + (match_operand: 2 "register_operand" " r, r, r, r, r"))) + (match_operand:VI_D 3 "register_operand" " 0, 0, vr, vr, vr")) + (match_operand:VI_D 4 "register_operand" " vr, vr, 0, 0, vr")) + (match_operand:VI_D 5 "vector_undef_operand" " vu, vu, vu, vu, vu")))] + "TARGET_VECTOR" + "@ + v.vx\t%0,%2,%4%p1 + v.vx\t%0,%2,%4%p1 + v.vx\t%0,%2,%3%p1 + v.vx\t%0,%2,%3%p1 + vmv.v.v\t%0,%4\;v.vx\t%0,%2,%3%p1" + [(set_attr "type" "vimuladd") + (set_attr "mode" "")]) + +(define_insn "*pred__extended_scalar" + [(set (match_operand:VI_D 0 "register_operand" "=vd, vr, ?&vr") + (if_then_else:VI_D + (unspec: + [(match_operand: 1 "vector_mask_operand" " vm,Wc1,vmWc1") + (match_operand 5 "vector_length_operand" " rK, rK, rK") + (match_operand 6 "const_int_operand" " i, i, i") + (match_operand 7 "const_int_operand" " i, i, i") + (match_operand 8 "const_int_operand" " i, i, i") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus_minus:VI_D + (mult:VI_D + (vec_duplicate:VI_D + (sign_extend: + (match_operand: 2 "register_operand" " r, r, vr"))) + (match_operand:VI_D 3 "register_operand" " 0, 0, vr")) + (match_operand:VI_D 4 "register_operand" " vr, vr, vr")) + (match_dup 3)))] + "TARGET_VECTOR" + "@ + v.vx\t%0,%2,%4%p1 + v.vx\t%0,%2,%4%p1 + vmv.v.v\t%0,%2\;v.vx\t%0,%2,%4%p1" + [(set_attr "type" "vimuladd") + (set_attr "mode" "") + (set_attr "merge_op_idx" "4") + (set_attr "vl_op_idx" "5") + (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])")) + (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])")) + (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))]) + +(define_insn "*pred__extended_scalar" + [(set (match_operand:VI_D 0 "register_operand" "=vd, vr, ?&vr") + (if_then_else:VI_D + (unspec: + [(match_operand: 1 "vector_mask_operand" " vm,Wc1,vmWc1") + (match_operand 5 "vector_length_operand" " rK, rK, rK") + (match_operand 6 "const_int_operand" " i, i, i") + (match_operand 7 "const_int_operand" " i, i, i") + (match_operand 8 "const_int_operand" " i, i, i") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus_minus:VI_D + (mult:VI_D + (vec_duplicate:VI_D + (sign_extend: + (match_operand: 2 "register_operand" " r, r, vr"))) + (match_operand:VI_D 3 "register_operand" " vr, vr, vr")) + (match_operand:VI_D 4 "register_operand" " 0, 0, vr")) + (match_dup 4)))] + "TARGET_VECTOR" + "@ + v.vx\t%0,%2,%3%p1 + v.vx\t%0,%2,%3%p1 + vmv.v.v\t%0,%4\;v.vx\t%0,%2,%3%p1" + [(set_attr "type" "vimuladd") + (set_attr "mode" "") + (set_attr "merge_op_idx" "2") + (set_attr "vl_op_idx" "5") + (set (attr "ta") (symbol_ref "riscv_vector::get_ta(operands[6])")) + (set (attr "ma") (symbol_ref "riscv_vector::get_ma(operands[7])")) + (set (attr "avl_type") (symbol_ref "INTVAL (operands[8])"))]) + +(define_insn_and_rewrite "*pred_mul__extended_scalar" + [(set (match_operand:VI_D 0 "register_operand" "=&vr, ?&vr, ?&vr, ?&vr") + (if_then_else:VI_D + (unspec: + [(match_operand: 1 "vector_mask_operand" "vmWc1,vmWc1,vmWc1,vmWc1") + (match_operand 6 "vector_length_operand" " rK, rK, rK, rK") + (match_operand 7 "const_int_operand" " i, i, i, i") + (match_operand 8 "const_int_operand" " i, i, i, i") + (match_operand 9 "const_int_operand" " i, i, i, i") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus_minus:VI_D + (mult:VI_D + (vec_duplicate:VI_D + (sign_extend: + (match_operand: 2 "register_operand" " r, r, r, r"))) + (match_operand:VI_D 3 "register_operand" " vr, vr, vr, vr")) + (match_operand:VI_D 4 "vector_arith_operand" " vr, vr, vr, vr")) + (match_operand:VI_D 5 "register_operand" " 0, vr, vr, vr")))] + "TARGET_VECTOR + && !rtx_equal_p (operands[3], operands[5]) + && !rtx_equal_p (operands[4], operands[5])" + "@ + vmv.v.v\t%0,%4\;v.vx\t%0,%2,%3%p1 + # + # + #" + "&& reload_completed + && !rtx_equal_p (operands[0], operands[5])" + { + if (satisfies_constraint_vi (operands[3])) + { + emit_insn (gen_pred_merge (operands[0], RVV_VUNDEF (mode), + operands[5], operands[3], operands[1], operands[6], + operands[7], operands[9])); + operands[5] = operands[3] = operands[0]; + } + else + { + emit_insn (gen_pred_merge (operands[0], RVV_VUNDEF (mode), + operands[5], operands[4], operands[1], operands[6], + operands[7], operands[9])); + operands[5] = operands[4] = operands[0]; + } + } + [(set_attr "type" "vimuladd") + (set_attr "mode" "")]) + +;; ------------------------------------------------------------------------------- +;; ---- Predicated integer ternary operations +;; ------------------------------------------------------------------------------- +;; Includes: +;; - 11.14 Vector Widening Integer Multiply-Add Instructions +;; ------------------------------------------------------------------------------- + +(define_insn "@pred_widen_mul_plus" + [(set (match_operand:VWEXTI 0 "register_operand" "=&vr") + (if_then_else:VWEXTI + (unspec: + [(match_operand: 1 "vector_mask_operand" "vmWc1") + (match_operand 6 "vector_length_operand" " rK") + (match_operand 7 "const_int_operand" " i") + (match_operand 8 "const_int_operand" " i") + (match_operand 9 "const_int_operand" " i") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus:VWEXTI + (mult:VWEXTI + (any_extend:VWEXTI + (match_operand: 3 "register_operand" " vr")) + (any_extend:VWEXTI + (match_operand: 4 "register_operand" " vr"))) + (match_operand:VWEXTI 2 "register_operand" " 0")) + (match_operand:VWEXTI 5 "vector_merge_operand" " 0vu")))] + "TARGET_VECTOR" + "vwmacc.vv\t%0,%3,%4%p1" + [(set_attr "type" "viwmuladd") + (set_attr "mode" "")]) + +(define_insn "@pred_widen_mul_plus_scalar" + [(set (match_operand:VWEXTI 0 "register_operand" "=&vr") + (if_then_else:VWEXTI + (unspec: + [(match_operand: 1 "vector_mask_operand" "vmWc1") + (match_operand 6 "vector_length_operand" " rK") + (match_operand 7 "const_int_operand" " i") + (match_operand 8 "const_int_operand" " i") + (match_operand 9 "const_int_operand" " i") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus:VWEXTI + (mult:VWEXTI + (any_extend:VWEXTI + (vec_duplicate: + (match_operand: 3 "register_operand" " r"))) + (any_extend:VWEXTI + (match_operand: 4 "register_operand" " vr"))) + (match_operand:VWEXTI 2 "register_operand" " 0")) + (match_operand:VWEXTI 5 "vector_merge_operand" " 0vu")))] + "TARGET_VECTOR" + "vwmacc.vx\t%0,%3,%4%p1" + [(set_attr "type" "viwmuladd") + (set_attr "mode" "")]) + +(define_insn "@pred_widen_mul_plussu" + [(set (match_operand:VWEXTI 0 "register_operand" "=&vr") + (if_then_else:VWEXTI + (unspec: + [(match_operand: 1 "vector_mask_operand" "vmWc1") + (match_operand 6 "vector_length_operand" " rK") + (match_operand 7 "const_int_operand" " i") + (match_operand 8 "const_int_operand" " i") + (match_operand 9 "const_int_operand" " i") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus:VWEXTI + (mult:VWEXTI + (sign_extend:VWEXTI + (match_operand: 3 "register_operand" " vr")) + (zero_extend:VWEXTI + (match_operand: 4 "register_operand" " vr"))) + (match_operand:VWEXTI 2 "register_operand" " 0")) + (match_operand:VWEXTI 5 "vector_merge_operand" " 0vu")))] + "TARGET_VECTOR" + "vwmaccsu.vv\t%0,%3,%4%p1" + [(set_attr "type" "viwmuladd") + (set_attr "mode" "")]) + +(define_insn "@pred_widen_mul_plussu_scalar" + [(set (match_operand:VWEXTI 0 "register_operand" "=&vr") + (if_then_else:VWEXTI + (unspec: + [(match_operand: 1 "vector_mask_operand" "vmWc1") + (match_operand 6 "vector_length_operand" " rK") + (match_operand 7 "const_int_operand" " i") + (match_operand 8 "const_int_operand" " i") + (match_operand 9 "const_int_operand" " i") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus:VWEXTI + (mult:VWEXTI + (sign_extend:VWEXTI + (vec_duplicate: + (match_operand: 3 "register_operand" " r"))) + (zero_extend:VWEXTI + (match_operand: 4 "register_operand" " vr"))) + (match_operand:VWEXTI 2 "register_operand" " 0")) + (match_operand:VWEXTI 5 "vector_merge_operand" " 0vu")))] + "TARGET_VECTOR" + "vwmaccsu.vx\t%0,%3,%4%p1" + [(set_attr "type" "viwmuladd") + (set_attr "mode" "")]) + +(define_insn "@pred_widen_mul_plusus_scalar" + [(set (match_operand:VWEXTI 0 "register_operand" "=&vr") + (if_then_else:VWEXTI + (unspec: + [(match_operand: 1 "vector_mask_operand" "vmWc1") + (match_operand 6 "vector_length_operand" " rK") + (match_operand 7 "const_int_operand" " i") + (match_operand 8 "const_int_operand" " i") + (match_operand 9 "const_int_operand" " i") + (reg:SI VL_REGNUM) + (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE) + (plus:VWEXTI + (mult:VWEXTI + (zero_extend:VWEXTI + (vec_duplicate: + (match_operand: 3 "register_operand" " r"))) + (sign_extend:VWEXTI + (match_operand: 4 "register_operand" " vr"))) + (match_operand:VWEXTI 2 "register_operand" " 0")) + (match_operand:VWEXTI 5 "vector_merge_operand" " 0vu")))] + "TARGET_VECTOR" + "vwmaccus.vx\t%0,%3,%4%p1" + [(set_attr "type" "viwmuladd") + (set_attr "mode" "")]) + +;; ------------------------------------------------------------------------------- ;; ---- Predicated BOOL mask operations ;; ------------------------------------------------------------------------------- ;; Includes: