2 Copyright (C) 2011-2015 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
30 #include "tree-ssanames.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "internal-fn.h"
37 #include "stor-layout.h"
42 /* The names of each internal function, indexed by function number. */
43 const char *const internal_fn_name_array[] = {
44 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
45 #include "internal-fn.def"
49 /* The ECF_* flags of each internal function, indexed by function number. */
50 const int internal_fn_flags_array[] = {
51 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
52 #include "internal-fn.def"
56 /* Fnspec of each internal function, indexed by function number. */
57 const_tree internal_fn_fnspec_array[IFN_LAST + 1];
62 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
63 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
64 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
65 #include "internal-fn.def"
66 internal_fn_fnspec_array[IFN_LAST] = 0;
69 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
70 for load-lanes-style optab OPTAB. The insn must exist. */
73 get_multi_vector_move (tree array_type, convert_optab optab)
79 gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
80 imode = TYPE_MODE (array_type);
81 vmode = TYPE_MODE (TREE_TYPE (array_type));
83 icode = convert_optab_handler (optab, imode, vmode);
84 gcc_assert (icode != CODE_FOR_nothing);
88 /* Expand LOAD_LANES call STMT. */
91 expand_LOAD_LANES (gcall *stmt)
93 struct expand_operand ops[2];
97 lhs = gimple_call_lhs (stmt);
98 rhs = gimple_call_arg (stmt, 0);
99 type = TREE_TYPE (lhs);
101 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
102 mem = expand_normal (rhs);
104 gcc_assert (MEM_P (mem));
105 PUT_MODE (mem, TYPE_MODE (type));
107 create_output_operand (&ops[0], target, TYPE_MODE (type));
108 create_fixed_operand (&ops[1], mem);
109 expand_insn (get_multi_vector_move (type, vec_load_lanes_optab), 2, ops);
112 /* Expand STORE_LANES call STMT. */
115 expand_STORE_LANES (gcall *stmt)
117 struct expand_operand ops[2];
121 lhs = gimple_call_lhs (stmt);
122 rhs = gimple_call_arg (stmt, 0);
123 type = TREE_TYPE (rhs);
125 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
126 reg = expand_normal (rhs);
128 gcc_assert (MEM_P (target));
129 PUT_MODE (target, TYPE_MODE (type));
131 create_fixed_operand (&ops[0], target);
132 create_input_operand (&ops[1], reg, TYPE_MODE (type));
133 expand_insn (get_multi_vector_move (type, vec_store_lanes_optab), 2, ops);
137 expand_ANNOTATE (gcall *)
142 /* This should get expanded in adjust_simduid_builtins. */
145 expand_GOMP_SIMD_LANE (gcall *)
150 /* This should get expanded in adjust_simduid_builtins. */
153 expand_GOMP_SIMD_VF (gcall *)
158 /* This should get expanded in adjust_simduid_builtins. */
161 expand_GOMP_SIMD_LAST_LANE (gcall *)
166 /* This should get expanded in adjust_simduid_builtins. */
169 expand_GOMP_SIMD_ORDERED_START (gcall *)
174 /* This should get expanded in adjust_simduid_builtins. */
177 expand_GOMP_SIMD_ORDERED_END (gcall *)
182 /* This should get expanded in the sanopt pass. */
185 expand_UBSAN_NULL (gcall *)
190 /* This should get expanded in the sanopt pass. */
193 expand_UBSAN_BOUNDS (gcall *)
198 /* This should get expanded in the sanopt pass. */
201 expand_UBSAN_VPTR (gcall *)
206 /* This should get expanded in the sanopt pass. */
209 expand_UBSAN_OBJECT_SIZE (gcall *)
214 /* This should get expanded in the sanopt pass. */
217 expand_ASAN_CHECK (gcall *)
222 /* This should get expanded in the tsan pass. */
225 expand_TSAN_FUNC_EXIT (gcall *)
230 /* Helper function for expand_addsub_overflow. Return 1
231 if ARG interpreted as signed in its precision is known to be always
232 positive or 2 if ARG is known to be always negative, or 3 if ARG may
233 be positive or negative. */
236 get_range_pos_neg (tree arg)
238 if (arg == error_mark_node)
241 int prec = TYPE_PRECISION (TREE_TYPE (arg));
243 if (TREE_CODE (arg) == INTEGER_CST)
245 wide_int w = wi::sext (arg, prec);
251 while (CONVERT_EXPR_P (arg)
252 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
253 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
255 arg = TREE_OPERAND (arg, 0);
256 /* Narrower value zero extended into wider type
257 will always result in positive values. */
258 if (TYPE_UNSIGNED (TREE_TYPE (arg))
259 && TYPE_PRECISION (TREE_TYPE (arg)) < prec)
261 prec = TYPE_PRECISION (TREE_TYPE (arg));
266 if (TREE_CODE (arg) != SSA_NAME)
268 wide_int arg_min, arg_max;
269 while (get_range_info (arg, &arg_min, &arg_max) != VR_RANGE)
271 gimple *g = SSA_NAME_DEF_STMT (arg);
272 if (is_gimple_assign (g)
273 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
275 tree t = gimple_assign_rhs1 (g);
276 if (INTEGRAL_TYPE_P (TREE_TYPE (t))
277 && TYPE_PRECISION (TREE_TYPE (t)) <= prec)
279 if (TYPE_UNSIGNED (TREE_TYPE (t))
280 && TYPE_PRECISION (TREE_TYPE (t)) < prec)
282 prec = TYPE_PRECISION (TREE_TYPE (t));
291 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
293 /* For unsigned values, the "positive" range comes
294 below the "negative" range. */
295 if (!wi::neg_p (wi::sext (arg_max, prec), SIGNED))
297 if (wi::neg_p (wi::sext (arg_min, prec), SIGNED))
302 if (!wi::neg_p (wi::sext (arg_min, prec), SIGNED))
304 if (wi::neg_p (wi::sext (arg_max, prec), SIGNED))
310 /* Return minimum precision needed to represent all values
311 of ARG in SIGNed integral type. */
314 get_min_precision (tree arg, signop sign)
316 int prec = TYPE_PRECISION (TREE_TYPE (arg));
318 signop orig_sign = sign;
319 if (TREE_CODE (arg) == INTEGER_CST)
322 if (TYPE_SIGN (TREE_TYPE (arg)) != sign)
324 widest_int w = wi::to_widest (arg);
325 w = wi::ext (w, prec, sign);
326 p = wi::min_precision (w, sign);
329 p = wi::min_precision (arg, sign);
330 return MIN (p, prec);
332 while (CONVERT_EXPR_P (arg)
333 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
334 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
336 arg = TREE_OPERAND (arg, 0);
337 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
339 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
341 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
342 return prec + (orig_sign != sign);
343 prec = TYPE_PRECISION (TREE_TYPE (arg));
346 return prec + (orig_sign != sign);
348 if (TREE_CODE (arg) != SSA_NAME)
349 return prec + (orig_sign != sign);
350 wide_int arg_min, arg_max;
351 while (get_range_info (arg, &arg_min, &arg_max) != VR_RANGE)
353 gimple *g = SSA_NAME_DEF_STMT (arg);
354 if (is_gimple_assign (g)
355 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
357 tree t = gimple_assign_rhs1 (g);
358 if (INTEGRAL_TYPE_P (TREE_TYPE (t))
359 && TYPE_PRECISION (TREE_TYPE (t)) <= prec)
362 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
364 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
366 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
367 return prec + (orig_sign != sign);
368 prec = TYPE_PRECISION (TREE_TYPE (arg));
371 return prec + (orig_sign != sign);
375 return prec + (orig_sign != sign);
377 if (sign == TYPE_SIGN (TREE_TYPE (arg)))
379 int p1 = wi::min_precision (arg_min, sign);
380 int p2 = wi::min_precision (arg_max, sign);
382 prec = MIN (prec, p1);
384 else if (sign == UNSIGNED && !wi::neg_p (arg_min, SIGNED))
386 int p = wi::min_precision (arg_max, UNSIGNED);
387 prec = MIN (prec, p);
389 return prec + (orig_sign != sign);
392 /* Helper for expand_*_overflow. Store RES into the __real__ part
393 of TARGET. If RES has larger MODE than __real__ part of TARGET,
394 set the __imag__ part to 1 if RES doesn't fit into it. */
397 expand_arith_overflow_result_store (tree lhs, rtx target,
398 machine_mode mode, rtx res)
400 machine_mode tgtmode = GET_MODE_INNER (GET_MODE (target));
404 rtx_code_label *done_label = gen_label_rtx ();
405 int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
406 lres = convert_modes (tgtmode, mode, res, uns);
407 gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode));
408 do_compare_rtx_and_jump (res, convert_modes (mode, tgtmode, lres, uns),
409 EQ, true, mode, NULL_RTX, NULL, done_label,
411 write_complex_part (target, const1_rtx, true);
412 emit_label (done_label);
414 write_complex_part (target, lres, false);
417 /* Helper for expand_*_overflow. Store RES into TARGET. */
420 expand_ubsan_result_store (rtx target, rtx res)
422 if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
423 /* If this is a scalar in a register that is stored in a wider mode
424 than the declared mode, compute the result into its declared mode
425 and then convert to the wider mode. Our value is the computed
427 convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target));
429 emit_move_insn (target, res);
432 /* Add sub/add overflow checking to the statement STMT.
433 CODE says whether the operation is +, or -. */
436 expand_addsub_overflow (location_t loc, tree_code code, tree lhs,
437 tree arg0, tree arg1, bool unsr_p, bool uns0_p,
438 bool uns1_p, bool is_ubsan)
440 rtx res, target = NULL_RTX;
442 rtx_code_label *done_label = gen_label_rtx ();
443 rtx_code_label *do_error = gen_label_rtx ();
444 do_pending_stack_adjust ();
445 rtx op0 = expand_normal (arg0);
446 rtx op1 = expand_normal (arg1);
447 machine_mode mode = TYPE_MODE (TREE_TYPE (arg0));
448 int prec = GET_MODE_PRECISION (mode);
449 rtx sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
453 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
457 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
459 write_complex_part (target, const0_rtx, true);
462 /* We assume both operands and result have the same precision
463 here (GET_MODE_BITSIZE (mode)), S stands for signed type
464 with that precision, U for unsigned type with that precision,
465 sgn for unsigned most significant bit in that precision.
466 s1 is signed first operand, u1 is unsigned first operand,
467 s2 is signed second operand, u2 is unsigned second operand,
468 sr is signed result, ur is unsigned result and the following
469 rules say how to compute result (which is always result of
470 the operands as if both were unsigned, cast to the right
471 signedness) and how to compute whether operation overflowed.
474 res = (S) ((U) s1 + (U) s2)
475 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
477 res = (S) ((U) s1 - (U) s2)
478 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
481 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
484 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
486 res = (S) ((U) s1 + u2)
487 ovf = ((U) res ^ sgn) < u2
492 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
494 res = (S) ((U) s1 - u2)
495 ovf = u2 > ((U) s1 ^ sgn)
498 ovf = s1 < 0 || u2 > (U) s1
501 ovf = u1 >= ((U) s2 ^ sgn)
506 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
508 res = (U) s1 + (U) s2
509 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
512 ovf = (U) res < u2 || res < 0
515 ovf = u1 >= u2 ? res < 0 : res >= 0
517 res = (U) s1 - (U) s2
518 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
520 if (code == PLUS_EXPR && uns0_p && !uns1_p)
522 /* PLUS_EXPR is commutative, if operand signedness differs,
523 canonicalize to the first operand being signed and second
524 unsigned to simplify following code. */
525 std::swap (op0, op1);
526 std::swap (arg0, arg1);
532 if (uns0_p && uns1_p && unsr_p)
534 /* Compute the operation. On RTL level, the addition is always
536 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
537 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
539 /* For PLUS_EXPR, the operation is commutative, so we can pick
540 operand to compare against. For prec <= BITS_PER_WORD, I think
541 preferring REG operand is better over CONST_INT, because
542 the CONST_INT might enlarge the instruction or CSE would need
543 to figure out we'd already loaded it into a register before.
544 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
545 as then the multi-word comparison can be perhaps simplified. */
546 if (code == PLUS_EXPR
547 && (prec <= BITS_PER_WORD
548 ? (CONST_SCALAR_INT_P (op0) && REG_P (op1))
549 : CONST_SCALAR_INT_P (op1)))
551 do_compare_rtx_and_jump (res, tem, code == PLUS_EXPR ? GEU : LEU,
552 true, mode, NULL_RTX, NULL, done_label,
558 if (!uns0_p && uns1_p && !unsr_p)
560 /* Compute the operation. On RTL level, the addition is always
562 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
563 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
564 rtx tem = expand_binop (mode, add_optab,
565 code == PLUS_EXPR ? res : op0, sgn,
566 NULL_RTX, false, OPTAB_LIB_WIDEN);
567 do_compare_rtx_and_jump (tem, op1, GEU, true, mode, NULL_RTX, NULL,
568 done_label, PROB_VERY_LIKELY);
573 if (code == PLUS_EXPR && !uns0_p && uns1_p && unsr_p)
575 op1 = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
577 /* As we've changed op1, we have to avoid using the value range
578 for the original argument. */
579 arg1 = error_mark_node;
585 if (code == MINUS_EXPR && uns0_p && !uns1_p && unsr_p)
587 op0 = expand_binop (mode, add_optab, op0, sgn, NULL_RTX, false,
589 /* As we've changed op0, we have to avoid using the value range
590 for the original argument. */
591 arg0 = error_mark_node;
597 if (code == MINUS_EXPR && !uns0_p && uns1_p && unsr_p)
599 /* Compute the operation. On RTL level, the addition is always
601 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
603 int pos_neg = get_range_pos_neg (arg0);
605 /* If ARG0 is known to be always negative, this is always overflow. */
606 emit_jump (do_error);
607 else if (pos_neg == 3)
608 /* If ARG0 is not known to be always positive, check at runtime. */
609 do_compare_rtx_and_jump (op0, const0_rtx, LT, false, mode, NULL_RTX,
610 NULL, do_error, PROB_VERY_UNLIKELY);
611 do_compare_rtx_and_jump (op1, op0, LEU, true, mode, NULL_RTX, NULL,
612 done_label, PROB_VERY_LIKELY);
617 if (code == MINUS_EXPR && uns0_p && !uns1_p && !unsr_p)
619 /* Compute the operation. On RTL level, the addition is always
621 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
623 rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
625 do_compare_rtx_and_jump (op0, tem, LTU, true, mode, NULL_RTX, NULL,
626 done_label, PROB_VERY_LIKELY);
631 if (code == PLUS_EXPR && uns0_p && uns1_p && !unsr_p)
633 /* Compute the operation. On RTL level, the addition is always
635 res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false,
637 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
638 NULL, do_error, PROB_VERY_UNLIKELY);
640 /* The operation is commutative, so we can pick operand to compare
641 against. For prec <= BITS_PER_WORD, I think preferring REG operand
642 is better over CONST_INT, because the CONST_INT might enlarge the
643 instruction or CSE would need to figure out we'd already loaded it
644 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
645 might be more beneficial, as then the multi-word comparison can be
646 perhaps simplified. */
647 if (prec <= BITS_PER_WORD
648 ? (CONST_SCALAR_INT_P (op1) && REG_P (op0))
649 : CONST_SCALAR_INT_P (op0))
651 do_compare_rtx_and_jump (res, tem, GEU, true, mode, NULL_RTX, NULL,
652 done_label, PROB_VERY_LIKELY);
657 if (!uns0_p && !uns1_p && unsr_p)
659 /* Compute the operation. On RTL level, the addition is always
661 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
662 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
663 int pos_neg = get_range_pos_neg (arg1);
664 if (code == PLUS_EXPR)
666 int pos_neg0 = get_range_pos_neg (arg0);
667 if (pos_neg0 != 3 && pos_neg == 3)
669 std::swap (op0, op1);
676 tem = expand_binop (mode, ((pos_neg == 1) ^ (code == MINUS_EXPR))
677 ? and_optab : ior_optab,
678 op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN);
679 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL,
680 NULL, done_label, PROB_VERY_LIKELY);
684 rtx_code_label *do_ior_label = gen_label_rtx ();
685 do_compare_rtx_and_jump (op1, const0_rtx,
686 code == MINUS_EXPR ? GE : LT, false, mode,
687 NULL_RTX, NULL, do_ior_label,
689 tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false,
691 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
692 NULL, done_label, PROB_VERY_LIKELY);
693 emit_jump (do_error);
694 emit_label (do_ior_label);
695 tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false,
697 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
698 NULL, done_label, PROB_VERY_LIKELY);
704 if (code == MINUS_EXPR && uns0_p && uns1_p && !unsr_p)
706 /* Compute the operation. On RTL level, the addition is always
708 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
710 rtx_code_label *op0_geu_op1 = gen_label_rtx ();
711 do_compare_rtx_and_jump (op0, op1, GEU, true, mode, NULL_RTX, NULL,
712 op0_geu_op1, PROB_EVEN);
713 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
714 NULL, done_label, PROB_VERY_LIKELY);
715 emit_jump (do_error);
716 emit_label (op0_geu_op1);
717 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
718 NULL, done_label, PROB_VERY_LIKELY);
722 gcc_assert (!uns0_p && !uns1_p && !unsr_p);
726 enum insn_code icode;
727 icode = optab_handler (code == PLUS_EXPR ? addv4_optab : subv4_optab, mode);
728 if (icode != CODE_FOR_nothing)
730 struct expand_operand ops[4];
731 rtx_insn *last = get_last_insn ();
733 res = gen_reg_rtx (mode);
734 create_output_operand (&ops[0], res, mode);
735 create_input_operand (&ops[1], op0, mode);
736 create_input_operand (&ops[2], op1, mode);
737 create_fixed_operand (&ops[3], do_error);
738 if (maybe_expand_insn (icode, 4, ops))
740 last = get_last_insn ();
741 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
743 && any_condjump_p (last)
744 && !find_reg_note (last, REG_BR_PROB, 0))
745 add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
746 emit_jump (done_label);
750 delete_insns_since (last);
751 icode = CODE_FOR_nothing;
755 if (icode == CODE_FOR_nothing)
757 rtx_code_label *sub_check = gen_label_rtx ();
760 /* Compute the operation. On RTL level, the addition is always
762 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
763 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
765 /* If we can prove one of the arguments (for MINUS_EXPR only
766 the second operand, as subtraction is not commutative) is always
767 non-negative or always negative, we can do just one comparison
768 and conditional jump instead of 2 at runtime, 3 present in the
769 emitted code. If one of the arguments is CONST_INT, all we
770 need is to make sure it is op1, then the first
771 do_compare_rtx_and_jump will be just folded. Otherwise try
772 to use range info if available. */
773 if (code == PLUS_EXPR && CONST_INT_P (op0))
774 std::swap (op0, op1);
775 else if (CONST_INT_P (op1))
777 else if (code == PLUS_EXPR && TREE_CODE (arg0) == SSA_NAME)
779 pos_neg = get_range_pos_neg (arg0);
781 std::swap (op0, op1);
783 if (pos_neg == 3 && !CONST_INT_P (op1) && TREE_CODE (arg1) == SSA_NAME)
784 pos_neg = get_range_pos_neg (arg1);
786 /* If the op1 is negative, we have to use a different check. */
788 do_compare_rtx_and_jump (op1, const0_rtx, LT, false, mode, NULL_RTX,
789 NULL, sub_check, PROB_EVEN);
791 /* Compare the result of the operation with one of the operands. */
793 do_compare_rtx_and_jump (res, op0, code == PLUS_EXPR ? GE : LE,
794 false, mode, NULL_RTX, NULL, done_label,
797 /* If we get here, we have to print the error. */
800 emit_jump (do_error);
802 emit_label (sub_check);
805 /* We have k = a + b for b < 0 here. k <= a must hold. */
807 do_compare_rtx_and_jump (res, op0, code == PLUS_EXPR ? LE : GE,
808 false, mode, NULL_RTX, NULL, done_label,
813 emit_label (do_error);
816 /* Expand the ubsan builtin call. */
818 fn = ubsan_build_overflow_builtin (code, loc, TREE_TYPE (arg0),
822 do_pending_stack_adjust ();
825 write_complex_part (target, const1_rtx, true);
828 emit_label (done_label);
833 expand_ubsan_result_store (target, res);
837 res = expand_binop (mode, add_optab, res, sgn, NULL_RTX, false,
840 expand_arith_overflow_result_store (lhs, target, mode, res);
845 /* Add negate overflow checking to the statement STMT. */
848 expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan)
852 rtx_code_label *done_label, *do_error;
853 rtx target = NULL_RTX;
855 done_label = gen_label_rtx ();
856 do_error = gen_label_rtx ();
858 do_pending_stack_adjust ();
859 op1 = expand_normal (arg1);
861 machine_mode mode = TYPE_MODE (TREE_TYPE (arg1));
864 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
866 write_complex_part (target, const0_rtx, true);
869 enum insn_code icode = optab_handler (negv3_optab, mode);
870 if (icode != CODE_FOR_nothing)
872 struct expand_operand ops[3];
873 rtx_insn *last = get_last_insn ();
875 res = gen_reg_rtx (mode);
876 create_output_operand (&ops[0], res, mode);
877 create_input_operand (&ops[1], op1, mode);
878 create_fixed_operand (&ops[2], do_error);
879 if (maybe_expand_insn (icode, 3, ops))
881 last = get_last_insn ();
882 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
884 && any_condjump_p (last)
885 && !find_reg_note (last, REG_BR_PROB, 0))
886 add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
887 emit_jump (done_label);
891 delete_insns_since (last);
892 icode = CODE_FOR_nothing;
896 if (icode == CODE_FOR_nothing)
898 /* Compute the operation. On RTL level, the addition is always
900 res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
902 /* Compare the operand with the most negative value. */
903 rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
904 do_compare_rtx_and_jump (op1, minv, NE, true, mode, NULL_RTX, NULL,
905 done_label, PROB_VERY_LIKELY);
908 emit_label (do_error);
911 /* Expand the ubsan builtin call. */
913 fn = ubsan_build_overflow_builtin (NEGATE_EXPR, loc, TREE_TYPE (arg1),
917 do_pending_stack_adjust ();
920 write_complex_part (target, const1_rtx, true);
923 emit_label (done_label);
928 expand_ubsan_result_store (target, res);
930 expand_arith_overflow_result_store (lhs, target, mode, res);
934 /* Add mul overflow checking to the statement STMT. */
937 expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1,
938 bool unsr_p, bool uns0_p, bool uns1_p, bool is_ubsan)
942 rtx_code_label *done_label, *do_error;
943 rtx target = NULL_RTX;
945 enum insn_code icode;
947 done_label = gen_label_rtx ();
948 do_error = gen_label_rtx ();
950 do_pending_stack_adjust ();
951 op0 = expand_normal (arg0);
952 op1 = expand_normal (arg1);
954 machine_mode mode = TYPE_MODE (TREE_TYPE (arg0));
958 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
960 write_complex_part (target, const0_rtx, true);
964 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
966 /* We assume both operands and result have the same precision
967 here (GET_MODE_BITSIZE (mode)), S stands for signed type
968 with that precision, U for unsigned type with that precision,
969 sgn for unsigned most significant bit in that precision.
970 s1 is signed first operand, u1 is unsigned first operand,
971 s2 is signed second operand, u2 is unsigned second operand,
972 sr is signed result, ur is unsigned result and the following
973 rules say how to compute result (which is always result of
974 the operands as if both were unsigned, cast to the right
975 signedness) and how to compute whether operation overflowed.
976 main_ovf (false) stands for jump on signed multiplication
977 overflow or the main algorithm with uns == false.
978 main_ovf (true) stands for jump on unsigned multiplication
979 overflow or the main algorithm with uns == true.
982 res = (S) ((U) s1 * (U) s2)
983 ovf = main_ovf (false)
986 ovf = main_ovf (true)
989 ovf = (s1 < 0 && u2) || main_ovf (true)
992 ovf = res < 0 || main_ovf (true)
994 res = (S) ((U) s1 * u2)
995 ovf = (S) u2 >= 0 ? main_ovf (false)
996 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
998 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
999 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1001 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1003 if (uns0_p && !uns1_p)
1005 /* Multiplication is commutative, if operand signedness differs,
1006 canonicalize to the first operand being signed and second
1007 unsigned to simplify following code. */
1008 std::swap (op0, op1);
1009 std::swap (arg0, arg1);
1014 int pos_neg0 = get_range_pos_neg (arg0);
1015 int pos_neg1 = get_range_pos_neg (arg1);
1018 if (!uns0_p && uns1_p && unsr_p)
1023 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1026 /* If s1 is negative, avoid the main code, just multiply and
1027 signal overflow if op1 is not 0. */
1028 struct separate_ops ops;
1029 ops.code = MULT_EXPR;
1030 ops.type = TREE_TYPE (arg1);
1031 ops.op0 = make_tree (ops.type, op0);
1032 ops.op1 = make_tree (ops.type, op1);
1033 ops.op2 = NULL_TREE;
1035 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1036 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1037 NULL, done_label, PROB_VERY_LIKELY);
1038 goto do_error_label;
1040 rtx_code_label *do_main_label;
1041 do_main_label = gen_label_rtx ();
1042 do_compare_rtx_and_jump (op0, const0_rtx, GE, false, mode, NULL_RTX,
1043 NULL, do_main_label, PROB_VERY_LIKELY);
1044 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1045 NULL, do_main_label, PROB_VERY_LIKELY);
1046 write_complex_part (target, const1_rtx, true);
1047 emit_label (do_main_label);
1055 if (uns0_p && uns1_p && !unsr_p)
1058 /* Rest of handling of this case after res is computed. */
1063 if (!uns0_p && uns1_p && !unsr_p)
1070 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1071 avoid the main code, just multiply and signal overflow
1072 unless 0 * u2 or -1 * ((U) Smin). */
1073 struct separate_ops ops;
1074 ops.code = MULT_EXPR;
1075 ops.type = TREE_TYPE (arg1);
1076 ops.op0 = make_tree (ops.type, op0);
1077 ops.op1 = make_tree (ops.type, op1);
1078 ops.op2 = NULL_TREE;
1080 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1081 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1082 NULL, done_label, PROB_VERY_LIKELY);
1083 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
1084 NULL, do_error, PROB_VERY_UNLIKELY);
1086 prec = GET_MODE_PRECISION (mode);
1088 sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
1089 do_compare_rtx_and_jump (op1, sgn, EQ, true, mode, NULL_RTX,
1090 NULL, done_label, PROB_VERY_LIKELY);
1091 goto do_error_label;
1093 /* Rest of handling of this case after res is computed. */
1101 if (!uns0_p && !uns1_p && unsr_p)
1104 switch (pos_neg0 | pos_neg1)
1106 case 1: /* Both operands known to be non-negative. */
1108 case 2: /* Both operands known to be negative. */
1109 op0 = expand_unop (mode, neg_optab, op0, NULL_RTX, false);
1110 op1 = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1111 /* Avoid looking at arg0/arg1 ranges, as we've changed
1113 arg0 = error_mark_node;
1114 arg1 = error_mark_node;
1117 if ((pos_neg0 ^ pos_neg1) == 3)
1119 /* If one operand is known to be negative and the other
1120 non-negative, this overflows always, unless the non-negative
1121 one is 0. Just do normal multiply and set overflow
1122 unless one of the operands is 0. */
1123 struct separate_ops ops;
1124 ops.code = MULT_EXPR;
1126 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1128 ops.op0 = make_tree (ops.type, op0);
1129 ops.op1 = make_tree (ops.type, op1);
1130 ops.op2 = NULL_TREE;
1132 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1133 tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
1135 do_compare_rtx_and_jump (tem, const0_rtx, EQ, true, mode,
1136 NULL_RTX, NULL, done_label,
1138 goto do_error_label;
1140 /* The general case, do all the needed comparisons at runtime. */
1141 rtx_code_label *do_main_label, *after_negate_label;
1143 rop0 = gen_reg_rtx (mode);
1144 rop1 = gen_reg_rtx (mode);
1145 emit_move_insn (rop0, op0);
1146 emit_move_insn (rop1, op1);
1149 do_main_label = gen_label_rtx ();
1150 after_negate_label = gen_label_rtx ();
1151 tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
1153 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1154 NULL, after_negate_label, PROB_VERY_LIKELY);
1155 /* Both arguments negative here, negate them and continue with
1156 normal unsigned overflow checking multiplication. */
1157 emit_move_insn (op0, expand_unop (mode, neg_optab, op0,
1159 emit_move_insn (op1, expand_unop (mode, neg_optab, op1,
1161 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1163 arg0 = error_mark_node;
1164 arg1 = error_mark_node;
1165 emit_jump (do_main_label);
1166 emit_label (after_negate_label);
1167 tem2 = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1169 do_compare_rtx_and_jump (tem2, const0_rtx, GE, false, mode, NULL_RTX,
1170 NULL, do_main_label, PROB_VERY_LIKELY);
1171 /* One argument is negative here, the other positive. This
1172 overflows always, unless one of the arguments is 0. But
1173 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1174 is, thus we can keep do_main code oring in overflow as is. */
1175 do_compare_rtx_and_jump (tem, const0_rtx, EQ, true, mode, NULL_RTX,
1176 NULL, do_main_label, PROB_VERY_LIKELY);
1177 write_complex_part (target, const1_rtx, true);
1178 emit_label (do_main_label);
1186 type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode), uns);
1187 sign = uns ? UNSIGNED : SIGNED;
1188 icode = optab_handler (uns ? umulv4_optab : mulv4_optab, mode);
1189 if (icode != CODE_FOR_nothing)
1191 struct expand_operand ops[4];
1192 rtx_insn *last = get_last_insn ();
1194 res = gen_reg_rtx (mode);
1195 create_output_operand (&ops[0], res, mode);
1196 create_input_operand (&ops[1], op0, mode);
1197 create_input_operand (&ops[2], op1, mode);
1198 create_fixed_operand (&ops[3], do_error);
1199 if (maybe_expand_insn (icode, 4, ops))
1201 last = get_last_insn ();
1202 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1204 && any_condjump_p (last)
1205 && !find_reg_note (last, REG_BR_PROB, 0))
1206 add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
1207 emit_jump (done_label);
1211 delete_insns_since (last);
1212 icode = CODE_FOR_nothing;
1216 if (icode == CODE_FOR_nothing)
1218 struct separate_ops ops;
1219 int prec = GET_MODE_PRECISION (mode);
1220 machine_mode hmode = mode_for_size (prec / 2, MODE_INT, 1);
1221 ops.op0 = make_tree (type, op0);
1222 ops.op1 = make_tree (type, op1);
1223 ops.op2 = NULL_TREE;
1225 if (GET_MODE_2XWIDER_MODE (mode) != VOIDmode
1226 && targetm.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode)))
1228 machine_mode wmode = GET_MODE_2XWIDER_MODE (mode);
1229 ops.code = WIDEN_MULT_EXPR;
1231 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), uns);
1233 res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
1234 rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, prec,
1236 hipart = gen_lowpart (mode, hipart);
1237 res = gen_lowpart (mode, res);
1239 /* For the unsigned multiplication, there was overflow if
1240 HIPART is non-zero. */
1241 do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
1242 NULL_RTX, NULL, done_label,
1246 rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
1248 /* RES is low half of the double width result, HIPART
1249 the high half. There was overflow if
1250 HIPART is different from RES < 0 ? -1 : 0. */
1251 do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
1252 NULL_RTX, NULL, done_label,
1256 else if (hmode != BLKmode && 2 * GET_MODE_PRECISION (hmode) == prec)
1258 rtx_code_label *large_op0 = gen_label_rtx ();
1259 rtx_code_label *small_op0_large_op1 = gen_label_rtx ();
1260 rtx_code_label *one_small_one_large = gen_label_rtx ();
1261 rtx_code_label *both_ops_large = gen_label_rtx ();
1262 rtx_code_label *after_hipart_neg = uns ? NULL : gen_label_rtx ();
1263 rtx_code_label *after_lopart_neg = uns ? NULL : gen_label_rtx ();
1264 rtx_code_label *do_overflow = gen_label_rtx ();
1265 rtx_code_label *hipart_different = uns ? NULL : gen_label_rtx ();
1267 unsigned int hprec = GET_MODE_PRECISION (hmode);
1268 rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
1270 hipart0 = gen_lowpart (hmode, hipart0);
1271 rtx lopart0 = gen_lowpart (hmode, op0);
1272 rtx signbit0 = const0_rtx;
1274 signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
1276 rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
1278 hipart1 = gen_lowpart (hmode, hipart1);
1279 rtx lopart1 = gen_lowpart (hmode, op1);
1280 rtx signbit1 = const0_rtx;
1282 signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
1285 res = gen_reg_rtx (mode);
1287 /* True if op0 resp. op1 are known to be in the range of
1289 bool op0_small_p = false;
1290 bool op1_small_p = false;
1291 /* True if op0 resp. op1 are known to have all zeros or all ones
1292 in the upper half of bits, but are not known to be
1294 bool op0_medium_p = false;
1295 bool op1_medium_p = false;
1296 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1297 nonnegative, 1 if unknown. */
1303 else if (pos_neg0 == 2)
1307 else if (pos_neg1 == 2)
1310 unsigned int mprec0 = prec;
1311 if (arg0 != error_mark_node)
1312 mprec0 = get_min_precision (arg0, sign);
1313 if (mprec0 <= hprec)
1315 else if (!uns && mprec0 <= hprec + 1)
1316 op0_medium_p = true;
1317 unsigned int mprec1 = prec;
1318 if (arg1 != error_mark_node)
1319 mprec1 = get_min_precision (arg1, sign);
1320 if (mprec1 <= hprec)
1322 else if (!uns && mprec1 <= hprec + 1)
1323 op1_medium_p = true;
1325 int smaller_sign = 1;
1326 int larger_sign = 1;
1329 smaller_sign = op0_sign;
1330 larger_sign = op1_sign;
1332 else if (op1_small_p)
1334 smaller_sign = op1_sign;
1335 larger_sign = op0_sign;
1337 else if (op0_sign == op1_sign)
1339 smaller_sign = op0_sign;
1340 larger_sign = op0_sign;
1344 do_compare_rtx_and_jump (signbit0, hipart0, NE, true, hmode,
1345 NULL_RTX, NULL, large_op0,
1349 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
1350 NULL_RTX, NULL, small_op0_large_op1,
1353 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1354 hmode to mode, the multiplication will never overflow. We can
1355 do just one hmode x hmode => mode widening multiplication. */
1356 rtx lopart0s = lopart0, lopart1s = lopart1;
1357 if (GET_CODE (lopart0) == SUBREG)
1359 lopart0s = shallow_copy_rtx (lopart0);
1360 SUBREG_PROMOTED_VAR_P (lopart0s) = 1;
1361 SUBREG_PROMOTED_SET (lopart0s, uns ? SRP_UNSIGNED : SRP_SIGNED);
1363 if (GET_CODE (lopart1) == SUBREG)
1365 lopart1s = shallow_copy_rtx (lopart1);
1366 SUBREG_PROMOTED_VAR_P (lopart1s) = 1;
1367 SUBREG_PROMOTED_SET (lopart1s, uns ? SRP_UNSIGNED : SRP_SIGNED);
1369 tree halfstype = build_nonstandard_integer_type (hprec, uns);
1370 ops.op0 = make_tree (halfstype, lopart0s);
1371 ops.op1 = make_tree (halfstype, lopart1s);
1372 ops.code = WIDEN_MULT_EXPR;
1375 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1376 emit_move_insn (res, thisres);
1377 emit_jump (done_label);
1379 emit_label (small_op0_large_op1);
1381 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1382 but op1 is not, just swap the arguments and handle it as op1
1383 sign/zero extended, op0 not. */
1384 rtx larger = gen_reg_rtx (mode);
1385 rtx hipart = gen_reg_rtx (hmode);
1386 rtx lopart = gen_reg_rtx (hmode);
1387 emit_move_insn (larger, op1);
1388 emit_move_insn (hipart, hipart1);
1389 emit_move_insn (lopart, lopart0);
1390 emit_jump (one_small_one_large);
1392 emit_label (large_op0);
1395 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
1396 NULL_RTX, NULL, both_ops_large,
1399 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1400 but op0 is not, prepare larger, hipart and lopart pseudos and
1401 handle it together with small_op0_large_op1. */
1402 emit_move_insn (larger, op0);
1403 emit_move_insn (hipart, hipart0);
1404 emit_move_insn (lopart, lopart1);
1406 emit_label (one_small_one_large);
1408 /* lopart is the low part of the operand that is sign extended
1409 to mode, larger is the other operand, hipart is the
1410 high part of larger and lopart0 and lopart1 are the low parts
1412 We perform lopart0 * lopart1 and lopart * hipart widening
1414 tree halfutype = build_nonstandard_integer_type (hprec, 1);
1415 ops.op0 = make_tree (halfutype, lopart0);
1416 ops.op1 = make_tree (halfutype, lopart1);
1418 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1420 ops.op0 = make_tree (halfutype, lopart);
1421 ops.op1 = make_tree (halfutype, hipart);
1422 rtx loxhi = gen_reg_rtx (mode);
1423 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1424 emit_move_insn (loxhi, tem);
1428 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1429 if (larger_sign == 0)
1430 emit_jump (after_hipart_neg);
1431 else if (larger_sign != -1)
1432 do_compare_rtx_and_jump (hipart, const0_rtx, GE, false, hmode,
1433 NULL_RTX, NULL, after_hipart_neg,
1436 tem = convert_modes (mode, hmode, lopart, 1);
1437 tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
1438 tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
1440 emit_move_insn (loxhi, tem);
1442 emit_label (after_hipart_neg);
1444 /* if (lopart < 0) loxhi -= larger; */
1445 if (smaller_sign == 0)
1446 emit_jump (after_lopart_neg);
1447 else if (smaller_sign != -1)
1448 do_compare_rtx_and_jump (lopart, const0_rtx, GE, false, hmode,
1449 NULL_RTX, NULL, after_lopart_neg,
1452 tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
1454 emit_move_insn (loxhi, tem);
1456 emit_label (after_lopart_neg);
1459 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1460 tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
1461 tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
1463 emit_move_insn (loxhi, tem);
1465 /* if (loxhi >> (bitsize / 2)
1466 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1467 if (loxhi >> (bitsize / 2) == 0 (if uns). */
1468 rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
1470 hipartloxhi = gen_lowpart (hmode, hipartloxhi);
1471 rtx signbitloxhi = const0_rtx;
1473 signbitloxhi = expand_shift (RSHIFT_EXPR, hmode,
1474 gen_lowpart (hmode, loxhi),
1475 hprec - 1, NULL_RTX, 0);
1477 do_compare_rtx_and_jump (signbitloxhi, hipartloxhi, NE, true, hmode,
1478 NULL_RTX, NULL, do_overflow,
1479 PROB_VERY_UNLIKELY);
1481 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1482 rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
1484 tem = convert_modes (mode, hmode, gen_lowpart (hmode, lo0xlo1), 1);
1486 tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
1489 emit_move_insn (res, tem);
1490 emit_jump (done_label);
1492 emit_label (both_ops_large);
1494 /* If both operands are large (not sign (!uns) or zero (uns)
1495 extended from hmode), then perform the full multiplication
1496 which will be the result of the operation.
1497 The only cases which don't overflow are for signed multiplication
1498 some cases where both hipart0 and highpart1 are 0 or -1.
1499 For unsigned multiplication when high parts are both non-zero
1500 this overflows always. */
1501 ops.code = MULT_EXPR;
1502 ops.op0 = make_tree (type, op0);
1503 ops.op1 = make_tree (type, op1);
1504 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1505 emit_move_insn (res, tem);
1511 tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
1512 NULL_RTX, 1, OPTAB_DIRECT);
1513 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
1514 NULL_RTX, NULL, do_error,
1515 PROB_VERY_UNLIKELY);
1520 tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
1521 NULL_RTX, 1, OPTAB_DIRECT);
1522 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
1523 NULL_RTX, NULL, do_error,
1524 PROB_VERY_UNLIKELY);
1527 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1528 the same, overflow happened if res is negative, if they are
1529 different, overflow happened if res is positive. */
1530 if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
1531 emit_jump (hipart_different);
1532 else if (op0_sign == 1 || op1_sign == 1)
1533 do_compare_rtx_and_jump (hipart0, hipart1, NE, true, hmode,
1534 NULL_RTX, NULL, hipart_different,
1537 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode,
1538 NULL_RTX, NULL, do_error,
1539 PROB_VERY_UNLIKELY);
1540 emit_jump (done_label);
1542 emit_label (hipart_different);
1544 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode,
1545 NULL_RTX, NULL, do_error,
1546 PROB_VERY_UNLIKELY);
1547 emit_jump (done_label);
1550 emit_label (do_overflow);
1552 /* Overflow, do full multiplication and fallthru into do_error. */
1553 ops.op0 = make_tree (type, op0);
1554 ops.op1 = make_tree (type, op1);
1555 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1556 emit_move_insn (res, tem);
1560 gcc_assert (!is_ubsan);
1561 ops.code = MULT_EXPR;
1563 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1564 emit_jump (done_label);
1569 emit_label (do_error);
1572 /* Expand the ubsan builtin call. */
1574 fn = ubsan_build_overflow_builtin (MULT_EXPR, loc, TREE_TYPE (arg0),
1578 do_pending_stack_adjust ();
1581 write_complex_part (target, const1_rtx, true);
1584 emit_label (done_label);
1587 if (uns0_p && uns1_p && !unsr_p)
1589 rtx_code_label *all_done_label = gen_label_rtx ();
1590 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
1591 NULL, all_done_label, PROB_VERY_LIKELY);
1592 write_complex_part (target, const1_rtx, true);
1593 emit_label (all_done_label);
1597 if (!uns0_p && uns1_p && !unsr_p && pos_neg1 == 3)
1599 rtx_code_label *all_done_label = gen_label_rtx ();
1600 rtx_code_label *set_noovf = gen_label_rtx ();
1601 do_compare_rtx_and_jump (op1, const0_rtx, GE, false, mode, NULL_RTX,
1602 NULL, all_done_label, PROB_VERY_LIKELY);
1603 write_complex_part (target, const1_rtx, true);
1604 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1605 NULL, set_noovf, PROB_VERY_LIKELY);
1606 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
1607 NULL, all_done_label, PROB_VERY_UNLIKELY);
1608 do_compare_rtx_and_jump (op1, res, NE, true, mode, NULL_RTX, NULL,
1609 all_done_label, PROB_VERY_UNLIKELY);
1610 emit_label (set_noovf);
1611 write_complex_part (target, const0_rtx, true);
1612 emit_label (all_done_label);
1618 expand_ubsan_result_store (target, res);
1620 expand_arith_overflow_result_store (lhs, target, mode, res);
1624 /* Expand UBSAN_CHECK_ADD call STMT. */
1627 expand_UBSAN_CHECK_ADD (gcall *stmt)
1629 location_t loc = gimple_location (stmt);
1630 tree lhs = gimple_call_lhs (stmt);
1631 tree arg0 = gimple_call_arg (stmt, 0);
1632 tree arg1 = gimple_call_arg (stmt, 1);
1633 expand_addsub_overflow (loc, PLUS_EXPR, lhs, arg0, arg1,
1634 false, false, false, true);
1637 /* Expand UBSAN_CHECK_SUB call STMT. */
1640 expand_UBSAN_CHECK_SUB (gcall *stmt)
1642 location_t loc = gimple_location (stmt);
1643 tree lhs = gimple_call_lhs (stmt);
1644 tree arg0 = gimple_call_arg (stmt, 0);
1645 tree arg1 = gimple_call_arg (stmt, 1);
1646 if (integer_zerop (arg0))
1647 expand_neg_overflow (loc, lhs, arg1, true);
1649 expand_addsub_overflow (loc, MINUS_EXPR, lhs, arg0, arg1,
1650 false, false, false, true);
1653 /* Expand UBSAN_CHECK_MUL call STMT. */
1656 expand_UBSAN_CHECK_MUL (gcall *stmt)
1658 location_t loc = gimple_location (stmt);
1659 tree lhs = gimple_call_lhs (stmt);
1660 tree arg0 = gimple_call_arg (stmt, 0);
1661 tree arg1 = gimple_call_arg (stmt, 1);
1662 expand_mul_overflow (loc, lhs, arg0, arg1, false, false, false, true);
1665 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
1668 expand_arith_overflow (enum tree_code code, gimple *stmt)
1670 tree lhs = gimple_call_lhs (stmt);
1671 if (lhs == NULL_TREE)
1673 tree arg0 = gimple_call_arg (stmt, 0);
1674 tree arg1 = gimple_call_arg (stmt, 1);
1675 tree type = TREE_TYPE (TREE_TYPE (lhs));
1676 int uns0_p = TYPE_UNSIGNED (TREE_TYPE (arg0));
1677 int uns1_p = TYPE_UNSIGNED (TREE_TYPE (arg1));
1678 int unsr_p = TYPE_UNSIGNED (type);
1679 int prec0 = TYPE_PRECISION (TREE_TYPE (arg0));
1680 int prec1 = TYPE_PRECISION (TREE_TYPE (arg1));
1681 int precres = TYPE_PRECISION (type);
1682 location_t loc = gimple_location (stmt);
1683 if (!uns0_p && get_range_pos_neg (arg0) == 1)
1685 if (!uns1_p && get_range_pos_neg (arg1) == 1)
1687 int pr = get_min_precision (arg0, uns0_p ? UNSIGNED : SIGNED);
1688 prec0 = MIN (prec0, pr);
1689 pr = get_min_precision (arg1, uns1_p ? UNSIGNED : SIGNED);
1690 prec1 = MIN (prec1, pr);
1692 /* If uns0_p && uns1_p, precop is minimum needed precision
1693 of unsigned type to hold the exact result, otherwise
1694 precop is minimum needed precision of signed type to
1695 hold the exact result. */
1697 if (code == MULT_EXPR)
1698 precop = prec0 + prec1 + (uns0_p != uns1_p);
1701 if (uns0_p == uns1_p)
1702 precop = MAX (prec0, prec1) + 1;
1704 precop = MAX (prec0 + 1, prec1) + 1;
1706 precop = MAX (prec0, prec1 + 1) + 1;
1708 int orig_precres = precres;
1712 if ((uns0_p && uns1_p)
1713 ? ((precop + !unsr_p) <= precres
1714 /* u1 - u2 -> ur can overflow, no matter what precision
1716 && (code != MINUS_EXPR || !unsr_p))
1717 : (!unsr_p && precop <= precres))
1719 /* The infinity precision result will always fit into result. */
1720 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1721 write_complex_part (target, const0_rtx, true);
1722 enum machine_mode mode = TYPE_MODE (type);
1723 struct separate_ops ops;
1726 ops.op0 = fold_convert_loc (loc, type, arg0);
1727 ops.op1 = fold_convert_loc (loc, type, arg1);
1728 ops.op2 = NULL_TREE;
1730 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1731 expand_arith_overflow_result_store (lhs, target, mode, tem);
1735 /* For sub-word operations, if target doesn't have them, start
1736 with precres widening right away, otherwise do it only
1737 if the most simple cases can't be used. */
1738 if (WORD_REGISTER_OPERATIONS
1739 && orig_precres == precres
1740 && precres < BITS_PER_WORD)
1742 else if ((uns0_p && uns1_p && unsr_p && prec0 <= precres
1743 && prec1 <= precres)
1744 || ((!uns0_p || !uns1_p) && !unsr_p
1745 && prec0 + uns0_p <= precres
1746 && prec1 + uns1_p <= precres))
1748 arg0 = fold_convert_loc (loc, type, arg0);
1749 arg1 = fold_convert_loc (loc, type, arg1);
1753 if (integer_zerop (arg0) && !unsr_p)
1754 expand_neg_overflow (loc, lhs, arg1, false);
1757 expand_addsub_overflow (loc, code, lhs, arg0, arg1,
1758 unsr_p, unsr_p, unsr_p, false);
1761 expand_mul_overflow (loc, lhs, arg0, arg1,
1762 unsr_p, unsr_p, unsr_p, false);
1769 /* For sub-word operations, retry with a wider type first. */
1770 if (orig_precres == precres && precop <= BITS_PER_WORD)
1772 #if WORD_REGISTER_OPERATIONS
1773 int p = BITS_PER_WORD;
1777 enum machine_mode m = smallest_mode_for_size (p, MODE_INT);
1778 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
1781 p = TYPE_PRECISION (optype);
1785 unsr_p = TYPE_UNSIGNED (optype);
1791 if (prec0 <= precres && prec1 <= precres)
1796 types[0] = build_nonstandard_integer_type (precres, 0);
1802 types[1] = build_nonstandard_integer_type (precres, 1);
1804 arg0 = fold_convert_loc (loc, types[uns0_p], arg0);
1805 arg1 = fold_convert_loc (loc, types[uns1_p], arg1);
1806 if (code != MULT_EXPR)
1807 expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
1808 uns0_p, uns1_p, false);
1810 expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
1811 uns0_p, uns1_p, false);
1815 /* Retry with a wider type. */
1816 if (orig_precres == precres)
1818 int p = MAX (prec0, prec1);
1819 enum machine_mode m = smallest_mode_for_size (p, MODE_INT);
1820 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
1823 p = TYPE_PRECISION (optype);
1827 unsr_p = TYPE_UNSIGNED (optype);
1838 /* Expand ADD_OVERFLOW STMT. */
1841 expand_ADD_OVERFLOW (gcall *stmt)
1843 expand_arith_overflow (PLUS_EXPR, stmt);
1846 /* Expand SUB_OVERFLOW STMT. */
1849 expand_SUB_OVERFLOW (gcall *stmt)
1851 expand_arith_overflow (MINUS_EXPR, stmt);
1854 /* Expand MUL_OVERFLOW STMT. */
1857 expand_MUL_OVERFLOW (gcall *stmt)
1859 expand_arith_overflow (MULT_EXPR, stmt);
1862 /* This should get folded in tree-vectorizer.c. */
1865 expand_LOOP_VECTORIZED (gcall *)
1871 expand_MASK_LOAD (gcall *stmt)
1873 struct expand_operand ops[3];
1874 tree type, lhs, rhs, maskt;
1875 rtx mem, target, mask;
1877 maskt = gimple_call_arg (stmt, 2);
1878 lhs = gimple_call_lhs (stmt);
1879 if (lhs == NULL_TREE)
1881 type = TREE_TYPE (lhs);
1882 rhs = fold_build2 (MEM_REF, type, gimple_call_arg (stmt, 0),
1883 gimple_call_arg (stmt, 1));
1885 mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1886 gcc_assert (MEM_P (mem));
1887 mask = expand_normal (maskt);
1888 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1889 create_output_operand (&ops[0], target, TYPE_MODE (type));
1890 create_fixed_operand (&ops[1], mem);
1891 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
1892 expand_insn (convert_optab_handler (maskload_optab, TYPE_MODE (type),
1893 TYPE_MODE (TREE_TYPE (maskt))),
1898 expand_MASK_STORE (gcall *stmt)
1900 struct expand_operand ops[3];
1901 tree type, lhs, rhs, maskt;
1904 maskt = gimple_call_arg (stmt, 2);
1905 rhs = gimple_call_arg (stmt, 3);
1906 type = TREE_TYPE (rhs);
1907 lhs = fold_build2 (MEM_REF, type, gimple_call_arg (stmt, 0),
1908 gimple_call_arg (stmt, 1));
1910 mem = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1911 gcc_assert (MEM_P (mem));
1912 mask = expand_normal (maskt);
1913 reg = expand_normal (rhs);
1914 create_fixed_operand (&ops[0], mem);
1915 create_input_operand (&ops[1], reg, TYPE_MODE (type));
1916 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
1917 expand_insn (convert_optab_handler (maskstore_optab, TYPE_MODE (type),
1918 TYPE_MODE (TREE_TYPE (maskt))),
1923 expand_ABNORMAL_DISPATCHER (gcall *)
1928 expand_BUILTIN_EXPECT (gcall *stmt)
1930 /* When guessing was done, the hints should be already stripped away. */
1931 gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ());
1934 tree lhs = gimple_call_lhs (stmt);
1936 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1938 target = const0_rtx;
1939 rtx val = expand_expr (gimple_call_arg (stmt, 0), target, VOIDmode, EXPAND_NORMAL);
1940 if (lhs && val != target)
1941 emit_move_insn (target, val);
1944 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
1945 should never be called. */
1948 expand_VA_ARG (gcall *stmt ATTRIBUTE_UNUSED)
1953 /* Expand the IFN_UNIQUE function according to its first argument. */
1956 expand_UNIQUE (gcall *stmt)
1958 rtx pattern = NULL_RTX;
1959 enum ifn_unique_kind kind
1960 = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (stmt, 0));
1967 case IFN_UNIQUE_UNSPEC:
1968 if (targetm.have_unique ())
1969 pattern = targetm.gen_unique ();
1972 case IFN_UNIQUE_OACC_FORK:
1973 case IFN_UNIQUE_OACC_JOIN:
1974 if (targetm.have_oacc_fork () && targetm.have_oacc_join ())
1976 tree lhs = gimple_call_lhs (stmt);
1977 rtx target = const0_rtx;
1980 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1982 rtx data_dep = expand_normal (gimple_call_arg (stmt, 1));
1983 rtx axis = expand_normal (gimple_call_arg (stmt, 2));
1985 if (kind == IFN_UNIQUE_OACC_FORK)
1986 pattern = targetm.gen_oacc_fork (target, data_dep, axis);
1988 pattern = targetm.gen_oacc_join (target, data_dep, axis);
1996 emit_insn (pattern);
1999 /* The size of an OpenACC compute dimension. */
2002 expand_GOACC_DIM_SIZE (gcall *stmt)
2004 tree lhs = gimple_call_lhs (stmt);
2009 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2010 if (targetm.have_oacc_dim_size ())
2012 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
2013 VOIDmode, EXPAND_NORMAL);
2014 emit_insn (targetm.gen_oacc_dim_size (target, dim));
2017 emit_move_insn (target, GEN_INT (1));
2020 /* The position of an OpenACC execution engine along one compute axis. */
2023 expand_GOACC_DIM_POS (gcall *stmt)
2025 tree lhs = gimple_call_lhs (stmt);
2030 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2031 if (targetm.have_oacc_dim_pos ())
2033 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
2034 VOIDmode, EXPAND_NORMAL);
2035 emit_insn (targetm.gen_oacc_dim_pos (target, dim));
2038 emit_move_insn (target, const0_rtx);
2041 /* This is expanded by oacc_device_lower pass. */
2044 expand_GOACC_LOOP (gcall *stmt ATTRIBUTE_UNUSED)
2049 /* This is expanded by oacc_device_lower pass. */
2052 expand_GOACC_REDUCTION (gcall *stmt ATTRIBUTE_UNUSED)
2057 /* Routines to expand each internal function, indexed by function number.
2058 Each routine has the prototype:
2060 expand_<NAME> (gcall *stmt)
2062 where STMT is the statement that performs the call. */
2063 static void (*const internal_fn_expanders[]) (gcall *) = {
2064 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
2065 #include "internal-fn.def"
2069 /* Expand STMT, which is a call to internal function FN. */
2072 expand_internal_call (gcall *stmt)
2074 internal_fn_expanders[(int) gimple_call_internal_fn (stmt)] (stmt);