1 /* Lower vector operations to scalar operations.
2 Copyright (C) 2004-2015 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
11 GCC is distributed in the hope that it will be useful, but WITHOUT
12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
30 #include "fold-const.h"
31 #include "stor-layout.h"
32 #include "langhooks.h"
33 #include "internal-fn.h"
35 #include "gimple-iterator.h"
36 #include "gimplify-me.h"
38 #include "tree-iterator.h"
39 #include "tree-pass.h"
41 #include "diagnostic.h"
44 #include "insn-codes.h"
45 #include "optabs-tree.h"
48 static void expand_vector_operations_1 (gimple_stmt_iterator *);
51 /* Build a constant of type TYPE, made of VALUE's bits replicated
52 every TYPE_SIZE (INNER_TYPE) bits to fit TYPE's precision. */
54 build_replicated_const (tree type, tree inner_type, HOST_WIDE_INT value)
56 int width = tree_to_uhwi (TYPE_SIZE (inner_type));
57 int n = (TYPE_PRECISION (type) + HOST_BITS_PER_WIDE_INT - 1)
58 / HOST_BITS_PER_WIDE_INT;
59 unsigned HOST_WIDE_INT low, mask;
60 HOST_WIDE_INT a[WIDE_INT_MAX_ELTS];
63 gcc_assert (n && n <= WIDE_INT_MAX_ELTS);
65 if (width == HOST_BITS_PER_WIDE_INT)
69 mask = ((HOST_WIDE_INT)1 << width) - 1;
70 low = (unsigned HOST_WIDE_INT) ~0 / mask * (value & mask);
73 for (i = 0; i < n; i++)
76 gcc_assert (TYPE_PRECISION (type) <= MAX_BITSIZE_MODE_ANY_INT);
77 return wide_int_to_tree
78 (type, wide_int::from_array (a, n, TYPE_PRECISION (type)));
81 static GTY(()) tree vector_inner_type;
82 static GTY(()) tree vector_last_type;
83 static GTY(()) int vector_last_nunits;
85 /* Return a suitable vector types made of SUBPARTS units each of mode
86 "word_mode" (the global variable). */
88 build_word_mode_vector_type (int nunits)
90 if (!vector_inner_type)
91 vector_inner_type = lang_hooks.types.type_for_mode (word_mode, 1);
92 else if (vector_last_nunits == nunits)
94 gcc_assert (TREE_CODE (vector_last_type) == VECTOR_TYPE);
95 return vector_last_type;
98 /* We build a new type, but we canonicalize it nevertheless,
99 because it still saves some memory. */
100 vector_last_nunits = nunits;
101 vector_last_type = type_hash_canon (nunits,
102 build_vector_type (vector_inner_type,
104 return vector_last_type;
107 typedef tree (*elem_op_func) (gimple_stmt_iterator *,
108 tree, tree, tree, tree, tree, enum tree_code,
112 tree_vec_extract (gimple_stmt_iterator *gsi, tree type,
113 tree t, tree bitsize, tree bitpos)
117 if (TREE_CODE (type) == BOOLEAN_TYPE)
120 = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 0);
121 tree field = gimplify_build3 (gsi, BIT_FIELD_REF, itype, t,
123 return gimplify_build2 (gsi, NE_EXPR, type, field,
124 build_zero_cst (itype));
127 return gimplify_build3 (gsi, BIT_FIELD_REF, type, t, bitsize, bitpos);
130 return gimplify_build1 (gsi, VIEW_CONVERT_EXPR, type, t);
134 do_unop (gimple_stmt_iterator *gsi, tree inner_type, tree a,
135 tree b ATTRIBUTE_UNUSED, tree bitpos, tree bitsize,
136 enum tree_code code, tree type ATTRIBUTE_UNUSED)
138 a = tree_vec_extract (gsi, inner_type, a, bitsize, bitpos);
139 return gimplify_build1 (gsi, code, inner_type, a);
143 do_binop (gimple_stmt_iterator *gsi, tree inner_type, tree a, tree b,
144 tree bitpos, tree bitsize, enum tree_code code,
145 tree type ATTRIBUTE_UNUSED)
147 if (TREE_CODE (TREE_TYPE (a)) == VECTOR_TYPE)
148 a = tree_vec_extract (gsi, inner_type, a, bitsize, bitpos);
149 if (TREE_CODE (TREE_TYPE (b)) == VECTOR_TYPE)
150 b = tree_vec_extract (gsi, inner_type, b, bitsize, bitpos);
151 return gimplify_build2 (gsi, code, inner_type, a, b);
154 /* Construct expression (A[BITPOS] code B[BITPOS]) ? -1 : 0
156 INNER_TYPE is the type of A and B elements
158 returned expression is of signed integer type with the
159 size equal to the size of INNER_TYPE. */
161 do_compare (gimple_stmt_iterator *gsi, tree inner_type, tree a, tree b,
162 tree bitpos, tree bitsize, enum tree_code code, tree type)
164 a = tree_vec_extract (gsi, inner_type, a, bitsize, bitpos);
165 b = tree_vec_extract (gsi, inner_type, b, bitsize, bitpos);
167 return gimplify_build2 (gsi, code, TREE_TYPE (type), a, b);
170 /* Expand vector addition to scalars. This does bit twiddling
171 in order to increase parallelism:
173 a + b = (((int) a & 0x7f7f7f7f) + ((int) b & 0x7f7f7f7f)) ^
176 a - b = (((int) a | 0x80808080) - ((int) b & 0x7f7f7f7f)) ^
177 (a ^ ~b) & 0x80808080
179 -b = (0x80808080 - ((int) b & 0x7f7f7f7f)) ^ (~b & 0x80808080)
181 This optimization should be done only if 4 vector items or more
184 do_plus_minus (gimple_stmt_iterator *gsi, tree word_type, tree a, tree b,
185 tree bitpos ATTRIBUTE_UNUSED, tree bitsize ATTRIBUTE_UNUSED,
186 enum tree_code code, tree type ATTRIBUTE_UNUSED)
188 tree inner_type = TREE_TYPE (TREE_TYPE (a));
189 unsigned HOST_WIDE_INT max;
190 tree low_bits, high_bits, a_low, b_low, result_low, signs;
192 max = GET_MODE_MASK (TYPE_MODE (inner_type));
193 low_bits = build_replicated_const (word_type, inner_type, max >> 1);
194 high_bits = build_replicated_const (word_type, inner_type, max & ~(max >> 1));
196 a = tree_vec_extract (gsi, word_type, a, bitsize, bitpos);
197 b = tree_vec_extract (gsi, word_type, b, bitsize, bitpos);
199 signs = gimplify_build2 (gsi, BIT_XOR_EXPR, word_type, a, b);
200 b_low = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, b, low_bits);
201 if (code == PLUS_EXPR)
202 a_low = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, a, low_bits);
205 a_low = gimplify_build2 (gsi, BIT_IOR_EXPR, word_type, a, high_bits);
206 signs = gimplify_build1 (gsi, BIT_NOT_EXPR, word_type, signs);
209 signs = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, signs, high_bits);
210 result_low = gimplify_build2 (gsi, code, word_type, a_low, b_low);
211 return gimplify_build2 (gsi, BIT_XOR_EXPR, word_type, result_low, signs);
215 do_negate (gimple_stmt_iterator *gsi, tree word_type, tree b,
216 tree unused ATTRIBUTE_UNUSED, tree bitpos ATTRIBUTE_UNUSED,
217 tree bitsize ATTRIBUTE_UNUSED,
218 enum tree_code code ATTRIBUTE_UNUSED,
219 tree type ATTRIBUTE_UNUSED)
221 tree inner_type = TREE_TYPE (TREE_TYPE (b));
223 tree low_bits, high_bits, b_low, result_low, signs;
225 max = GET_MODE_MASK (TYPE_MODE (inner_type));
226 low_bits = build_replicated_const (word_type, inner_type, max >> 1);
227 high_bits = build_replicated_const (word_type, inner_type, max & ~(max >> 1));
229 b = tree_vec_extract (gsi, word_type, b, bitsize, bitpos);
231 b_low = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, b, low_bits);
232 signs = gimplify_build1 (gsi, BIT_NOT_EXPR, word_type, b);
233 signs = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, signs, high_bits);
234 result_low = gimplify_build2 (gsi, MINUS_EXPR, word_type, high_bits, b_low);
235 return gimplify_build2 (gsi, BIT_XOR_EXPR, word_type, result_low, signs);
238 /* Expand a vector operation to scalars, by using many operations
239 whose type is the vector type's inner type. */
241 expand_vector_piecewise (gimple_stmt_iterator *gsi, elem_op_func f,
242 tree type, tree inner_type,
243 tree a, tree b, enum tree_code code)
245 vec<constructor_elt, va_gc> *v;
246 tree part_width = TYPE_SIZE (inner_type);
247 tree index = bitsize_int (0);
248 int nunits = TYPE_VECTOR_SUBPARTS (type);
249 int delta = tree_to_uhwi (part_width)
250 / tree_to_uhwi (TYPE_SIZE (TREE_TYPE (type)));
252 location_t loc = gimple_location (gsi_stmt (*gsi));
254 if (types_compatible_p (gimple_expr_type (gsi_stmt (*gsi)), type))
255 warning_at (loc, OPT_Wvector_operation_performance,
256 "vector operation will be expanded piecewise");
258 warning_at (loc, OPT_Wvector_operation_performance,
259 "vector operation will be expanded in parallel");
261 vec_alloc (v, (nunits + delta - 1) / delta);
262 for (i = 0; i < nunits;
263 i += delta, index = int_const_binop (PLUS_EXPR, index, part_width))
265 tree result = f (gsi, inner_type, a, b, index, part_width, code, type);
266 constructor_elt ce = {NULL_TREE, result};
270 return build_constructor (type, v);
273 /* Expand a vector operation to scalars with the freedom to use
274 a scalar integer type, or to use a different size for the items
275 in the vector type. */
277 expand_vector_parallel (gimple_stmt_iterator *gsi, elem_op_func f, tree type,
281 tree result, compute_type;
283 int n_words = tree_to_uhwi (TYPE_SIZE_UNIT (type)) / UNITS_PER_WORD;
284 location_t loc = gimple_location (gsi_stmt (*gsi));
286 /* We have three strategies. If the type is already correct, just do
287 the operation an element at a time. Else, if the vector is wider than
288 one word, do it a word at a time; finally, if the vector is smaller
289 than one word, do it as a scalar. */
290 if (TYPE_MODE (TREE_TYPE (type)) == word_mode)
291 return expand_vector_piecewise (gsi, f,
292 type, TREE_TYPE (type),
294 else if (n_words > 1)
296 tree word_type = build_word_mode_vector_type (n_words);
297 result = expand_vector_piecewise (gsi, f,
298 word_type, TREE_TYPE (word_type),
300 result = force_gimple_operand_gsi (gsi, result, true, NULL, true,
305 /* Use a single scalar operation with a mode no wider than word_mode. */
306 mode = mode_for_size (tree_to_uhwi (TYPE_SIZE (type)), MODE_INT, 0);
307 compute_type = lang_hooks.types.type_for_mode (mode, 1);
308 result = f (gsi, compute_type, a, b, NULL_TREE, NULL_TREE, code, type);
309 warning_at (loc, OPT_Wvector_operation_performance,
310 "vector operation will be expanded with a "
311 "single scalar operation");
317 /* Expand a vector operation to scalars; for integer types we can use
318 special bit twiddling tricks to do the sums a word at a time, using
319 function F_PARALLEL instead of F. These tricks are done only if
320 they can process at least four items, that is, only if the vector
321 holds at least four items and if a word can hold four items. */
323 expand_vector_addition (gimple_stmt_iterator *gsi,
324 elem_op_func f, elem_op_func f_parallel,
325 tree type, tree a, tree b, enum tree_code code)
327 int parts_per_word = UNITS_PER_WORD
328 / tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type)));
330 if (INTEGRAL_TYPE_P (TREE_TYPE (type))
331 && parts_per_word >= 4
332 && TYPE_VECTOR_SUBPARTS (type) >= 4)
333 return expand_vector_parallel (gsi, f_parallel,
336 return expand_vector_piecewise (gsi, f,
337 type, TREE_TYPE (type),
341 /* Try to expand vector comparison expression OP0 CODE OP1 by
342 querying optab if the following expression:
343 VEC_COND_EXPR< OP0 CODE OP1, {-1,...}, {0,...}>
346 expand_vector_comparison (gimple_stmt_iterator *gsi, tree type, tree op0,
347 tree op1, enum tree_code code)
350 if (! expand_vec_cond_expr_p (type, TREE_TYPE (op0)))
351 t = expand_vector_piecewise (gsi, do_compare, type,
352 TREE_TYPE (TREE_TYPE (op0)), op0, op1, code);
359 /* Helper function of expand_vector_divmod. Gimplify a RSHIFT_EXPR in type
360 of OP0 with shift counts in SHIFTCNTS array and return the temporary holding
361 the result if successful, otherwise return NULL_TREE. */
363 add_rshift (gimple_stmt_iterator *gsi, tree type, tree op0, int *shiftcnts)
366 unsigned int i, nunits = TYPE_VECTOR_SUBPARTS (type);
367 bool scalar_shift = true;
369 for (i = 1; i < nunits; i++)
371 if (shiftcnts[i] != shiftcnts[0])
372 scalar_shift = false;
375 if (scalar_shift && shiftcnts[0] == 0)
380 op = optab_for_tree_code (RSHIFT_EXPR, type, optab_scalar);
381 if (op != unknown_optab
382 && optab_handler (op, TYPE_MODE (type)) != CODE_FOR_nothing)
383 return gimplify_build2 (gsi, RSHIFT_EXPR, type, op0,
384 build_int_cst (NULL_TREE, shiftcnts[0]));
387 op = optab_for_tree_code (RSHIFT_EXPR, type, optab_vector);
388 if (op != unknown_optab
389 && optab_handler (op, TYPE_MODE (type)) != CODE_FOR_nothing)
391 tree *vec = XALLOCAVEC (tree, nunits);
392 for (i = 0; i < nunits; i++)
393 vec[i] = build_int_cst (TREE_TYPE (type), shiftcnts[i]);
394 return gimplify_build2 (gsi, RSHIFT_EXPR, type, op0,
395 build_vector (type, vec));
401 /* Try to expand integer vector division by constant using
402 widening multiply, shifts and additions. */
404 expand_vector_divmod (gimple_stmt_iterator *gsi, tree type, tree op0,
405 tree op1, enum tree_code code)
407 bool use_pow2 = true;
408 bool has_vector_shift = true;
409 int mode = -1, this_mode;
410 int pre_shift = -1, post_shift;
411 unsigned int nunits = TYPE_VECTOR_SUBPARTS (type);
412 int *shifts = XALLOCAVEC (int, nunits * 4);
413 int *pre_shifts = shifts + nunits;
414 int *post_shifts = pre_shifts + nunits;
415 int *shift_temps = post_shifts + nunits;
416 unsigned HOST_WIDE_INT *mulc = XALLOCAVEC (unsigned HOST_WIDE_INT, nunits);
417 int prec = TYPE_PRECISION (TREE_TYPE (type));
420 signop sign_p = TYPE_SIGN (TREE_TYPE (type));
421 unsigned HOST_WIDE_INT mask = GET_MODE_MASK (TYPE_MODE (TREE_TYPE (type)));
423 tree cur_op, mulcst, tem;
426 if (prec > HOST_BITS_PER_WIDE_INT)
429 op = optab_for_tree_code (RSHIFT_EXPR, type, optab_vector);
430 if (op == unknown_optab
431 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing)
432 has_vector_shift = false;
434 /* Analysis phase. Determine if all op1 elements are either power
435 of two and it is possible to expand it using shifts (or for remainder
436 using masking). Additionally compute the multiplicative constants
437 and pre and post shifts if the division is to be expanded using
438 widening or high part multiplication plus shifts. */
439 for (i = 0; i < nunits; i++)
441 tree cst = VECTOR_CST_ELT (op1, i);
442 unsigned HOST_WIDE_INT ml;
444 if (TREE_CODE (cst) != INTEGER_CST || integer_zerop (cst))
450 && (!integer_pow2p (cst) || tree_int_cst_sgn (cst) != 1))
454 shifts[i] = tree_log2 (cst);
455 if (shifts[i] != shifts[0]
456 && code == TRUNC_DIV_EXPR
457 && !has_vector_shift)
462 if (sign_p == UNSIGNED)
464 unsigned HOST_WIDE_INT mh;
465 unsigned HOST_WIDE_INT d = TREE_INT_CST_LOW (cst) & mask;
467 if (d >= ((unsigned HOST_WIDE_INT) 1 << (prec - 1)))
468 /* FIXME: Can transform this into op0 >= op1 ? 1 : 0. */
477 /* Find a suitable multiplier and right shift count
478 instead of multiplying with D. */
479 mh = choose_multiplier (d, prec, prec, &ml, &post_shift, &dummy_int);
481 /* If the suggested multiplier is more than SIZE bits, we can
482 do better for even divisors, using an initial right shift. */
483 if ((mh != 0 && (d & 1) == 0)
484 || (!has_vector_shift && pre_shift != -1))
486 if (has_vector_shift)
487 pre_shift = floor_log2 (d & -d);
488 else if (pre_shift == -1)
491 for (j = 0; j < nunits; j++)
493 tree cst2 = VECTOR_CST_ELT (op1, j);
494 unsigned HOST_WIDE_INT d2;
497 if (!tree_fits_uhwi_p (cst2))
499 d2 = tree_to_uhwi (cst2) & mask;
502 this_pre_shift = floor_log2 (d2 & -d2);
503 if (pre_shift == -1 || this_pre_shift < pre_shift)
504 pre_shift = this_pre_shift;
506 if (i != 0 && pre_shift != 0)
516 if ((d >> pre_shift) <= 1)
521 mh = choose_multiplier (d >> pre_shift, prec,
523 &ml, &post_shift, &dummy_int);
525 pre_shifts[i] = pre_shift;
535 HOST_WIDE_INT d = TREE_INT_CST_LOW (cst);
536 unsigned HOST_WIDE_INT abs_d;
541 /* Since d might be INT_MIN, we have to cast to
542 unsigned HOST_WIDE_INT before negating to avoid
543 undefined signed overflow. */
545 ? (unsigned HOST_WIDE_INT) d
546 : - (unsigned HOST_WIDE_INT) d);
548 /* n rem d = n rem -d */
549 if (code == TRUNC_MOD_EXPR && d < 0)
551 else if (abs_d == (unsigned HOST_WIDE_INT) 1 << (prec - 1))
553 /* This case is not handled correctly below. */
563 choose_multiplier (abs_d, prec, prec - 1, &ml,
564 &post_shift, &dummy_int);
565 if (ml >= (unsigned HOST_WIDE_INT) 1 << (prec - 1))
567 this_mode = 4 + (d < 0);
568 ml |= (~(unsigned HOST_WIDE_INT) 0) << (prec - 1);
571 this_mode = 2 + (d < 0);
574 post_shifts[i] = post_shift;
575 if ((i && !has_vector_shift && post_shifts[0] != post_shift)
576 || post_shift >= prec
577 || pre_shifts[i] >= prec)
582 else if (mode != this_mode)
586 vec = XALLOCAVEC (tree, nunits);
590 tree addend = NULL_TREE;
591 if (sign_p == SIGNED)
595 /* Both division and remainder sequences need
596 op0 < 0 ? mask : 0 computed. It can be either computed as
597 (type) (((uns_type) (op0 >> (prec - 1))) >> (prec - shifts[i]))
598 if none of the shifts is 0, or as the conditional. */
599 for (i = 0; i < nunits; i++)
603 = build_vector_type (build_nonstandard_integer_type (prec, 1),
605 if (i == nunits && TYPE_MODE (uns_type) == TYPE_MODE (type))
607 for (i = 0; i < nunits; i++)
608 shift_temps[i] = prec - 1;
609 cur_op = add_rshift (gsi, type, op0, shift_temps);
610 if (cur_op != NULL_TREE)
612 cur_op = gimplify_build1 (gsi, VIEW_CONVERT_EXPR,
614 for (i = 0; i < nunits; i++)
615 shift_temps[i] = prec - shifts[i];
616 cur_op = add_rshift (gsi, uns_type, cur_op, shift_temps);
617 if (cur_op != NULL_TREE)
618 addend = gimplify_build1 (gsi, VIEW_CONVERT_EXPR,
622 if (addend == NULL_TREE
623 && expand_vec_cond_expr_p (type, type))
625 tree zero, cst, cond, mask_type;
628 mask_type = build_same_sized_truth_vector_type (type);
629 zero = build_zero_cst (type);
630 cond = build2 (LT_EXPR, mask_type, op0, zero);
631 for (i = 0; i < nunits; i++)
632 vec[i] = build_int_cst (TREE_TYPE (type),
633 ((unsigned HOST_WIDE_INT) 1
635 cst = build_vector (type, vec);
636 addend = make_ssa_name (type);
637 stmt = gimple_build_assign (addend, VEC_COND_EXPR, cond,
639 gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
642 if (code == TRUNC_DIV_EXPR)
644 if (sign_p == UNSIGNED)
646 /* q = op0 >> shift; */
647 cur_op = add_rshift (gsi, type, op0, shifts);
648 if (cur_op != NULL_TREE)
651 else if (addend != NULL_TREE)
653 /* t1 = op0 + addend;
655 op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
656 if (op != unknown_optab
657 && optab_handler (op, TYPE_MODE (type)) != CODE_FOR_nothing)
659 cur_op = gimplify_build2 (gsi, PLUS_EXPR, type, op0, addend);
660 cur_op = add_rshift (gsi, type, cur_op, shifts);
661 if (cur_op != NULL_TREE)
669 for (i = 0; i < nunits; i++)
670 vec[i] = build_int_cst (TREE_TYPE (type),
671 ((unsigned HOST_WIDE_INT) 1
673 mask = build_vector (type, vec);
674 op = optab_for_tree_code (BIT_AND_EXPR, type, optab_default);
675 if (op != unknown_optab
676 && optab_handler (op, TYPE_MODE (type)) != CODE_FOR_nothing)
678 if (sign_p == UNSIGNED)
679 /* r = op0 & mask; */
680 return gimplify_build2 (gsi, BIT_AND_EXPR, type, op0, mask);
681 else if (addend != NULL_TREE)
683 /* t1 = op0 + addend;
686 op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
687 if (op != unknown_optab
688 && optab_handler (op, TYPE_MODE (type))
691 cur_op = gimplify_build2 (gsi, PLUS_EXPR, type, op0,
693 cur_op = gimplify_build2 (gsi, BIT_AND_EXPR, type,
695 op = optab_for_tree_code (MINUS_EXPR, type,
697 if (op != unknown_optab
698 && optab_handler (op, TYPE_MODE (type))
700 return gimplify_build2 (gsi, MINUS_EXPR, type,
708 if (mode == -2 || BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
711 if (!can_mult_highpart_p (TYPE_MODE (type), TYPE_UNSIGNED (type)))
719 gcc_assert (sign_p == UNSIGNED);
720 /* t1 = oprnd0 >> pre_shift;
722 q = t2 >> post_shift; */
723 cur_op = add_rshift (gsi, type, cur_op, pre_shifts);
724 if (cur_op == NULL_TREE)
728 gcc_assert (sign_p == UNSIGNED);
729 for (i = 0; i < nunits; i++)
739 gcc_assert (sign_p == SIGNED);
740 for (i = 0; i < nunits; i++)
741 shift_temps[i] = prec - 1;
747 for (i = 0; i < nunits; i++)
748 vec[i] = build_int_cst (TREE_TYPE (type), mulc[i]);
749 mulcst = build_vector (type, vec);
751 cur_op = gimplify_build2 (gsi, MULT_HIGHPART_EXPR, type, cur_op, mulcst);
756 /* t1 = oprnd0 >> pre_shift;
758 q = t2 >> post_shift; */
759 cur_op = add_rshift (gsi, type, cur_op, post_shifts);
762 /* t1 = oprnd0 h* ml;
766 q = t4 >> (post_shift - 1); */
767 op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
768 if (op == unknown_optab
769 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing)
771 tem = gimplify_build2 (gsi, MINUS_EXPR, type, op0, cur_op);
772 tem = add_rshift (gsi, type, tem, shift_temps);
773 op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
774 if (op == unknown_optab
775 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing)
777 tem = gimplify_build2 (gsi, PLUS_EXPR, type, cur_op, tem);
778 cur_op = add_rshift (gsi, type, tem, post_shifts);
779 if (cur_op == NULL_TREE)
786 /* t1 = oprnd0 h* ml;
787 t2 = t1; [ iff (mode & 2) != 0 ]
788 t2 = t1 + oprnd0; [ iff (mode & 2) == 0 ]
789 t3 = t2 >> post_shift;
790 t4 = oprnd0 >> (prec - 1);
791 q = t3 - t4; [ iff (mode & 1) == 0 ]
792 q = t4 - t3; [ iff (mode & 1) != 0 ] */
795 op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
796 if (op == unknown_optab
797 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing)
799 cur_op = gimplify_build2 (gsi, PLUS_EXPR, type, cur_op, op0);
801 cur_op = add_rshift (gsi, type, cur_op, post_shifts);
802 if (cur_op == NULL_TREE)
804 tem = add_rshift (gsi, type, op0, shift_temps);
805 if (tem == NULL_TREE)
807 op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
808 if (op == unknown_optab
809 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing)
812 cur_op = gimplify_build2 (gsi, MINUS_EXPR, type, cur_op, tem);
814 cur_op = gimplify_build2 (gsi, MINUS_EXPR, type, tem, cur_op);
820 if (code == TRUNC_DIV_EXPR)
823 /* We divided. Now finish by:
826 op = optab_for_tree_code (MULT_EXPR, type, optab_default);
827 if (op == unknown_optab
828 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing)
830 tem = gimplify_build2 (gsi, MULT_EXPR, type, cur_op, op1);
831 op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
832 if (op == unknown_optab
833 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing)
835 return gimplify_build2 (gsi, MINUS_EXPR, type, op0, tem);
838 /* Expand a vector condition to scalars, by using many conditions
839 on the vector's elements. */
841 expand_vector_condition (gimple_stmt_iterator *gsi)
843 gassign *stmt = as_a <gassign *> (gsi_stmt (*gsi));
844 tree type = gimple_expr_type (stmt);
845 tree a = gimple_assign_rhs1 (stmt);
848 bool a_is_comparison = false;
849 tree b = gimple_assign_rhs2 (stmt);
850 tree c = gimple_assign_rhs3 (stmt);
851 vec<constructor_elt, va_gc> *v;
853 tree inner_type = TREE_TYPE (type);
854 tree cond_type = TREE_TYPE (TREE_TYPE (a));
855 tree comp_inner_type = cond_type;
856 tree width = TYPE_SIZE (inner_type);
857 tree index = bitsize_int (0);
858 int nunits = TYPE_VECTOR_SUBPARTS (type);
860 location_t loc = gimple_location (gsi_stmt (*gsi));
862 if (!is_gimple_val (a))
864 gcc_assert (COMPARISON_CLASS_P (a));
865 a_is_comparison = true;
866 a1 = TREE_OPERAND (a, 0);
867 a2 = TREE_OPERAND (a, 1);
868 comp_inner_type = TREE_TYPE (TREE_TYPE (a1));
871 if (expand_vec_cond_expr_p (type, TREE_TYPE (a1)))
874 /* TODO: try and find a smaller vector type. */
876 warning_at (loc, OPT_Wvector_operation_performance,
877 "vector condition will be expanded piecewise");
879 vec_alloc (v, nunits);
880 for (i = 0; i < nunits;
881 i++, index = int_const_binop (PLUS_EXPR, index, width))
884 tree bb = tree_vec_extract (gsi, inner_type, b, width, index);
885 tree cc = tree_vec_extract (gsi, inner_type, c, width, index);
888 tree aa1 = tree_vec_extract (gsi, comp_inner_type, a1, width, index);
889 tree aa2 = tree_vec_extract (gsi, comp_inner_type, a2, width, index);
890 aa = build2 (TREE_CODE (a), cond_type, aa1, aa2);
893 aa = tree_vec_extract (gsi, cond_type, a, width, index);
894 result = gimplify_build3 (gsi, COND_EXPR, inner_type, aa, bb, cc);
895 constructor_elt ce = {NULL_TREE, result};
899 constr = build_constructor (type, v);
900 gimple_assign_set_rhs_from_tree (gsi, constr);
901 update_stmt (gsi_stmt (*gsi));
905 expand_vector_operation (gimple_stmt_iterator *gsi, tree type, tree compute_type,
906 gassign *assign, enum tree_code code)
908 machine_mode compute_mode = TYPE_MODE (compute_type);
910 /* If the compute mode is not a vector mode (hence we are not decomposing
911 a BLKmode vector to smaller, hardware-supported vectors), we may want
912 to expand the operations in parallel. */
913 if (GET_MODE_CLASS (compute_mode) != MODE_VECTOR_INT
914 && GET_MODE_CLASS (compute_mode) != MODE_VECTOR_FLOAT
915 && GET_MODE_CLASS (compute_mode) != MODE_VECTOR_FRACT
916 && GET_MODE_CLASS (compute_mode) != MODE_VECTOR_UFRACT
917 && GET_MODE_CLASS (compute_mode) != MODE_VECTOR_ACCUM
918 && GET_MODE_CLASS (compute_mode) != MODE_VECTOR_UACCUM)
923 if (ANY_INTEGRAL_TYPE_P (type) && !TYPE_OVERFLOW_TRAPS (type))
924 return expand_vector_addition (gsi, do_binop, do_plus_minus, type,
925 gimple_assign_rhs1 (assign),
926 gimple_assign_rhs2 (assign), code);
930 if (ANY_INTEGRAL_TYPE_P (type) && !TYPE_OVERFLOW_TRAPS (type))
931 return expand_vector_addition (gsi, do_unop, do_negate, type,
932 gimple_assign_rhs1 (assign),
939 return expand_vector_parallel (gsi, do_binop, type,
940 gimple_assign_rhs1 (assign),
941 gimple_assign_rhs2 (assign), code);
944 return expand_vector_parallel (gsi, do_unop, type,
945 gimple_assign_rhs1 (assign),
962 tree rhs1 = gimple_assign_rhs1 (assign);
963 tree rhs2 = gimple_assign_rhs2 (assign);
965 return expand_vector_comparison (gsi, type, rhs1, rhs2, code);
971 tree rhs1 = gimple_assign_rhs1 (assign);
972 tree rhs2 = gimple_assign_rhs2 (assign);
976 || !VECTOR_INTEGER_TYPE_P (type)
977 || TREE_CODE (rhs2) != VECTOR_CST
978 || !VECTOR_MODE_P (TYPE_MODE (type)))
981 ret = expand_vector_divmod (gsi, type, rhs1, rhs2, code);
982 if (ret != NULL_TREE)
991 if (TREE_CODE_CLASS (code) == tcc_unary)
992 return expand_vector_piecewise (gsi, do_unop, type, compute_type,
993 gimple_assign_rhs1 (assign),
996 return expand_vector_piecewise (gsi, do_binop, type, compute_type,
997 gimple_assign_rhs1 (assign),
998 gimple_assign_rhs2 (assign), code);
1002 a_5 = { b_7, b_7 + 3, b_7 + 6, b_7 + 9 };
1004 _9 = { b_7, b_7, b_7, b_7 };
1005 a_5 = _9 + { 0, 3, 6, 9 };
1006 because vector splat operation is usually more efficient
1007 than piecewise initialization of the vector. */
1010 optimize_vector_constructor (gimple_stmt_iterator *gsi)
1012 gassign *stmt = as_a <gassign *> (gsi_stmt (*gsi));
1013 tree lhs = gimple_assign_lhs (stmt);
1014 tree rhs = gimple_assign_rhs1 (stmt);
1015 tree type = TREE_TYPE (rhs);
1016 unsigned int i, j, nelts = TYPE_VECTOR_SUBPARTS (type);
1017 bool all_same = true;
1018 constructor_elt *elt;
1021 tree base = NULL_TREE;
1024 if (nelts <= 2 || CONSTRUCTOR_NELTS (rhs) != nelts)
1026 op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
1027 if (op == unknown_optab
1028 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing)
1030 FOR_EACH_VEC_SAFE_ELT (CONSTRUCTOR_ELTS (rhs), i, elt)
1031 if (TREE_CODE (elt->value) != SSA_NAME
1032 || TREE_CODE (TREE_TYPE (elt->value)) == VECTOR_TYPE)
1036 tree this_base = elt->value;
1037 if (this_base != CONSTRUCTOR_ELT (rhs, 0)->value)
1039 for (j = 0; j < nelts + 1; j++)
1041 g = SSA_NAME_DEF_STMT (this_base);
1042 if (is_gimple_assign (g)
1043 && gimple_assign_rhs_code (g) == PLUS_EXPR
1044 && TREE_CODE (gimple_assign_rhs2 (g)) == INTEGER_CST
1045 && TREE_CODE (gimple_assign_rhs1 (g)) == SSA_NAME
1046 && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (gimple_assign_rhs1 (g)))
1047 this_base = gimple_assign_rhs1 (g);
1053 else if (this_base != base)
1058 cst = XALLOCAVEC (tree, nelts);
1059 for (i = 0; i < nelts; i++)
1061 tree this_base = CONSTRUCTOR_ELT (rhs, i)->value;;
1062 cst[i] = build_zero_cst (TREE_TYPE (base));
1063 while (this_base != base)
1065 g = SSA_NAME_DEF_STMT (this_base);
1066 cst[i] = fold_binary (PLUS_EXPR, TREE_TYPE (base),
1067 cst[i], gimple_assign_rhs2 (g));
1068 if (cst[i] == NULL_TREE
1069 || TREE_CODE (cst[i]) != INTEGER_CST
1070 || TREE_OVERFLOW (cst[i]))
1072 this_base = gimple_assign_rhs1 (g);
1075 for (i = 0; i < nelts; i++)
1076 CONSTRUCTOR_ELT (rhs, i)->value = base;
1077 g = gimple_build_assign (make_ssa_name (type), rhs);
1078 gsi_insert_before (gsi, g, GSI_SAME_STMT);
1079 g = gimple_build_assign (lhs, PLUS_EXPR, gimple_assign_lhs (g),
1080 build_vector (type, cst));
1081 gsi_replace (gsi, g, false);
1084 /* Return a type for the widest vector mode whose components are of type
1085 TYPE, or NULL_TREE if none is found. */
1088 type_for_widest_vector_mode (tree type, optab op)
1090 machine_mode inner_mode = TYPE_MODE (type);
1091 machine_mode best_mode = VOIDmode, mode;
1092 int best_nunits = 0;
1094 if (SCALAR_FLOAT_MODE_P (inner_mode))
1095 mode = MIN_MODE_VECTOR_FLOAT;
1096 else if (SCALAR_FRACT_MODE_P (inner_mode))
1097 mode = MIN_MODE_VECTOR_FRACT;
1098 else if (SCALAR_UFRACT_MODE_P (inner_mode))
1099 mode = MIN_MODE_VECTOR_UFRACT;
1100 else if (SCALAR_ACCUM_MODE_P (inner_mode))
1101 mode = MIN_MODE_VECTOR_ACCUM;
1102 else if (SCALAR_UACCUM_MODE_P (inner_mode))
1103 mode = MIN_MODE_VECTOR_UACCUM;
1105 mode = MIN_MODE_VECTOR_INT;
1107 for (; mode != VOIDmode; mode = GET_MODE_WIDER_MODE (mode))
1108 if (GET_MODE_INNER (mode) == inner_mode
1109 && GET_MODE_NUNITS (mode) > best_nunits
1110 && optab_handler (op, mode) != CODE_FOR_nothing)
1111 best_mode = mode, best_nunits = GET_MODE_NUNITS (mode);
1113 if (best_mode == VOIDmode)
1116 return build_vector_type_for_mode (type, best_mode);
1120 /* Build a reference to the element of the vector VECT. Function
1121 returns either the element itself, either BIT_FIELD_REF, or an
1122 ARRAY_REF expression.
1124 GSI is required to insert temporary variables while building a
1125 refernece to the element of the vector VECT.
1127 PTMPVEC is a pointer to the temporary variable for caching
1128 purposes. In case when PTMPVEC is NULL new temporary variable
1131 vector_element (gimple_stmt_iterator *gsi, tree vect, tree idx, tree *ptmpvec)
1133 tree vect_type, vect_elt_type;
1137 bool need_asgn = true;
1138 unsigned int elements;
1140 vect_type = TREE_TYPE (vect);
1141 vect_elt_type = TREE_TYPE (vect_type);
1142 elements = TYPE_VECTOR_SUBPARTS (vect_type);
1144 if (TREE_CODE (idx) == INTEGER_CST)
1146 unsigned HOST_WIDE_INT index;
1148 /* Given that we're about to compute a binary modulus,
1149 we don't care about the high bits of the value. */
1150 index = TREE_INT_CST_LOW (idx);
1151 if (!tree_fits_uhwi_p (idx) || index >= elements)
1153 index &= elements - 1;
1154 idx = build_int_cst (TREE_TYPE (idx), index);
1157 /* When lowering a vector statement sequence do some easy
1158 simplification by looking through intermediate vector results. */
1159 if (TREE_CODE (vect) == SSA_NAME)
1161 gimple *def_stmt = SSA_NAME_DEF_STMT (vect);
1162 if (is_gimple_assign (def_stmt)
1163 && (gimple_assign_rhs_code (def_stmt) == VECTOR_CST
1164 || gimple_assign_rhs_code (def_stmt) == CONSTRUCTOR))
1165 vect = gimple_assign_rhs1 (def_stmt);
1168 if (TREE_CODE (vect) == VECTOR_CST)
1169 return VECTOR_CST_ELT (vect, index);
1170 else if (TREE_CODE (vect) == CONSTRUCTOR
1171 && (CONSTRUCTOR_NELTS (vect) == 0
1172 || TREE_CODE (TREE_TYPE (CONSTRUCTOR_ELT (vect, 0)->value))
1175 if (index < CONSTRUCTOR_NELTS (vect))
1176 return CONSTRUCTOR_ELT (vect, index)->value;
1177 return build_zero_cst (vect_elt_type);
1181 tree size = TYPE_SIZE (vect_elt_type);
1182 tree pos = fold_build2 (MULT_EXPR, bitsizetype, bitsize_int (index),
1184 return fold_build3 (BIT_FIELD_REF, vect_elt_type, vect, size, pos);
1189 tmpvec = create_tmp_var (vect_type, "vectmp");
1191 tmpvec = *ptmpvec = create_tmp_var (vect_type, "vectmp");
1200 TREE_ADDRESSABLE (tmpvec) = 1;
1201 asgn = gimple_build_assign (tmpvec, vect);
1202 gsi_insert_before (gsi, asgn, GSI_SAME_STMT);
1205 arraytype = build_array_type_nelts (vect_elt_type, elements);
1206 return build4 (ARRAY_REF, vect_elt_type,
1207 build1 (VIEW_CONVERT_EXPR, arraytype, tmpvec),
1208 idx, NULL_TREE, NULL_TREE);
1211 /* Check if VEC_PERM_EXPR within the given setting is supported
1212 by hardware, or lower it piecewise.
1214 When VEC_PERM_EXPR has the same first and second operands:
1215 VEC_PERM_EXPR <v0, v0, mask> the lowered version would be
1216 {v0[mask[0]], v0[mask[1]], ...}
1217 MASK and V0 must have the same number of elements.
1219 Otherwise VEC_PERM_EXPR <v0, v1, mask> is lowered to
1220 {mask[0] < len(v0) ? v0[mask[0]] : v1[mask[0]], ...}
1221 V0 and V1 must have the same type. MASK, V0, V1 must have the
1222 same number of arguments. */
1225 lower_vec_perm (gimple_stmt_iterator *gsi)
1227 gassign *stmt = as_a <gassign *> (gsi_stmt (*gsi));
1228 tree mask = gimple_assign_rhs3 (stmt);
1229 tree vec0 = gimple_assign_rhs1 (stmt);
1230 tree vec1 = gimple_assign_rhs2 (stmt);
1231 tree vect_type = TREE_TYPE (vec0);
1232 tree mask_type = TREE_TYPE (mask);
1233 tree vect_elt_type = TREE_TYPE (vect_type);
1234 tree mask_elt_type = TREE_TYPE (mask_type);
1235 unsigned int elements = TYPE_VECTOR_SUBPARTS (vect_type);
1236 vec<constructor_elt, va_gc> *v;
1237 tree constr, t, si, i_val;
1238 tree vec0tmp = NULL_TREE, vec1tmp = NULL_TREE, masktmp = NULL_TREE;
1239 bool two_operand_p = !operand_equal_p (vec0, vec1, 0);
1240 location_t loc = gimple_location (gsi_stmt (*gsi));
1243 if (TREE_CODE (mask) == SSA_NAME)
1245 gimple *def_stmt = SSA_NAME_DEF_STMT (mask);
1246 if (is_gimple_assign (def_stmt)
1247 && gimple_assign_rhs_code (def_stmt) == VECTOR_CST)
1248 mask = gimple_assign_rhs1 (def_stmt);
1251 if (TREE_CODE (mask) == VECTOR_CST)
1253 unsigned char *sel_int = XALLOCAVEC (unsigned char, elements);
1255 for (i = 0; i < elements; ++i)
1256 sel_int[i] = (TREE_INT_CST_LOW (VECTOR_CST_ELT (mask, i))
1257 & (2 * elements - 1));
1259 if (can_vec_perm_p (TYPE_MODE (vect_type), false, sel_int))
1261 gimple_assign_set_rhs3 (stmt, mask);
1266 else if (can_vec_perm_p (TYPE_MODE (vect_type), true, NULL))
1269 warning_at (loc, OPT_Wvector_operation_performance,
1270 "vector shuffling operation will be expanded piecewise");
1272 vec_alloc (v, elements);
1273 for (i = 0; i < elements; i++)
1276 i_val = vector_element (gsi, mask, si, &masktmp);
1278 if (TREE_CODE (i_val) == INTEGER_CST)
1280 unsigned HOST_WIDE_INT index;
1282 index = TREE_INT_CST_LOW (i_val);
1283 if (!tree_fits_uhwi_p (i_val) || index >= elements)
1284 i_val = build_int_cst (mask_elt_type, index & (elements - 1));
1286 if (two_operand_p && (index & elements) != 0)
1287 t = vector_element (gsi, vec1, i_val, &vec1tmp);
1289 t = vector_element (gsi, vec0, i_val, &vec0tmp);
1291 t = force_gimple_operand_gsi (gsi, t, true, NULL_TREE,
1292 true, GSI_SAME_STMT);
1296 tree cond = NULL_TREE, v0_val;
1300 cond = fold_build2 (BIT_AND_EXPR, mask_elt_type, i_val,
1301 build_int_cst (mask_elt_type, elements));
1302 cond = force_gimple_operand_gsi (gsi, cond, true, NULL_TREE,
1303 true, GSI_SAME_STMT);
1306 i_val = fold_build2 (BIT_AND_EXPR, mask_elt_type, i_val,
1307 build_int_cst (mask_elt_type, elements - 1));
1308 i_val = force_gimple_operand_gsi (gsi, i_val, true, NULL_TREE,
1309 true, GSI_SAME_STMT);
1311 v0_val = vector_element (gsi, vec0, i_val, &vec0tmp);
1312 v0_val = force_gimple_operand_gsi (gsi, v0_val, true, NULL_TREE,
1313 true, GSI_SAME_STMT);
1319 v1_val = vector_element (gsi, vec1, i_val, &vec1tmp);
1320 v1_val = force_gimple_operand_gsi (gsi, v1_val, true, NULL_TREE,
1321 true, GSI_SAME_STMT);
1323 cond = fold_build2 (EQ_EXPR, boolean_type_node,
1324 cond, build_zero_cst (mask_elt_type));
1325 cond = fold_build3 (COND_EXPR, vect_elt_type,
1326 cond, v0_val, v1_val);
1327 t = force_gimple_operand_gsi (gsi, cond, true, NULL_TREE,
1328 true, GSI_SAME_STMT);
1334 CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, t);
1337 constr = build_constructor (vect_type, v);
1338 gimple_assign_set_rhs_from_tree (gsi, constr);
1339 update_stmt (gsi_stmt (*gsi));
1342 /* If OP is a uniform vector return the element it is a splat from. */
1345 ssa_uniform_vector_p (tree op)
1347 if (TREE_CODE (op) == VECTOR_CST
1348 || TREE_CODE (op) == CONSTRUCTOR)
1349 return uniform_vector_p (op);
1350 if (TREE_CODE (op) == SSA_NAME)
1352 gimple *def_stmt = SSA_NAME_DEF_STMT (op);
1353 if (gimple_assign_single_p (def_stmt))
1354 return uniform_vector_p (gimple_assign_rhs1 (def_stmt));
1359 /* Return type in which CODE operation with optab OP can be
1363 get_compute_type (enum tree_code code, optab op, tree type)
1365 /* For very wide vectors, try using a smaller vector mode. */
1366 tree compute_type = type;
1368 && (!VECTOR_MODE_P (TYPE_MODE (type))
1369 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing))
1371 tree vector_compute_type
1372 = type_for_widest_vector_mode (TREE_TYPE (type), op);
1373 if (vector_compute_type != NULL_TREE
1374 && (TYPE_VECTOR_SUBPARTS (vector_compute_type)
1375 < TYPE_VECTOR_SUBPARTS (compute_type))
1376 && (optab_handler (op, TYPE_MODE (vector_compute_type))
1377 != CODE_FOR_nothing))
1378 compute_type = vector_compute_type;
1381 /* If we are breaking a BLKmode vector into smaller pieces,
1382 type_for_widest_vector_mode has already looked into the optab,
1383 so skip these checks. */
1384 if (compute_type == type)
1386 machine_mode compute_mode = TYPE_MODE (compute_type);
1387 if (VECTOR_MODE_P (compute_mode))
1389 if (op && optab_handler (op, compute_mode) != CODE_FOR_nothing)
1390 return compute_type;
1391 if (code == MULT_HIGHPART_EXPR
1392 && can_mult_highpart_p (compute_mode,
1393 TYPE_UNSIGNED (compute_type)))
1394 return compute_type;
1396 /* There is no operation in hardware, so fall back to scalars. */
1397 compute_type = TREE_TYPE (type);
1400 return compute_type;
1403 /* Helper function of expand_vector_operations_1. Return number of
1404 vector elements for vector types or 1 for other types. */
1407 count_type_subparts (tree type)
1409 return VECTOR_TYPE_P (type) ? TYPE_VECTOR_SUBPARTS (type) : 1;
1413 do_cond (gimple_stmt_iterator *gsi, tree inner_type, tree a, tree b,
1414 tree bitpos, tree bitsize, enum tree_code code,
1415 tree type ATTRIBUTE_UNUSED)
1417 if (TREE_CODE (TREE_TYPE (a)) == VECTOR_TYPE)
1418 a = tree_vec_extract (gsi, inner_type, a, bitsize, bitpos);
1419 if (TREE_CODE (TREE_TYPE (b)) == VECTOR_TYPE)
1420 b = tree_vec_extract (gsi, inner_type, b, bitsize, bitpos);
1421 tree cond = gimple_assign_rhs1 (gsi_stmt (*gsi));
1422 return gimplify_build3 (gsi, code, inner_type, cond, a, b);
1425 /* Expand a vector COND_EXPR to scalars, piecewise. */
1427 expand_vector_scalar_condition (gimple_stmt_iterator *gsi)
1429 gassign *stmt = as_a <gassign *> (gsi_stmt (*gsi));
1430 tree type = gimple_expr_type (stmt);
1431 tree compute_type = get_compute_type (COND_EXPR, mov_optab, type);
1432 machine_mode compute_mode = TYPE_MODE (compute_type);
1433 gcc_assert (compute_mode != BLKmode);
1434 tree lhs = gimple_assign_lhs (stmt);
1435 tree rhs2 = gimple_assign_rhs2 (stmt);
1436 tree rhs3 = gimple_assign_rhs3 (stmt);
1439 /* If the compute mode is not a vector mode (hence we are not decomposing
1440 a BLKmode vector to smaller, hardware-supported vectors), we may want
1441 to expand the operations in parallel. */
1442 if (GET_MODE_CLASS (compute_mode) != MODE_VECTOR_INT
1443 && GET_MODE_CLASS (compute_mode) != MODE_VECTOR_FLOAT
1444 && GET_MODE_CLASS (compute_mode) != MODE_VECTOR_FRACT
1445 && GET_MODE_CLASS (compute_mode) != MODE_VECTOR_UFRACT
1446 && GET_MODE_CLASS (compute_mode) != MODE_VECTOR_ACCUM
1447 && GET_MODE_CLASS (compute_mode) != MODE_VECTOR_UACCUM)
1448 new_rhs = expand_vector_parallel (gsi, do_cond, type, rhs2, rhs3,
1451 new_rhs = expand_vector_piecewise (gsi, do_cond, type, compute_type,
1452 rhs2, rhs3, COND_EXPR);
1453 if (!useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (new_rhs)))
1454 new_rhs = gimplify_build1 (gsi, VIEW_CONVERT_EXPR, TREE_TYPE (lhs),
1457 /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
1458 way to do it is change expand_vector_operation and its callees to
1459 return a tree_code, RHS1 and RHS2 instead of a tree. */
1460 gimple_assign_set_rhs_from_tree (gsi, new_rhs);
1461 update_stmt (gsi_stmt (*gsi));
1464 /* Process one statement. If we identify a vector operation, expand it. */
1467 expand_vector_operations_1 (gimple_stmt_iterator *gsi)
1469 tree lhs, rhs1, rhs2 = NULL, type, compute_type = NULL_TREE;
1470 enum tree_code code;
1471 optab op = unknown_optab;
1472 enum gimple_rhs_class rhs_class;
1475 /* Only consider code == GIMPLE_ASSIGN. */
1476 gassign *stmt = dyn_cast <gassign *> (gsi_stmt (*gsi));
1480 code = gimple_assign_rhs_code (stmt);
1481 rhs_class = get_gimple_rhs_class (code);
1482 lhs = gimple_assign_lhs (stmt);
1484 if (code == VEC_PERM_EXPR)
1486 lower_vec_perm (gsi);
1490 if (code == VEC_COND_EXPR)
1492 expand_vector_condition (gsi);
1496 if (code == COND_EXPR
1497 && TREE_CODE (TREE_TYPE (gimple_assign_lhs (stmt))) == VECTOR_TYPE
1498 && TYPE_MODE (TREE_TYPE (gimple_assign_lhs (stmt))) == BLKmode)
1500 expand_vector_scalar_condition (gsi);
1504 if (code == CONSTRUCTOR
1505 && TREE_CODE (lhs) == SSA_NAME
1506 && VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (lhs)))
1507 && !gimple_clobber_p (stmt)
1510 optimize_vector_constructor (gsi);
1514 if (rhs_class != GIMPLE_UNARY_RHS && rhs_class != GIMPLE_BINARY_RHS)
1517 rhs1 = gimple_assign_rhs1 (stmt);
1518 type = gimple_expr_type (stmt);
1519 if (rhs_class == GIMPLE_BINARY_RHS)
1520 rhs2 = gimple_assign_rhs2 (stmt);
1522 if (TREE_CODE (type) != VECTOR_TYPE)
1525 /* If the vector operation is operating on all same vector elements
1526 implement it with a scalar operation and a splat if the target
1527 supports the scalar operation. */
1528 tree srhs1, srhs2 = NULL_TREE;
1529 if ((srhs1 = ssa_uniform_vector_p (rhs1)) != NULL_TREE
1530 && (rhs2 == NULL_TREE
1531 || (srhs2 = ssa_uniform_vector_p (rhs2)) != NULL_TREE)
1532 /* As we query direct optabs restrict to non-convert operations. */
1533 && TYPE_MODE (TREE_TYPE (type)) == TYPE_MODE (TREE_TYPE (srhs1)))
1535 op = optab_for_tree_code (code, TREE_TYPE (type), optab_scalar);
1536 if (op >= FIRST_NORM_OPTAB && op <= LAST_NORM_OPTAB
1537 && optab_handler (op, TYPE_MODE (TREE_TYPE (type))) != CODE_FOR_nothing)
1539 tree slhs = make_ssa_name (TREE_TYPE (srhs1));
1540 gimple *repl = gimple_build_assign (slhs, code, srhs1, srhs2);
1541 gsi_insert_before (gsi, repl, GSI_SAME_STMT);
1542 gimple_assign_set_rhs_from_tree (gsi,
1543 build_vector_from_val (type, slhs));
1549 /* A scalar operation pretending to be a vector one. */
1550 if (VECTOR_BOOLEAN_TYPE_P (type)
1551 && !VECTOR_MODE_P (TYPE_MODE (type))
1552 && TYPE_MODE (type) != BLKmode)
1555 if (CONVERT_EXPR_CODE_P (code)
1556 || code == FLOAT_EXPR
1557 || code == FIX_TRUNC_EXPR
1558 || code == VIEW_CONVERT_EXPR)
1561 /* The signedness is determined from input argument. */
1562 if (code == VEC_UNPACK_FLOAT_HI_EXPR
1563 || code == VEC_UNPACK_FLOAT_LO_EXPR)
1564 type = TREE_TYPE (rhs1);
1566 /* For widening/narrowing vector operations, the relevant type is of the
1567 arguments, not the widened result. VEC_UNPACK_FLOAT_*_EXPR is
1568 calculated in the same way above. */
1569 if (code == WIDEN_SUM_EXPR
1570 || code == VEC_WIDEN_MULT_HI_EXPR
1571 || code == VEC_WIDEN_MULT_LO_EXPR
1572 || code == VEC_WIDEN_MULT_EVEN_EXPR
1573 || code == VEC_WIDEN_MULT_ODD_EXPR
1574 || code == VEC_UNPACK_HI_EXPR
1575 || code == VEC_UNPACK_LO_EXPR
1576 || code == VEC_PACK_TRUNC_EXPR
1577 || code == VEC_PACK_SAT_EXPR
1578 || code == VEC_PACK_FIX_TRUNC_EXPR
1579 || code == VEC_WIDEN_LSHIFT_HI_EXPR
1580 || code == VEC_WIDEN_LSHIFT_LO_EXPR)
1581 type = TREE_TYPE (rhs1);
1583 /* Choose between vector shift/rotate by vector and vector shift/rotate by
1585 if (code == LSHIFT_EXPR
1586 || code == RSHIFT_EXPR
1587 || code == LROTATE_EXPR
1588 || code == RROTATE_EXPR)
1592 /* Check whether we have vector <op> {x,x,x,x} where x
1593 could be a scalar variable or a constant. Transform
1594 vector <op> {x,x,x,x} ==> vector <op> scalar. */
1595 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2)))
1599 if ((first = ssa_uniform_vector_p (rhs2)) != NULL_TREE)
1601 gimple_assign_set_rhs2 (stmt, first);
1607 opv = optab_for_tree_code (code, type, optab_vector);
1608 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2)))
1612 op = optab_for_tree_code (code, type, optab_scalar);
1614 compute_type = get_compute_type (code, op, type);
1615 if (compute_type == type)
1617 /* The rtl expander will expand vector/scalar as vector/vector
1618 if necessary. Pick one with wider vector type. */
1619 tree compute_vtype = get_compute_type (code, opv, type);
1620 if (count_type_subparts (compute_vtype)
1621 > count_type_subparts (compute_type))
1623 compute_type = compute_vtype;
1628 if (code == LROTATE_EXPR || code == RROTATE_EXPR)
1630 if (compute_type == NULL_TREE)
1631 compute_type = get_compute_type (code, op, type);
1632 if (compute_type == type)
1634 /* Before splitting vector rotates into scalar rotates,
1635 see if we can't use vector shifts and BIT_IOR_EXPR
1636 instead. For vector by vector rotates we'd also
1637 need to check BIT_AND_EXPR and NEGATE_EXPR, punt there
1638 for now, fold doesn't seem to create such rotates anyway. */
1639 if (compute_type == TREE_TYPE (type)
1640 && !VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2)))
1642 optab oplv = vashl_optab, opl = ashl_optab;
1643 optab oprv = vlshr_optab, opr = lshr_optab, opo = ior_optab;
1644 tree compute_lvtype = get_compute_type (LSHIFT_EXPR, oplv, type);
1645 tree compute_rvtype = get_compute_type (RSHIFT_EXPR, oprv, type);
1646 tree compute_otype = get_compute_type (BIT_IOR_EXPR, opo, type);
1647 tree compute_ltype = get_compute_type (LSHIFT_EXPR, opl, type);
1648 tree compute_rtype = get_compute_type (RSHIFT_EXPR, opr, type);
1649 /* The rtl expander will expand vector/scalar as vector/vector
1650 if necessary. Pick one with wider vector type. */
1651 if (count_type_subparts (compute_lvtype)
1652 > count_type_subparts (compute_ltype))
1654 compute_ltype = compute_lvtype;
1657 if (count_type_subparts (compute_rvtype)
1658 > count_type_subparts (compute_rtype))
1660 compute_rtype = compute_rvtype;
1663 /* Pick the narrowest type from LSHIFT_EXPR, RSHIFT_EXPR and
1665 compute_type = compute_ltype;
1666 if (count_type_subparts (compute_type)
1667 > count_type_subparts (compute_rtype))
1668 compute_type = compute_rtype;
1669 if (count_type_subparts (compute_type)
1670 > count_type_subparts (compute_otype))
1671 compute_type = compute_otype;
1672 /* Verify all 3 operations can be performed in that type. */
1673 if (compute_type != TREE_TYPE (type))
1675 if (optab_handler (opl, TYPE_MODE (compute_type))
1677 || optab_handler (opr, TYPE_MODE (compute_type))
1679 || optab_handler (opo, TYPE_MODE (compute_type))
1680 == CODE_FOR_nothing)
1681 compute_type = TREE_TYPE (type);
1687 op = optab_for_tree_code (code, type, optab_default);
1689 /* Optabs will try converting a negation into a subtraction, so
1690 look for it as well. TODO: negation of floating-point vectors
1691 might be turned into an exclusive OR toggling the sign bit. */
1692 if (op == unknown_optab
1693 && code == NEGATE_EXPR
1694 && INTEGRAL_TYPE_P (TREE_TYPE (type)))
1695 op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
1697 if (compute_type == NULL_TREE)
1698 compute_type = get_compute_type (code, op, type);
1699 if (compute_type == type)
1702 new_rhs = expand_vector_operation (gsi, type, compute_type, stmt, code);
1704 /* Leave expression untouched for later expansion. */
1705 if (new_rhs == NULL_TREE)
1708 if (!useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (new_rhs)))
1709 new_rhs = gimplify_build1 (gsi, VIEW_CONVERT_EXPR, TREE_TYPE (lhs),
1712 /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
1713 way to do it is change expand_vector_operation and its callees to
1714 return a tree_code, RHS1 and RHS2 instead of a tree. */
1715 gimple_assign_set_rhs_from_tree (gsi, new_rhs);
1716 update_stmt (gsi_stmt (*gsi));
1719 /* Use this to lower vector operations introduced by the vectorizer,
1720 if it may need the bit-twiddling tricks implemented in this file. */
1723 expand_vector_operations (void)
1725 gimple_stmt_iterator gsi;
1727 bool cfg_changed = false;
1729 FOR_EACH_BB_FN (bb, cfun)
1731 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1733 expand_vector_operations_1 (&gsi);
1734 /* ??? If we do not cleanup EH then we will ICE in
1735 verification. But in reality we have created wrong-code
1736 as we did not properly transition EH info and edges to
1737 the piecewise computations. */
1738 if (maybe_clean_eh_stmt (gsi_stmt (gsi))
1739 && gimple_purge_dead_eh_edges (bb))
1744 return cfg_changed ? TODO_cleanup_cfg : 0;
1749 const pass_data pass_data_lower_vector =
1751 GIMPLE_PASS, /* type */
1752 "veclower", /* name */
1753 OPTGROUP_VEC, /* optinfo_flags */
1754 TV_NONE, /* tv_id */
1755 PROP_cfg, /* properties_required */
1756 PROP_gimple_lvec, /* properties_provided */
1757 0, /* properties_destroyed */
1758 0, /* todo_flags_start */
1759 TODO_update_ssa, /* todo_flags_finish */
1762 class pass_lower_vector : public gimple_opt_pass
1765 pass_lower_vector (gcc::context *ctxt)
1766 : gimple_opt_pass (pass_data_lower_vector, ctxt)
1769 /* opt_pass methods: */
1770 virtual bool gate (function *fun)
1772 return !(fun->curr_properties & PROP_gimple_lvec);
1775 virtual unsigned int execute (function *)
1777 return expand_vector_operations ();
1780 }; // class pass_lower_vector
1785 make_pass_lower_vector (gcc::context *ctxt)
1787 return new pass_lower_vector (ctxt);
1792 const pass_data pass_data_lower_vector_ssa =
1794 GIMPLE_PASS, /* type */
1795 "veclower2", /* name */
1796 OPTGROUP_VEC, /* optinfo_flags */
1797 TV_NONE, /* tv_id */
1798 PROP_cfg, /* properties_required */
1799 PROP_gimple_lvec, /* properties_provided */
1800 0, /* properties_destroyed */
1801 0, /* todo_flags_start */
1803 | TODO_cleanup_cfg ), /* todo_flags_finish */
1806 class pass_lower_vector_ssa : public gimple_opt_pass
1809 pass_lower_vector_ssa (gcc::context *ctxt)
1810 : gimple_opt_pass (pass_data_lower_vector_ssa, ctxt)
1813 /* opt_pass methods: */
1814 opt_pass * clone () { return new pass_lower_vector_ssa (m_ctxt); }
1815 virtual unsigned int execute (function *)
1817 return expand_vector_operations ();
1820 }; // class pass_lower_vector_ssa
1825 make_pass_lower_vector_ssa (gcc::context *ctxt)
1827 return new pass_lower_vector_ssa (ctxt);
1830 #include "gt-tree-vect-generic.h"