1 /* Lower vector operations to scalar operations.
2 Copyright (C) 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012
3 Free Software Foundation, Inc.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by the
9 Free Software Foundation; either version 3, or (at your option) any
12 GCC is distributed in the hope that it will be useful, but WITHOUT
13 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
26 #include "langhooks.h"
27 #include "tree-flow.h"
29 #include "tree-iterator.h"
30 #include "tree-pass.h"
33 #include "diagnostic.h"
36 /* Need to include rtl.h, expr.h, etc. for optabs. */
41 static void expand_vector_operations_1 (gimple_stmt_iterator *);
44 /* Build a constant of type TYPE, made of VALUE's bits replicated
45 every TYPE_SIZE (INNER_TYPE) bits to fit TYPE's precision. */
47 build_replicated_const (tree type, tree inner_type, HOST_WIDE_INT value)
49 int width = tree_low_cst (TYPE_SIZE (inner_type), 1);
50 int n = HOST_BITS_PER_WIDE_INT / width;
51 unsigned HOST_WIDE_INT low, high, mask;
56 if (width == HOST_BITS_PER_WIDE_INT)
60 mask = ((HOST_WIDE_INT)1 << width) - 1;
61 low = (unsigned HOST_WIDE_INT) ~0 / mask * (value & mask);
64 if (TYPE_PRECISION (type) < HOST_BITS_PER_WIDE_INT)
65 low &= ((HOST_WIDE_INT)1 << TYPE_PRECISION (type)) - 1, high = 0;
66 else if (TYPE_PRECISION (type) == HOST_BITS_PER_WIDE_INT)
68 else if (TYPE_PRECISION (type) == HOST_BITS_PER_DOUBLE_INT)
73 ret = build_int_cst_wide (type, low, high);
77 static GTY(()) tree vector_inner_type;
78 static GTY(()) tree vector_last_type;
79 static GTY(()) int vector_last_nunits;
81 /* Return a suitable vector types made of SUBPARTS units each of mode
82 "word_mode" (the global variable). */
84 build_word_mode_vector_type (int nunits)
86 if (!vector_inner_type)
87 vector_inner_type = lang_hooks.types.type_for_mode (word_mode, 1);
88 else if (vector_last_nunits == nunits)
90 gcc_assert (TREE_CODE (vector_last_type) == VECTOR_TYPE);
91 return vector_last_type;
94 /* We build a new type, but we canonicalize it nevertheless,
95 because it still saves some memory. */
96 vector_last_nunits = nunits;
97 vector_last_type = type_hash_canon (nunits,
98 build_vector_type (vector_inner_type,
100 return vector_last_type;
103 typedef tree (*elem_op_func) (gimple_stmt_iterator *,
104 tree, tree, tree, tree, tree, enum tree_code);
107 tree_vec_extract (gimple_stmt_iterator *gsi, tree type,
108 tree t, tree bitsize, tree bitpos)
111 return gimplify_build3 (gsi, BIT_FIELD_REF, type, t, bitsize, bitpos);
113 return gimplify_build1 (gsi, VIEW_CONVERT_EXPR, type, t);
117 do_unop (gimple_stmt_iterator *gsi, tree inner_type, tree a,
118 tree b ATTRIBUTE_UNUSED, tree bitpos, tree bitsize,
121 a = tree_vec_extract (gsi, inner_type, a, bitsize, bitpos);
122 return gimplify_build1 (gsi, code, inner_type, a);
126 do_binop (gimple_stmt_iterator *gsi, tree inner_type, tree a, tree b,
127 tree bitpos, tree bitsize, enum tree_code code)
129 if (TREE_CODE (TREE_TYPE (a)) == VECTOR_TYPE)
130 a = tree_vec_extract (gsi, inner_type, a, bitsize, bitpos);
131 if (TREE_CODE (TREE_TYPE (b)) == VECTOR_TYPE)
132 b = tree_vec_extract (gsi, inner_type, b, bitsize, bitpos);
133 return gimplify_build2 (gsi, code, inner_type, a, b);
136 /* Construct expression (A[BITPOS] code B[BITPOS]) ? -1 : 0
138 INNER_TYPE is the type of A and B elements
140 returned expression is of signed integer type with the
141 size equal to the size of INNER_TYPE. */
143 do_compare (gimple_stmt_iterator *gsi, tree inner_type, tree a, tree b,
144 tree bitpos, tree bitsize, enum tree_code code)
148 a = tree_vec_extract (gsi, inner_type, a, bitsize, bitpos);
149 b = tree_vec_extract (gsi, inner_type, b, bitsize, bitpos);
151 comp_type = build_nonstandard_integer_type
152 (GET_MODE_BITSIZE (TYPE_MODE (inner_type)), 0);
154 return gimplify_build3 (gsi, COND_EXPR, comp_type,
155 fold_build2 (code, boolean_type_node, a, b),
156 build_int_cst (comp_type, -1),
157 build_int_cst (comp_type, 0));
160 /* Expand vector addition to scalars. This does bit twiddling
161 in order to increase parallelism:
163 a + b = (((int) a & 0x7f7f7f7f) + ((int) b & 0x7f7f7f7f)) ^
166 a - b = (((int) a | 0x80808080) - ((int) b & 0x7f7f7f7f)) ^
167 (a ^ ~b) & 0x80808080
169 -b = (0x80808080 - ((int) b & 0x7f7f7f7f)) ^ (~b & 0x80808080)
171 This optimization should be done only if 4 vector items or more
174 do_plus_minus (gimple_stmt_iterator *gsi, tree word_type, tree a, tree b,
175 tree bitpos ATTRIBUTE_UNUSED, tree bitsize ATTRIBUTE_UNUSED,
178 tree inner_type = TREE_TYPE (TREE_TYPE (a));
179 unsigned HOST_WIDE_INT max;
180 tree low_bits, high_bits, a_low, b_low, result_low, signs;
182 max = GET_MODE_MASK (TYPE_MODE (inner_type));
183 low_bits = build_replicated_const (word_type, inner_type, max >> 1);
184 high_bits = build_replicated_const (word_type, inner_type, max & ~(max >> 1));
186 a = tree_vec_extract (gsi, word_type, a, bitsize, bitpos);
187 b = tree_vec_extract (gsi, word_type, b, bitsize, bitpos);
189 signs = gimplify_build2 (gsi, BIT_XOR_EXPR, word_type, a, b);
190 b_low = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, b, low_bits);
191 if (code == PLUS_EXPR)
192 a_low = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, a, low_bits);
195 a_low = gimplify_build2 (gsi, BIT_IOR_EXPR, word_type, a, high_bits);
196 signs = gimplify_build1 (gsi, BIT_NOT_EXPR, word_type, signs);
199 signs = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, signs, high_bits);
200 result_low = gimplify_build2 (gsi, code, word_type, a_low, b_low);
201 return gimplify_build2 (gsi, BIT_XOR_EXPR, word_type, result_low, signs);
205 do_negate (gimple_stmt_iterator *gsi, tree word_type, tree b,
206 tree unused ATTRIBUTE_UNUSED, tree bitpos ATTRIBUTE_UNUSED,
207 tree bitsize ATTRIBUTE_UNUSED,
208 enum tree_code code ATTRIBUTE_UNUSED)
210 tree inner_type = TREE_TYPE (TREE_TYPE (b));
212 tree low_bits, high_bits, b_low, result_low, signs;
214 max = GET_MODE_MASK (TYPE_MODE (inner_type));
215 low_bits = build_replicated_const (word_type, inner_type, max >> 1);
216 high_bits = build_replicated_const (word_type, inner_type, max & ~(max >> 1));
218 b = tree_vec_extract (gsi, word_type, b, bitsize, bitpos);
220 b_low = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, b, low_bits);
221 signs = gimplify_build1 (gsi, BIT_NOT_EXPR, word_type, b);
222 signs = gimplify_build2 (gsi, BIT_AND_EXPR, word_type, signs, high_bits);
223 result_low = gimplify_build2 (gsi, MINUS_EXPR, word_type, high_bits, b_low);
224 return gimplify_build2 (gsi, BIT_XOR_EXPR, word_type, result_low, signs);
227 /* Expand a vector operation to scalars, by using many operations
228 whose type is the vector type's inner type. */
230 expand_vector_piecewise (gimple_stmt_iterator *gsi, elem_op_func f,
231 tree type, tree inner_type,
232 tree a, tree b, enum tree_code code)
234 VEC(constructor_elt,gc) *v;
235 tree part_width = TYPE_SIZE (inner_type);
236 tree index = bitsize_int (0);
237 int nunits = TYPE_VECTOR_SUBPARTS (type);
238 int delta = tree_low_cst (part_width, 1)
239 / tree_low_cst (TYPE_SIZE (TREE_TYPE (type)), 1);
241 location_t loc = gimple_location (gsi_stmt (*gsi));
243 if (types_compatible_p (gimple_expr_type (gsi_stmt (*gsi)), type))
244 warning_at (loc, OPT_Wvector_operation_performance,
245 "vector operation will be expanded piecewise");
247 warning_at (loc, OPT_Wvector_operation_performance,
248 "vector operation will be expanded in parallel");
250 v = VEC_alloc(constructor_elt, gc, (nunits + delta - 1) / delta);
251 for (i = 0; i < nunits;
252 i += delta, index = int_const_binop (PLUS_EXPR, index, part_width))
254 tree result = f (gsi, inner_type, a, b, index, part_width, code);
255 constructor_elt *ce = VEC_quick_push (constructor_elt, v, NULL);
256 ce->index = NULL_TREE;
260 return build_constructor (type, v);
263 /* Expand a vector operation to scalars with the freedom to use
264 a scalar integer type, or to use a different size for the items
265 in the vector type. */
267 expand_vector_parallel (gimple_stmt_iterator *gsi, elem_op_func f, tree type,
271 tree result, compute_type;
272 enum machine_mode mode;
273 int n_words = tree_low_cst (TYPE_SIZE_UNIT (type), 1) / UNITS_PER_WORD;
274 location_t loc = gimple_location (gsi_stmt (*gsi));
276 /* We have three strategies. If the type is already correct, just do
277 the operation an element at a time. Else, if the vector is wider than
278 one word, do it a word at a time; finally, if the vector is smaller
279 than one word, do it as a scalar. */
280 if (TYPE_MODE (TREE_TYPE (type)) == word_mode)
281 return expand_vector_piecewise (gsi, f,
282 type, TREE_TYPE (type),
284 else if (n_words > 1)
286 tree word_type = build_word_mode_vector_type (n_words);
287 result = expand_vector_piecewise (gsi, f,
288 word_type, TREE_TYPE (word_type),
290 result = force_gimple_operand_gsi (gsi, result, true, NULL, true,
295 /* Use a single scalar operation with a mode no wider than word_mode. */
296 mode = mode_for_size (tree_low_cst (TYPE_SIZE (type), 1), MODE_INT, 0);
297 compute_type = lang_hooks.types.type_for_mode (mode, 1);
298 result = f (gsi, compute_type, a, b, NULL_TREE, NULL_TREE, code);
299 warning_at (loc, OPT_Wvector_operation_performance,
300 "vector operation will be expanded with a "
301 "single scalar operation");
307 /* Expand a vector operation to scalars; for integer types we can use
308 special bit twiddling tricks to do the sums a word at a time, using
309 function F_PARALLEL instead of F. These tricks are done only if
310 they can process at least four items, that is, only if the vector
311 holds at least four items and if a word can hold four items. */
313 expand_vector_addition (gimple_stmt_iterator *gsi,
314 elem_op_func f, elem_op_func f_parallel,
315 tree type, tree a, tree b, enum tree_code code)
317 int parts_per_word = UNITS_PER_WORD
318 / tree_low_cst (TYPE_SIZE_UNIT (TREE_TYPE (type)), 1);
320 if (INTEGRAL_TYPE_P (TREE_TYPE (type))
321 && parts_per_word >= 4
322 && TYPE_VECTOR_SUBPARTS (type) >= 4)
323 return expand_vector_parallel (gsi, f_parallel,
326 return expand_vector_piecewise (gsi, f,
327 type, TREE_TYPE (type),
331 /* Check if vector VEC consists of all the equal elements and
332 that the number of elements corresponds to the type of VEC.
333 The function returns first element of the vector
334 or NULL_TREE if the vector is not uniform. */
336 uniform_vector_p (tree vec)
341 if (vec == NULL_TREE)
344 if (TREE_CODE (vec) == VECTOR_CST)
346 first = VECTOR_CST_ELT (vec, 0);
347 for (i = 1; i < VECTOR_CST_NELTS (vec); ++i)
348 if (!operand_equal_p (first, VECTOR_CST_ELT (vec, i), 0))
354 else if (TREE_CODE (vec) == CONSTRUCTOR)
356 first = error_mark_node;
358 FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (vec), i, t)
365 if (!operand_equal_p (first, t, 0))
368 if (i != TYPE_VECTOR_SUBPARTS (TREE_TYPE (vec)))
377 /* Try to expand vector comparison expression OP0 CODE OP1 by
378 querying optab if the following expression:
379 VEC_COND_EXPR< OP0 CODE OP1, {-1,...}, {0,...}>
382 expand_vector_comparison (gimple_stmt_iterator *gsi, tree type, tree op0,
383 tree op1, enum tree_code code)
386 if (! expand_vec_cond_expr_p (type, TREE_TYPE (op0)))
387 t = expand_vector_piecewise (gsi, do_compare, type,
388 TREE_TYPE (TREE_TYPE (op0)), op0, op1, code);
395 /* Helper function of expand_vector_divmod. Gimplify a RSHIFT_EXPR in type
396 of OP0 with shift counts in SHIFTCNTS array and return the temporary holding
397 the result if successful, otherwise return NULL_TREE. */
399 add_rshift (gimple_stmt_iterator *gsi, tree type, tree op0, int *shiftcnts)
402 unsigned int i, nunits = TYPE_VECTOR_SUBPARTS (type);
403 bool scalar_shift = true;
405 for (i = 1; i < nunits; i++)
407 if (shiftcnts[i] != shiftcnts[0])
408 scalar_shift = false;
411 if (scalar_shift && shiftcnts[0] == 0)
416 op = optab_for_tree_code (RSHIFT_EXPR, type, optab_scalar);
418 && optab_handler (op, TYPE_MODE (type)) != CODE_FOR_nothing)
419 return gimplify_build2 (gsi, RSHIFT_EXPR, type, op0,
420 build_int_cst (NULL_TREE, shiftcnts[0]));
423 op = optab_for_tree_code (RSHIFT_EXPR, type, optab_vector);
425 && optab_handler (op, TYPE_MODE (type)) != CODE_FOR_nothing)
427 tree *vec = XALLOCAVEC (tree, nunits);
428 for (i = 0; i < nunits; i++)
429 vec[i] = build_int_cst (TREE_TYPE (type), shiftcnts[i]);
430 return gimplify_build2 (gsi, RSHIFT_EXPR, type, op0,
431 build_vector (type, vec));
437 /* Try to expand integer vector division by constant using
438 widening multiply, shifts and additions. */
440 expand_vector_divmod (gimple_stmt_iterator *gsi, tree type, tree op0,
441 tree op1, enum tree_code code)
443 bool use_pow2 = true;
444 bool has_vector_shift = true;
445 int mode = -1, this_mode;
446 int pre_shift = -1, post_shift;
447 unsigned int nunits = TYPE_VECTOR_SUBPARTS (type);
448 int *shifts = XALLOCAVEC (int, nunits * 4);
449 int *pre_shifts = shifts + nunits;
450 int *post_shifts = pre_shifts + nunits;
451 int *shift_temps = post_shifts + nunits;
452 unsigned HOST_WIDE_INT *mulc = XALLOCAVEC (unsigned HOST_WIDE_INT, nunits);
453 int prec = TYPE_PRECISION (TREE_TYPE (type));
455 unsigned int i, unsignedp = TYPE_UNSIGNED (TREE_TYPE (type));
456 unsigned HOST_WIDE_INT mask = GET_MODE_MASK (TYPE_MODE (TREE_TYPE (type)));
459 unsigned char *sel = NULL;
460 tree cur_op, m1, m2, mulcst, perm_mask, wider_type, tem, decl_e, decl_o;
462 if (prec > HOST_BITS_PER_WIDE_INT)
465 op = optab_for_tree_code (RSHIFT_EXPR, type, optab_vector);
467 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing)
468 has_vector_shift = false;
470 /* Analysis phase. Determine if all op1 elements are either power
471 of two and it is possible to expand it using shifts (or for remainder
472 using masking). Additionally compute the multiplicative constants
473 and pre and post shifts if the division is to be expanded using
474 widening or high part multiplication plus shifts. */
475 for (i = 0; i < nunits; i++)
477 tree cst = VECTOR_CST_ELT (op1, i);
478 unsigned HOST_WIDE_INT ml;
480 if (!host_integerp (cst, unsignedp) || integer_zerop (cst))
486 && (!integer_pow2p (cst) || tree_int_cst_sgn (cst) != 1))
490 shifts[i] = tree_log2 (cst);
491 if (shifts[i] != shifts[0]
492 && code == TRUNC_DIV_EXPR
493 && !has_vector_shift)
500 unsigned HOST_WIDE_INT mh;
501 unsigned HOST_WIDE_INT d = tree_low_cst (cst, 1) & mask;
503 if (d >= ((unsigned HOST_WIDE_INT) 1 << (prec - 1)))
504 /* FIXME: Can transform this into op0 >= op1 ? 1 : 0. */
513 /* Find a suitable multiplier and right shift count
514 instead of multiplying with D. */
515 mh = choose_multiplier (d, prec, prec, &ml, &post_shift, &dummy_int);
517 /* If the suggested multiplier is more than SIZE bits, we can
518 do better for even divisors, using an initial right shift. */
519 if ((mh != 0 && (d & 1) == 0)
520 || (!has_vector_shift && pre_shift != -1))
522 if (has_vector_shift)
523 pre_shift = floor_log2 (d & -d);
524 else if (pre_shift == -1)
527 for (j = 0; j < nunits; j++)
529 tree cst2 = VECTOR_CST_ELT (op1, j);
530 unsigned HOST_WIDE_INT d2;
533 if (!host_integerp (cst2, 1))
535 d2 = tree_low_cst (cst2, 1) & mask;
538 this_pre_shift = floor_log2 (d2 & -d2);
539 if (pre_shift == -1 || this_pre_shift < pre_shift)
540 pre_shift = this_pre_shift;
542 if (i != 0 && pre_shift != 0)
552 if ((d >> pre_shift) <= 1)
557 mh = choose_multiplier (d >> pre_shift, prec,
559 &ml, &post_shift, &dummy_int);
561 pre_shifts[i] = pre_shift;
571 HOST_WIDE_INT d = tree_low_cst (cst, 0);
572 unsigned HOST_WIDE_INT abs_d;
577 /* Since d might be INT_MIN, we have to cast to
578 unsigned HOST_WIDE_INT before negating to avoid
579 undefined signed overflow. */
581 ? (unsigned HOST_WIDE_INT) d
582 : - (unsigned HOST_WIDE_INT) d);
584 /* n rem d = n rem -d */
585 if (code == TRUNC_MOD_EXPR && d < 0)
587 else if (abs_d == (unsigned HOST_WIDE_INT) 1 << (prec - 1))
589 /* This case is not handled correctly below. */
599 choose_multiplier (abs_d, prec, prec - 1, &ml,
600 &post_shift, &dummy_int);
601 if (ml >= (unsigned HOST_WIDE_INT) 1 << (prec - 1))
603 this_mode = 4 + (d < 0);
604 ml |= (~(unsigned HOST_WIDE_INT) 0) << (prec - 1);
607 this_mode = 2 + (d < 0);
610 post_shifts[i] = post_shift;
611 if ((i && !has_vector_shift && post_shifts[0] != post_shift)
612 || post_shift >= prec
613 || pre_shifts[i] >= prec)
618 else if (mode != this_mode)
622 vec = XALLOCAVEC (tree, nunits);
626 tree addend = NULL_TREE;
631 /* Both division and remainder sequences need
632 op0 < 0 ? mask : 0 computed. It can be either computed as
633 (type) (((uns_type) (op0 >> (prec - 1))) >> (prec - shifts[i]))
634 if none of the shifts is 0, or as the conditional. */
635 for (i = 0; i < nunits; i++)
639 = build_vector_type (build_nonstandard_integer_type (prec, 1),
641 if (i == nunits && TYPE_MODE (uns_type) == TYPE_MODE (type))
643 for (i = 0; i < nunits; i++)
644 shift_temps[i] = prec - 1;
645 cur_op = add_rshift (gsi, type, op0, shift_temps);
646 if (cur_op != NULL_TREE)
648 cur_op = gimplify_build1 (gsi, VIEW_CONVERT_EXPR,
650 for (i = 0; i < nunits; i++)
651 shift_temps[i] = prec - shifts[i];
652 cur_op = add_rshift (gsi, uns_type, cur_op, shift_temps);
653 if (cur_op != NULL_TREE)
654 addend = gimplify_build1 (gsi, VIEW_CONVERT_EXPR,
658 if (addend == NULL_TREE
659 && expand_vec_cond_expr_p (type, type))
661 tree zero, cst, cond;
664 zero = build_zero_cst (type);
665 cond = build2 (LT_EXPR, type, op0, zero);
666 for (i = 0; i < nunits; i++)
667 vec[i] = build_int_cst (TREE_TYPE (type),
668 ((unsigned HOST_WIDE_INT) 1
670 cst = build_vector (type, vec);
671 addend = create_tmp_reg (type, NULL);
672 add_referenced_var (addend);
673 addend = make_ssa_name (addend, NULL);
674 stmt = gimple_build_assign_with_ops3 (VEC_COND_EXPR, addend,
676 gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
679 if (code == TRUNC_DIV_EXPR)
683 /* q = op0 >> shift; */
684 cur_op = add_rshift (gsi, type, op0, shifts);
685 if (cur_op != NULL_TREE)
688 else if (addend != NULL_TREE)
690 /* t1 = op0 + addend;
692 op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
694 && optab_handler (op, TYPE_MODE (type)) != CODE_FOR_nothing)
696 cur_op = gimplify_build2 (gsi, PLUS_EXPR, type, op0, addend);
697 cur_op = add_rshift (gsi, type, cur_op, shifts);
698 if (cur_op != NULL_TREE)
706 for (i = 0; i < nunits; i++)
707 vec[i] = build_int_cst (TREE_TYPE (type),
708 ((unsigned HOST_WIDE_INT) 1
710 mask = build_vector (type, vec);
711 op = optab_for_tree_code (BIT_AND_EXPR, type, optab_default);
713 && optab_handler (op, TYPE_MODE (type)) != CODE_FOR_nothing)
716 /* r = op0 & mask; */
717 return gimplify_build2 (gsi, BIT_AND_EXPR, type, op0, mask);
718 else if (addend != NULL_TREE)
720 /* t1 = op0 + addend;
723 op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
725 && optab_handler (op, TYPE_MODE (type))
728 cur_op = gimplify_build2 (gsi, PLUS_EXPR, type, op0,
730 cur_op = gimplify_build2 (gsi, BIT_AND_EXPR, type,
732 op = optab_for_tree_code (MINUS_EXPR, type,
735 && optab_handler (op, TYPE_MODE (type))
737 return gimplify_build2 (gsi, MINUS_EXPR, type,
745 if (mode == -2 || BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
748 op = optab_for_tree_code (MULT_HIGHPART_EXPR, type, optab_default);
749 if (op != NULL && optab_handler (op, TYPE_MODE (type)) != CODE_FOR_nothing)
750 wider_type = decl_e = decl_o = NULL_TREE;
753 wider_type = build_nonstandard_integer_type (prec * 2, unsignedp),
754 wider_type = build_vector_type (wider_type, nunits / 2);
755 if (GET_MODE_CLASS (TYPE_MODE (wider_type)) != MODE_VECTOR_INT
756 || GET_MODE_BITSIZE (TYPE_MODE (wider_type))
757 != GET_MODE_BITSIZE (TYPE_MODE (type)))
760 sel = XALLOCAVEC (unsigned char, nunits);
762 if (targetm.vectorize.builtin_mul_widen_even
763 && targetm.vectorize.builtin_mul_widen_odd
764 && (decl_e = targetm.vectorize.builtin_mul_widen_even (type))
765 && (decl_o = targetm.vectorize.builtin_mul_widen_odd (type))
766 && (TYPE_MODE (TREE_TYPE (TREE_TYPE (decl_e)))
767 == TYPE_MODE (wider_type)))
769 for (i = 0; i < nunits; i++)
770 sel[i] = !BYTES_BIG_ENDIAN + (i & ~1) + ((i & 1) ? nunits : 0);
771 if (!can_vec_perm_p (TYPE_MODE (type), false, sel))
772 decl_e = decl_o = NULL_TREE;
775 decl_e = decl_o = NULL_TREE;
777 if (decl_e == NULL_TREE)
779 op = optab_for_tree_code (VEC_WIDEN_MULT_LO_EXPR,
780 type, optab_default);
782 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing)
784 op = optab_for_tree_code (VEC_WIDEN_MULT_HI_EXPR,
785 type, optab_default);
787 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing)
790 for (i = 0; i < nunits; i++)
791 sel[i] = 2 * i + (BYTES_BIG_ENDIAN ? 0 : 1);
792 if (!can_vec_perm_p (TYPE_MODE (type), false, sel))
802 gcc_assert (unsignedp);
803 /* t1 = oprnd0 >> pre_shift;
805 q = t2 >> post_shift; */
806 cur_op = add_rshift (gsi, type, cur_op, pre_shifts);
807 if (cur_op == NULL_TREE)
811 gcc_assert (unsignedp);
812 for (i = 0; i < nunits; i++)
822 gcc_assert (!unsignedp);
823 for (i = 0; i < nunits; i++)
824 shift_temps[i] = prec - 1;
830 for (i = 0; i < nunits; i++)
831 vec[i] = build_int_cst (TREE_TYPE (type), mulc[i]);
832 mulcst = build_vector (type, vec);
833 if (wider_type == NULL_TREE)
834 cur_op = gimplify_build2 (gsi, MULT_HIGHPART_EXPR, type, cur_op, mulcst);
837 for (i = 0; i < nunits; i++)
838 vec[i] = build_int_cst (TREE_TYPE (type), sel[i]);
839 perm_mask = build_vector (type, vec);
841 if (decl_e != NULL_TREE)
845 call = gimple_build_call (decl_e, 2, cur_op, mulcst);
846 m1 = create_tmp_reg (wider_type, NULL);
847 add_referenced_var (m1);
848 m1 = make_ssa_name (m1, call);
849 gimple_call_set_lhs (call, m1);
850 gsi_insert_seq_before (gsi, call, GSI_SAME_STMT);
852 call = gimple_build_call (decl_o, 2, cur_op, mulcst);
853 m2 = create_tmp_reg (wider_type, NULL);
854 add_referenced_var (m2);
855 m2 = make_ssa_name (m2, call);
856 gimple_call_set_lhs (call, m2);
857 gsi_insert_seq_before (gsi, call, GSI_SAME_STMT);
861 m1 = gimplify_build2 (gsi, BYTES_BIG_ENDIAN ? VEC_WIDEN_MULT_HI_EXPR
862 : VEC_WIDEN_MULT_LO_EXPR,
863 wider_type, cur_op, mulcst);
864 m2 = gimplify_build2 (gsi, BYTES_BIG_ENDIAN ? VEC_WIDEN_MULT_LO_EXPR
865 : VEC_WIDEN_MULT_HI_EXPR,
866 wider_type, cur_op, mulcst);
869 m1 = gimplify_build1 (gsi, VIEW_CONVERT_EXPR, type, m1);
870 m2 = gimplify_build1 (gsi, VIEW_CONVERT_EXPR, type, m2);
871 cur_op = gimplify_build3 (gsi, VEC_PERM_EXPR, type, m1, m2, perm_mask);
877 /* t1 = oprnd0 >> pre_shift;
879 q = t2 >> post_shift; */
880 cur_op = add_rshift (gsi, type, cur_op, post_shifts);
883 /* t1 = oprnd0 h* ml;
887 q = t4 >> (post_shift - 1); */
888 op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
890 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing)
892 tem = gimplify_build2 (gsi, MINUS_EXPR, type, op0, cur_op);
893 tem = add_rshift (gsi, type, tem, shift_temps);
894 op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
896 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing)
898 tem = gimplify_build2 (gsi, PLUS_EXPR, type, cur_op, tem);
899 cur_op = add_rshift (gsi, type, tem, post_shifts);
900 if (cur_op == NULL_TREE)
907 /* t1 = oprnd0 h* ml;
908 t2 = t1; [ iff (mode & 2) != 0 ]
909 t2 = t1 + oprnd0; [ iff (mode & 2) == 0 ]
910 t3 = t2 >> post_shift;
911 t4 = oprnd0 >> (prec - 1);
912 q = t3 - t4; [ iff (mode & 1) == 0 ]
913 q = t4 - t3; [ iff (mode & 1) != 0 ] */
916 op = optab_for_tree_code (PLUS_EXPR, type, optab_default);
918 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing)
920 cur_op = gimplify_build2 (gsi, PLUS_EXPR, type, cur_op, op0);
922 cur_op = add_rshift (gsi, type, cur_op, post_shifts);
923 if (cur_op == NULL_TREE)
925 tem = add_rshift (gsi, type, op0, shift_temps);
926 if (tem == NULL_TREE)
928 op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
930 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing)
933 cur_op = gimplify_build2 (gsi, MINUS_EXPR, type, cur_op, tem);
935 cur_op = gimplify_build2 (gsi, MINUS_EXPR, type, tem, cur_op);
941 if (code == TRUNC_DIV_EXPR)
944 /* We divided. Now finish by:
947 op = optab_for_tree_code (MULT_EXPR, type, optab_default);
949 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing)
951 tem = gimplify_build2 (gsi, MULT_EXPR, type, cur_op, op1);
952 op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
954 || optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing)
956 return gimplify_build2 (gsi, MINUS_EXPR, type, op0, tem);
960 expand_vector_operation (gimple_stmt_iterator *gsi, tree type, tree compute_type,
961 gimple assign, enum tree_code code)
963 enum machine_mode compute_mode = TYPE_MODE (compute_type);
965 /* If the compute mode is not a vector mode (hence we are not decomposing
966 a BLKmode vector to smaller, hardware-supported vectors), we may want
967 to expand the operations in parallel. */
968 if (GET_MODE_CLASS (compute_mode) != MODE_VECTOR_INT
969 && GET_MODE_CLASS (compute_mode) != MODE_VECTOR_FLOAT
970 && GET_MODE_CLASS (compute_mode) != MODE_VECTOR_FRACT
971 && GET_MODE_CLASS (compute_mode) != MODE_VECTOR_UFRACT
972 && GET_MODE_CLASS (compute_mode) != MODE_VECTOR_ACCUM
973 && GET_MODE_CLASS (compute_mode) != MODE_VECTOR_UACCUM)
978 if (!TYPE_OVERFLOW_TRAPS (type))
979 return expand_vector_addition (gsi, do_binop, do_plus_minus, type,
980 gimple_assign_rhs1 (assign),
981 gimple_assign_rhs2 (assign), code);
985 if (!TYPE_OVERFLOW_TRAPS (type))
986 return expand_vector_addition (gsi, do_unop, do_negate, type,
987 gimple_assign_rhs1 (assign),
994 return expand_vector_parallel (gsi, do_binop, type,
995 gimple_assign_rhs1 (assign),
996 gimple_assign_rhs2 (assign), code);
999 return expand_vector_parallel (gsi, do_unop, type,
1000 gimple_assign_rhs1 (assign),
1015 case UNORDERED_EXPR:
1017 tree rhs1 = gimple_assign_rhs1 (assign);
1018 tree rhs2 = gimple_assign_rhs2 (assign);
1020 return expand_vector_comparison (gsi, type, rhs1, rhs2, code);
1023 case TRUNC_DIV_EXPR:
1024 case TRUNC_MOD_EXPR:
1026 tree rhs1 = gimple_assign_rhs1 (assign);
1027 tree rhs2 = gimple_assign_rhs2 (assign);
1031 || !VECTOR_INTEGER_TYPE_P (type)
1032 || TREE_CODE (rhs2) != VECTOR_CST)
1035 ret = expand_vector_divmod (gsi, type, rhs1, rhs2, code);
1036 if (ret != NULL_TREE)
1045 if (TREE_CODE_CLASS (code) == tcc_unary)
1046 return expand_vector_piecewise (gsi, do_unop, type, compute_type,
1047 gimple_assign_rhs1 (assign),
1050 return expand_vector_piecewise (gsi, do_binop, type, compute_type,
1051 gimple_assign_rhs1 (assign),
1052 gimple_assign_rhs2 (assign), code);
1055 /* Return a type for the widest vector mode whose components are of type
1056 TYPE, or NULL_TREE if none is found. */
1059 type_for_widest_vector_mode (tree type, optab op)
1061 enum machine_mode inner_mode = TYPE_MODE (type);
1062 enum machine_mode best_mode = VOIDmode, mode;
1063 int best_nunits = 0;
1065 if (SCALAR_FLOAT_MODE_P (inner_mode))
1066 mode = MIN_MODE_VECTOR_FLOAT;
1067 else if (SCALAR_FRACT_MODE_P (inner_mode))
1068 mode = MIN_MODE_VECTOR_FRACT;
1069 else if (SCALAR_UFRACT_MODE_P (inner_mode))
1070 mode = MIN_MODE_VECTOR_UFRACT;
1071 else if (SCALAR_ACCUM_MODE_P (inner_mode))
1072 mode = MIN_MODE_VECTOR_ACCUM;
1073 else if (SCALAR_UACCUM_MODE_P (inner_mode))
1074 mode = MIN_MODE_VECTOR_UACCUM;
1076 mode = MIN_MODE_VECTOR_INT;
1078 for (; mode != VOIDmode; mode = GET_MODE_WIDER_MODE (mode))
1079 if (GET_MODE_INNER (mode) == inner_mode
1080 && GET_MODE_NUNITS (mode) > best_nunits
1081 && optab_handler (op, mode) != CODE_FOR_nothing)
1082 best_mode = mode, best_nunits = GET_MODE_NUNITS (mode);
1084 if (best_mode == VOIDmode)
1087 return build_vector_type_for_mode (type, best_mode);
1091 /* Build a reference to the element of the vector VECT. Function
1092 returns either the element itself, either BIT_FIELD_REF, or an
1093 ARRAY_REF expression.
1095 GSI is required to insert temporary variables while building a
1096 refernece to the element of the vector VECT.
1098 PTMPVEC is a pointer to the temporary variable for caching
1099 purposes. In case when PTMPVEC is NULL new temporary variable
1102 vector_element (gimple_stmt_iterator *gsi, tree vect, tree idx, tree *ptmpvec)
1104 tree vect_type, vect_elt_type;
1108 bool need_asgn = true;
1109 unsigned int elements;
1111 vect_type = TREE_TYPE (vect);
1112 vect_elt_type = TREE_TYPE (vect_type);
1113 elements = TYPE_VECTOR_SUBPARTS (vect_type);
1115 if (TREE_CODE (idx) == INTEGER_CST)
1117 unsigned HOST_WIDE_INT index;
1119 /* Given that we're about to compute a binary modulus,
1120 we don't care about the high bits of the value. */
1121 index = TREE_INT_CST_LOW (idx);
1122 if (!host_integerp (idx, 1) || index >= elements)
1124 index &= elements - 1;
1125 idx = build_int_cst (TREE_TYPE (idx), index);
1128 /* When lowering a vector statement sequence do some easy
1129 simplification by looking through intermediate vector results. */
1130 if (TREE_CODE (vect) == SSA_NAME)
1132 gimple def_stmt = SSA_NAME_DEF_STMT (vect);
1133 if (is_gimple_assign (def_stmt)
1134 && (gimple_assign_rhs_code (def_stmt) == VECTOR_CST
1135 || gimple_assign_rhs_code (def_stmt) == CONSTRUCTOR))
1136 vect = gimple_assign_rhs1 (def_stmt);
1139 if (TREE_CODE (vect) == VECTOR_CST)
1140 return VECTOR_CST_ELT (vect, index);
1141 else if (TREE_CODE (vect) == CONSTRUCTOR)
1146 FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (vect), i, elt_i, elt_v)
1147 if (operand_equal_p (elt_i, idx, 0))
1149 return build_zero_cst (vect_elt_type);
1153 tree size = TYPE_SIZE (vect_elt_type);
1154 tree pos = fold_build2 (MULT_EXPR, bitsizetype, bitsize_int (index),
1156 return fold_build3 (BIT_FIELD_REF, vect_elt_type, vect, size, pos);
1161 tmpvec = create_tmp_var (vect_type, "vectmp");
1163 tmpvec = *ptmpvec = create_tmp_var (vect_type, "vectmp");
1172 TREE_ADDRESSABLE (tmpvec) = 1;
1173 asgn = gimple_build_assign (tmpvec, vect);
1174 gsi_insert_before (gsi, asgn, GSI_SAME_STMT);
1177 arraytype = build_array_type_nelts (vect_elt_type, elements);
1178 return build4 (ARRAY_REF, vect_elt_type,
1179 build1 (VIEW_CONVERT_EXPR, arraytype, tmpvec),
1180 idx, NULL_TREE, NULL_TREE);
1183 /* Check if VEC_PERM_EXPR within the given setting is supported
1184 by hardware, or lower it piecewise.
1186 When VEC_PERM_EXPR has the same first and second operands:
1187 VEC_PERM_EXPR <v0, v0, mask> the lowered version would be
1188 {v0[mask[0]], v0[mask[1]], ...}
1189 MASK and V0 must have the same number of elements.
1191 Otherwise VEC_PERM_EXPR <v0, v1, mask> is lowered to
1192 {mask[0] < len(v0) ? v0[mask[0]] : v1[mask[0]], ...}
1193 V0 and V1 must have the same type. MASK, V0, V1 must have the
1194 same number of arguments. */
1197 lower_vec_perm (gimple_stmt_iterator *gsi)
1199 gimple stmt = gsi_stmt (*gsi);
1200 tree mask = gimple_assign_rhs3 (stmt);
1201 tree vec0 = gimple_assign_rhs1 (stmt);
1202 tree vec1 = gimple_assign_rhs2 (stmt);
1203 tree vect_type = TREE_TYPE (vec0);
1204 tree mask_type = TREE_TYPE (mask);
1205 tree vect_elt_type = TREE_TYPE (vect_type);
1206 tree mask_elt_type = TREE_TYPE (mask_type);
1207 unsigned int elements = TYPE_VECTOR_SUBPARTS (vect_type);
1208 VEC(constructor_elt,gc) *v;
1209 tree constr, t, si, i_val;
1210 tree vec0tmp = NULL_TREE, vec1tmp = NULL_TREE, masktmp = NULL_TREE;
1211 bool two_operand_p = !operand_equal_p (vec0, vec1, 0);
1212 location_t loc = gimple_location (gsi_stmt (*gsi));
1215 if (TREE_CODE (mask) == SSA_NAME)
1217 gimple def_stmt = SSA_NAME_DEF_STMT (mask);
1218 if (is_gimple_assign (def_stmt)
1219 && gimple_assign_rhs_code (def_stmt) == VECTOR_CST)
1220 mask = gimple_assign_rhs1 (def_stmt);
1223 if (TREE_CODE (mask) == VECTOR_CST)
1225 unsigned char *sel_int = XALLOCAVEC (unsigned char, elements);
1227 for (i = 0; i < elements; ++i)
1228 sel_int[i] = (TREE_INT_CST_LOW (VECTOR_CST_ELT (mask, i))
1229 & (2 * elements - 1));
1231 if (can_vec_perm_p (TYPE_MODE (vect_type), false, sel_int))
1233 gimple_assign_set_rhs3 (stmt, mask);
1238 else if (can_vec_perm_p (TYPE_MODE (vect_type), true, NULL))
1241 warning_at (loc, OPT_Wvector_operation_performance,
1242 "vector shuffling operation will be expanded piecewise");
1244 v = VEC_alloc (constructor_elt, gc, elements);
1245 for (i = 0; i < elements; i++)
1248 i_val = vector_element (gsi, mask, si, &masktmp);
1250 if (TREE_CODE (i_val) == INTEGER_CST)
1252 unsigned HOST_WIDE_INT index;
1254 index = TREE_INT_CST_LOW (i_val);
1255 if (!host_integerp (i_val, 1) || index >= elements)
1256 i_val = build_int_cst (mask_elt_type, index & (elements - 1));
1258 if (two_operand_p && (index & elements) != 0)
1259 t = vector_element (gsi, vec1, i_val, &vec1tmp);
1261 t = vector_element (gsi, vec0, i_val, &vec0tmp);
1263 t = force_gimple_operand_gsi (gsi, t, true, NULL_TREE,
1264 true, GSI_SAME_STMT);
1268 tree cond = NULL_TREE, v0_val;
1272 cond = fold_build2 (BIT_AND_EXPR, mask_elt_type, i_val,
1273 build_int_cst (mask_elt_type, elements));
1274 cond = force_gimple_operand_gsi (gsi, cond, true, NULL_TREE,
1275 true, GSI_SAME_STMT);
1278 i_val = fold_build2 (BIT_AND_EXPR, mask_elt_type, i_val,
1279 build_int_cst (mask_elt_type, elements - 1));
1280 i_val = force_gimple_operand_gsi (gsi, i_val, true, NULL_TREE,
1281 true, GSI_SAME_STMT);
1283 v0_val = vector_element (gsi, vec0, i_val, &vec0tmp);
1284 v0_val = force_gimple_operand_gsi (gsi, v0_val, true, NULL_TREE,
1285 true, GSI_SAME_STMT);
1291 v1_val = vector_element (gsi, vec1, i_val, &vec1tmp);
1292 v1_val = force_gimple_operand_gsi (gsi, v1_val, true, NULL_TREE,
1293 true, GSI_SAME_STMT);
1295 cond = fold_build2 (EQ_EXPR, boolean_type_node,
1296 cond, build_zero_cst (mask_elt_type));
1297 cond = fold_build3 (COND_EXPR, vect_elt_type,
1298 cond, v0_val, v1_val);
1299 t = force_gimple_operand_gsi (gsi, cond, true, NULL_TREE,
1300 true, GSI_SAME_STMT);
1306 CONSTRUCTOR_APPEND_ELT (v, si, t);
1309 constr = build_constructor (vect_type, v);
1310 gimple_assign_set_rhs_from_tree (gsi, constr);
1311 update_stmt (gsi_stmt (*gsi));
1314 /* Process one statement. If we identify a vector operation, expand it. */
1317 expand_vector_operations_1 (gimple_stmt_iterator *gsi)
1319 gimple stmt = gsi_stmt (*gsi);
1320 tree lhs, rhs1, rhs2 = NULL, type, compute_type;
1321 enum tree_code code;
1322 enum machine_mode compute_mode;
1324 enum gimple_rhs_class rhs_class;
1327 if (gimple_code (stmt) != GIMPLE_ASSIGN)
1330 code = gimple_assign_rhs_code (stmt);
1331 rhs_class = get_gimple_rhs_class (code);
1332 lhs = gimple_assign_lhs (stmt);
1334 if (code == VEC_PERM_EXPR)
1336 lower_vec_perm (gsi);
1340 if (rhs_class != GIMPLE_UNARY_RHS && rhs_class != GIMPLE_BINARY_RHS)
1343 rhs1 = gimple_assign_rhs1 (stmt);
1344 type = gimple_expr_type (stmt);
1345 if (rhs_class == GIMPLE_BINARY_RHS)
1346 rhs2 = gimple_assign_rhs2 (stmt);
1348 if (TREE_CODE (type) != VECTOR_TYPE)
1351 if (code == NOP_EXPR
1352 || code == FLOAT_EXPR
1353 || code == FIX_TRUNC_EXPR
1354 || code == VIEW_CONVERT_EXPR)
1357 gcc_assert (code != CONVERT_EXPR);
1359 /* The signedness is determined from input argument. */
1360 if (code == VEC_UNPACK_FLOAT_HI_EXPR
1361 || code == VEC_UNPACK_FLOAT_LO_EXPR)
1362 type = TREE_TYPE (rhs1);
1364 /* For widening/narrowing vector operations, the relevant type is of the
1365 arguments, not the widened result. VEC_UNPACK_FLOAT_*_EXPR is
1366 calculated in the same way above. */
1367 if (code == WIDEN_SUM_EXPR
1368 || code == VEC_WIDEN_MULT_HI_EXPR
1369 || code == VEC_WIDEN_MULT_LO_EXPR
1370 || code == VEC_WIDEN_MULT_EVEN_EXPR
1371 || code == VEC_WIDEN_MULT_ODD_EXPR
1372 || code == VEC_UNPACK_HI_EXPR
1373 || code == VEC_UNPACK_LO_EXPR
1374 || code == VEC_PACK_TRUNC_EXPR
1375 || code == VEC_PACK_SAT_EXPR
1376 || code == VEC_PACK_FIX_TRUNC_EXPR
1377 || code == VEC_WIDEN_LSHIFT_HI_EXPR
1378 || code == VEC_WIDEN_LSHIFT_LO_EXPR)
1379 type = TREE_TYPE (rhs1);
1381 /* Choose between vector shift/rotate by vector and vector shift/rotate by
1383 if (code == LSHIFT_EXPR
1384 || code == RSHIFT_EXPR
1385 || code == LROTATE_EXPR
1386 || code == RROTATE_EXPR)
1390 /* Check whether we have vector <op> {x,x,x,x} where x
1391 could be a scalar variable or a constant. Transform
1392 vector <op> {x,x,x,x} ==> vector <op> scalar. */
1393 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2)))
1398 if ((TREE_CODE (rhs2) == VECTOR_CST
1399 && (first = uniform_vector_p (rhs2)) != NULL_TREE)
1400 || (TREE_CODE (rhs2) == SSA_NAME
1401 && (def_stmt = SSA_NAME_DEF_STMT (rhs2))
1402 && gimple_assign_single_p (def_stmt)
1403 && (first = uniform_vector_p
1404 (gimple_assign_rhs1 (def_stmt))) != NULL_TREE))
1406 gimple_assign_set_rhs2 (stmt, first);
1412 opv = optab_for_tree_code (code, type, optab_vector);
1413 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2)))
1417 op = optab_for_tree_code (code, type, optab_scalar);
1419 /* The rtl expander will expand vector/scalar as vector/vector
1420 if necessary. Don't bother converting the stmt here. */
1421 if (optab_handler (op, TYPE_MODE (type)) == CODE_FOR_nothing
1422 && optab_handler (opv, TYPE_MODE (type)) != CODE_FOR_nothing)
1427 op = optab_for_tree_code (code, type, optab_default);
1429 /* Optabs will try converting a negation into a subtraction, so
1430 look for it as well. TODO: negation of floating-point vectors
1431 might be turned into an exclusive OR toggling the sign bit. */
1433 && code == NEGATE_EXPR
1434 && INTEGRAL_TYPE_P (TREE_TYPE (type)))
1435 op = optab_for_tree_code (MINUS_EXPR, type, optab_default);
1437 /* For very wide vectors, try using a smaller vector mode. */
1438 compute_type = type;
1439 if (!VECTOR_MODE_P (TYPE_MODE (type)) && op)
1441 tree vector_compute_type
1442 = type_for_widest_vector_mode (TREE_TYPE (type), op);
1443 if (vector_compute_type != NULL_TREE
1444 && (TYPE_VECTOR_SUBPARTS (vector_compute_type)
1445 < TYPE_VECTOR_SUBPARTS (compute_type))
1446 && (optab_handler (op, TYPE_MODE (vector_compute_type))
1447 != CODE_FOR_nothing))
1448 compute_type = vector_compute_type;
1451 /* If we are breaking a BLKmode vector into smaller pieces,
1452 type_for_widest_vector_mode has already looked into the optab,
1453 so skip these checks. */
1454 if (compute_type == type)
1456 compute_mode = TYPE_MODE (compute_type);
1457 if (VECTOR_MODE_P (compute_mode)
1459 && optab_handler (op, compute_mode) != CODE_FOR_nothing)
1462 /* There is no operation in hardware, so fall back to scalars. */
1463 compute_type = TREE_TYPE (type);
1466 gcc_assert (code != VEC_LSHIFT_EXPR && code != VEC_RSHIFT_EXPR);
1467 new_rhs = expand_vector_operation (gsi, type, compute_type, stmt, code);
1469 /* Leave expression untouched for later expansion. */
1470 if (new_rhs == NULL_TREE)
1473 if (!useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (new_rhs)))
1474 new_rhs = gimplify_build1 (gsi, VIEW_CONVERT_EXPR, TREE_TYPE (lhs),
1477 /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
1478 way to do it is change expand_vector_operation and its callees to
1479 return a tree_code, RHS1 and RHS2 instead of a tree. */
1480 gimple_assign_set_rhs_from_tree (gsi, new_rhs);
1481 update_stmt (gsi_stmt (*gsi));
1484 /* Use this to lower vector operations introduced by the vectorizer,
1485 if it may need the bit-twiddling tricks implemented in this file. */
1488 gate_expand_vector_operations_ssa (void)
1490 return optimize == 0;
1494 expand_vector_operations (void)
1496 gimple_stmt_iterator gsi;
1498 bool cfg_changed = false;
1502 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1504 expand_vector_operations_1 (&gsi);
1505 /* ??? If we do not cleanup EH then we will ICE in
1506 verification. But in reality we have created wrong-code
1507 as we did not properly transition EH info and edges to
1508 the piecewise computations. */
1509 if (maybe_clean_eh_stmt (gsi_stmt (gsi))
1510 && gimple_purge_dead_eh_edges (bb))
1515 return cfg_changed ? TODO_cleanup_cfg : 0;
1518 struct gimple_opt_pass pass_lower_vector =
1522 "veclower", /* name */
1523 gate_expand_vector_operations_ssa, /* gate */
1524 expand_vector_operations, /* execute */
1527 0, /* static_pass_number */
1528 TV_NONE, /* tv_id */
1529 PROP_cfg, /* properties_required */
1530 0, /* properties_provided */
1531 0, /* properties_destroyed */
1532 0, /* todo_flags_start */
1533 TODO_update_ssa /* todo_flags_finish */
1535 | TODO_verify_stmts | TODO_verify_flow
1540 struct gimple_opt_pass pass_lower_vector_ssa =
1544 "veclower2", /* name */
1546 expand_vector_operations, /* execute */
1549 0, /* static_pass_number */
1550 TV_NONE, /* tv_id */
1551 PROP_cfg, /* properties_required */
1552 0, /* properties_provided */
1553 0, /* properties_destroyed */
1554 0, /* todo_flags_start */
1555 TODO_update_ssa /* todo_flags_finish */
1557 | TODO_verify_stmts | TODO_verify_flow
1562 #include "gt-tree-vect-generic.h"