return false;
}
}
- if (loop_vinfo
- && LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo)
- && reduction_type == EXTRACT_LAST_REDUCTION)
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "can't yet use a fully-masked loop for"
- " EXTRACT_LAST_REDUCTION.\n");
- LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo) = false;
- }
if (expand_vec_cond_expr_p (vectype, comp_vectype,
cond_code))
{
/* Handle cond expr. */
for (j = 0; j < ncopies; j++)
{
- tree loop_mask = NULL_TREE;
bool swap_cond_operands = false;
/* See whether another part of the vectorized code applies a loop
mask to the condition, or to its inverse. */
+ vec_loop_masks *masks = NULL;
if (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
{
- scalar_cond_masked_key cond (cond_expr, ncopies);
- if (loop_vinfo->scalar_cond_masked_set.contains (cond))
- {
- vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
- loop_mask = vect_get_loop_mask (gsi, masks, ncopies, vectype, j);
- }
+ if (reduction_type == EXTRACT_LAST_REDUCTION)
+ masks = &LOOP_VINFO_MASKS (loop_vinfo);
else
{
- bool honor_nans = HONOR_NANS (TREE_TYPE (cond.op0));
- cond.code = invert_tree_comparison (cond.code, honor_nans);
+ scalar_cond_masked_key cond (cond_expr, ncopies);
if (loop_vinfo->scalar_cond_masked_set.contains (cond))
+ masks = &LOOP_VINFO_MASKS (loop_vinfo);
+ else
{
- vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
- loop_mask = vect_get_loop_mask (gsi, masks, ncopies,
- vectype, j);
- cond_code = cond.code;
- swap_cond_operands = true;
+ bool honor_nans = HONOR_NANS (TREE_TYPE (cond.op0));
+ cond.code = invert_tree_comparison (cond.code, honor_nans);
+ if (loop_vinfo->scalar_cond_masked_set.contains (cond))
+ {
+ masks = &LOOP_VINFO_MASKS (loop_vinfo);
+ cond_code = cond.code;
+ swap_cond_operands = true;
+ }
}
}
}
vec != { 0, ... } (masked in the MASK_LOAD,
unmasked in the VEC_COND_EXPR). */
- if (loop_mask)
- {
- if (COMPARISON_CLASS_P (vec_compare))
- {
- tree tmp = make_ssa_name (vec_cmp_type);
- tree op0 = TREE_OPERAND (vec_compare, 0);
- tree op1 = TREE_OPERAND (vec_compare, 1);
- gassign *g = gimple_build_assign (tmp,
- TREE_CODE (vec_compare),
- op0, op1);
- vect_finish_stmt_generation (stmt_info, g, gsi);
- vec_compare = tmp;
- }
-
- tree tmp2 = make_ssa_name (vec_cmp_type);
- gassign *g = gimple_build_assign (tmp2, BIT_AND_EXPR,
- vec_compare, loop_mask);
- vect_finish_stmt_generation (stmt_info, g, gsi);
- vec_compare = tmp2;
- }
+ /* Force vec_compare to be an SSA_NAME rather than a comparison,
+ in cases where that's necessary. */
- if (reduction_type == EXTRACT_LAST_REDUCTION)
+ if (masks || reduction_type == EXTRACT_LAST_REDUCTION)
{
if (!is_gimple_val (vec_compare))
{
vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
vec_compare = vec_compare_name;
}
+
if (must_invert_cmp_result)
{
tree vec_compare_name = make_ssa_name (vec_cmp_type);
vect_finish_stmt_generation (stmt_info, new_stmt, gsi);
vec_compare = vec_compare_name;
}
+
+ if (masks)
+ {
+ unsigned vec_num = vec_oprnds0.length ();
+ tree loop_mask
+ = vect_get_loop_mask (gsi, masks, vec_num * ncopies,
+ vectype, vec_num * j + i);
+ tree tmp2 = make_ssa_name (vec_cmp_type);
+ gassign *g
+ = gimple_build_assign (tmp2, BIT_AND_EXPR, vec_compare,
+ loop_mask);
+ vect_finish_stmt_generation (stmt_info, g, gsi);
+ vec_compare = tmp2;
+ }
+ }
+
+ if (reduction_type == EXTRACT_LAST_REDUCTION)
+ {
gcall *new_stmt = gimple_build_call_internal
(IFN_FOLD_EXTRACT_LAST, 3, else_clause, vec_compare,
vec_then_clause);