/* If-conversion for vectorizer.
- Copyright (C) 2004-2016 Free Software Foundation, Inc.
+ Copyright (C) 2004-2018 Free Software Foundation, Inc.
Contributed by Devang Patel <dpatel@apple.com>
This file is part of GCC.
static inline void
add_bb_predicate_gimplified_stmts (basic_block bb, gimple_seq stmts)
{
- gimple_seq_add_seq
+ gimple_seq_add_seq_without_update
(&(((struct bb_predicate *) bb->aux)->predicate_gimplified_stmts), stmts);
}
gimple_seq stmts = bb_predicate_gimplified_stmts (bb);
if (stmts)
{
- gimple_stmt_iterator i;
+ if (flag_checking)
+ for (gimple_stmt_iterator i = gsi_start (stmts);
+ !gsi_end_p (i); gsi_next (&i))
+ gcc_assert (! gimple_use_ops (gsi_stmt (i)));
- for (i = gsi_start (stmts); !gsi_end_p (i); gsi_next (&i))
- free_stmt_operands (cfun, gsi_stmt (i));
set_bb_predicate_gimplified_stmts (bb, NULL);
}
}
{
tree new_name = make_temp_ssa_name (type, NULL, "_ifc_");
gimple *stmt = gimple_build_assign (new_name, expr);
+ gimple_set_vuse (stmt, gimple_vuse (gsi_stmt (*gsi)));
gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
return new_name;
}
/* Mask should be integer mode of the same size as the load/store
mode. */
mode = TYPE_MODE (TREE_TYPE (lhs));
- if (int_mode_for_mode (mode) == BLKmode
- || VECTOR_MODE_P (mode))
+ if (!int_mode_for_mode (mode).exists () || VECTOR_MODE_P (mode))
return false;
if (can_vec_mask_load_store_p (mode, VOIDmode, is_load))
print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
}
return false;
- break;
}
return true;
&& bb_predicate_gimplified_stmts (loop->latch) == NULL);
}
+/* Build region by adding loop pre-header and post-header blocks. */
+
+static vec<basic_block>
+build_region (struct loop *loop)
+{
+ vec<basic_block> region = vNULL;
+ basic_block exit_bb = NULL;
+
+ gcc_assert (ifc_bbs);
+ /* The first element is loop pre-header. */
+ region.safe_push (loop_preheader_edge (loop)->src);
+
+ for (unsigned int i = 0; i < loop->num_nodes; i++)
+ {
+ basic_block bb = ifc_bbs[i];
+ region.safe_push (bb);
+ /* Find loop postheader. */
+ edge e;
+ edge_iterator ei;
+ FOR_EACH_EDGE (e, ei, bb->succs)
+ if (loop_exit_edge_p (loop, e))
+ {
+ exit_bb = e->dest;
+ break;
+ }
+ }
+ /* The last element is loop post-header. */
+ gcc_assert (exit_bb);
+ region.safe_push (exit_bb);
+ return region;
+}
+
/* Return true when LOOP is if-convertible. This is a helper function
for if_convertible_loop_p. REFS and DDRS are initialized and freed
in if_convertible_loop_p. */
{
unsigned int i;
basic_block exit_bb = NULL;
+ vec<basic_block> region;
if (find_data_references_in_loop (loop, refs) == chrec_dont_know)
return false;
calculate_dominance_info (CDI_DOMINATORS);
- calculate_dominance_info (CDI_POST_DOMINATORS);
/* Allow statements that can be handled during if-conversion. */
ifc_bbs = get_loop_body_in_if_conv_order (loop);
= new hash_map<innermost_loop_behavior_hash, data_reference_p>;
baseref_DR_map = new hash_map<tree_operand_hash, data_reference_p>;
+ /* Compute post-dominator tree locally. */
+ region = build_region (loop);
+ calculate_dominance_info_for_region (CDI_POST_DOMINATORS, region);
+
predicate_bbs (loop);
+ /* Free post-dominator tree since it is not used after predication. */
+ free_dominance_info_for_region (cfun, CDI_POST_DOMINATORS, region);
+ region.release ();
+
for (i = 0; refs->iterate (i, &dr); i++)
{
tree ref = DR_REF (dr);
|| TREE_CODE (ref) == REALPART_EXPR)
ref = TREE_OPERAND (ref, 0);
- DR_BASE_ADDRESS (dr) = ref;
- DR_OFFSET (dr) = NULL;
- DR_INIT (dr) = NULL;
- DR_STEP (dr) = NULL;
- DR_ALIGNED_TO (dr) = NULL;
+ memset (&DR_INNERMOST (dr), 0, sizeof (DR_INNERMOST (dr)));
+ DR_BASE_ADDRESS (dr) = ref;
}
hash_memrefs_baserefs_and_store_DRs_read_written_info (dr);
}
return false;
}
- for (i = 0; i < loop->num_nodes; i++)
- free_bb_predicate (ifc_bbs[i]);
-
/* Checking PHIs needs to be done after stmts, as the fact whether there
are any masked loads or stores affects the tests. */
for (i = 0; i < loop->num_nodes; i++)
e = gimple_phi_arg_edge (phi, (*occur)[i]);
c = bb_predicate (e->src);
if (is_true_predicate (c))
- continue;
+ {
+ cond = c;
+ break;
+ }
c = force_gimple_operand_gsi_1 (gsi, unshare_expr (c),
is_gimple_condexpr, NULL_TREE,
true, GSI_SAME_STMT);
return cond;
}
+/* Local valueization callback that follows all-use SSA edges. */
+
+static tree
+ifcvt_follow_ssa_use_edges (tree val)
+{
+ return val;
+}
+
/* Replace a scalar PHI node with a COND_EXPR using COND as condition.
This routine can handle PHI nodes with more than two arguments.
arg0, arg1);
new_stmt = gimple_build_assign (res, rhs);
gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
- update_stmt (new_stmt);
+ gimple_stmt_iterator new_gsi = gsi_for_stmt (new_stmt);
+ if (fold_stmt (&new_gsi, ifcvt_follow_ssa_use_edges))
+ {
+ new_stmt = gsi_stmt (new_gsi);
+ update_stmt (new_stmt);
+ }
if (dump_file && (dump_flags & TDF_DETAILS))
{
while (!gsi_end_p (phi_gsi))
{
phi = phi_gsi.phi ();
- predicate_scalar_phi (phi, &gsi);
- release_phi_node (phi);
- gsi_next (&phi_gsi);
+ if (virtual_operand_p (gimple_phi_result (phi)))
+ gsi_next (&phi_gsi);
+ else
+ {
+ predicate_scalar_phi (phi, &gsi);
+ remove_phi_node (&phi_gsi, false);
+ }
}
-
- set_phi_nodes (bb, NULL);
}
}
gimple *stmt;
int index;
- if (is_true_predicate (cond) || is_false_predicate (cond))
+ if (is_true_predicate (cond))
continue;
swap = false;
vect_sizes.truncate (0);
vect_masks.truncate (0);
- for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
- if (!gimple_assign_single_p (stmt = gsi_stmt (gsi)))
- continue;
- else if (gimple_plf (stmt, GF_PLF_2))
- {
- tree lhs = gimple_assign_lhs (stmt);
- tree rhs = gimple_assign_rhs1 (stmt);
- tree ref, addr, ptr, mask;
- gimple *new_stmt;
- gimple_seq stmts = NULL;
- int bitsize = GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (lhs)));
- ref = TREE_CODE (lhs) == SSA_NAME ? rhs : lhs;
- mark_addressable (ref);
- addr = force_gimple_operand_gsi (&gsi, build_fold_addr_expr (ref),
- true, NULL_TREE, true,
- GSI_SAME_STMT);
- if (!vect_sizes.is_empty ()
- && (index = mask_exists (bitsize, vect_sizes)) != -1)
- /* Use created mask. */
- mask = vect_masks[index];
- else
- {
- if (COMPARISON_CLASS_P (cond))
- mask = gimple_build (&stmts, TREE_CODE (cond),
- boolean_type_node,
- TREE_OPERAND (cond, 0),
- TREE_OPERAND (cond, 1));
- else
- {
- gcc_assert (TREE_CODE (cond) == SSA_NAME);
+ for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);)
+ {
+ if (!gimple_assign_single_p (stmt = gsi_stmt (gsi)))
+ ;
+ else if (is_false_predicate (cond)
+ && gimple_vdef (stmt))
+ {
+ unlink_stmt_vdef (stmt);
+ gsi_remove (&gsi, true);
+ release_defs (stmt);
+ continue;
+ }
+ else if (gimple_plf (stmt, GF_PLF_2))
+ {
+ tree lhs = gimple_assign_lhs (stmt);
+ tree rhs = gimple_assign_rhs1 (stmt);
+ tree ref, addr, ptr, mask;
+ gcall *new_stmt;
+ gimple_seq stmts = NULL;
+ machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
+ /* We checked before setting GF_PLF_2 that an equivalent
+ integer mode exists. */
+ int bitsize = GET_MODE_BITSIZE (mode).to_constant ();
+ ref = TREE_CODE (lhs) == SSA_NAME ? rhs : lhs;
+ mark_addressable (ref);
+ addr = force_gimple_operand_gsi (&gsi, build_fold_addr_expr (ref),
+ true, NULL_TREE, true,
+ GSI_SAME_STMT);
+ if (!vect_sizes.is_empty ()
+ && (index = mask_exists (bitsize, vect_sizes)) != -1)
+ /* Use created mask. */
+ mask = vect_masks[index];
+ else
+ {
+ if (COMPARISON_CLASS_P (cond))
+ mask = gimple_build (&stmts, TREE_CODE (cond),
+ boolean_type_node,
+ TREE_OPERAND (cond, 0),
+ TREE_OPERAND (cond, 1));
+ else
mask = cond;
- }
-
- if (swap)
- {
- tree true_val
- = constant_boolean_node (true, TREE_TYPE (mask));
- mask = gimple_build (&stmts, BIT_XOR_EXPR,
- TREE_TYPE (mask), mask, true_val);
- }
- gsi_insert_seq_before (&gsi, stmts, GSI_SAME_STMT);
- mask = ifc_temp_var (TREE_TYPE (mask), mask, &gsi);
- /* Save mask and its size for further use. */
- vect_sizes.safe_push (bitsize);
- vect_masks.safe_push (mask);
- }
- ptr = build_int_cst (reference_alias_ptr_type (ref),
- get_object_alignment (ref));
- /* Copy points-to info if possible. */
- if (TREE_CODE (addr) == SSA_NAME && !SSA_NAME_PTR_INFO (addr))
- copy_ref_info (build2 (MEM_REF, TREE_TYPE (ref), addr, ptr),
- ref);
- if (TREE_CODE (lhs) == SSA_NAME)
- {
- new_stmt
- = gimple_build_call_internal (IFN_MASK_LOAD, 3, addr,
- ptr, mask);
- gimple_call_set_lhs (new_stmt, lhs);
- }
- else
- new_stmt
- = gimple_build_call_internal (IFN_MASK_STORE, 4, addr, ptr,
- mask, rhs);
- gsi_replace (&gsi, new_stmt, true);
- }
- else if (gimple_vdef (stmt))
- {
- tree lhs = gimple_assign_lhs (stmt);
- tree rhs = gimple_assign_rhs1 (stmt);
- tree type = TREE_TYPE (lhs);
-
- lhs = ifc_temp_var (type, unshare_expr (lhs), &gsi);
- rhs = ifc_temp_var (type, unshare_expr (rhs), &gsi);
- if (swap)
- std::swap (lhs, rhs);
- cond = force_gimple_operand_gsi_1 (&gsi, unshare_expr (cond),
- is_gimple_condexpr, NULL_TREE,
- true, GSI_SAME_STMT);
- rhs = fold_build_cond_expr (type, unshare_expr (cond), rhs, lhs);
- gimple_assign_set_rhs1 (stmt, ifc_temp_var (type, rhs, &gsi));
- update_stmt (stmt);
- }
+ if (swap)
+ {
+ tree true_val
+ = constant_boolean_node (true, TREE_TYPE (mask));
+ mask = gimple_build (&stmts, BIT_XOR_EXPR,
+ TREE_TYPE (mask), mask, true_val);
+ }
+ gsi_insert_seq_before (&gsi, stmts, GSI_SAME_STMT);
+
+ mask = ifc_temp_var (TREE_TYPE (mask), mask, &gsi);
+ /* Save mask and its size for further use. */
+ vect_sizes.safe_push (bitsize);
+ vect_masks.safe_push (mask);
+ }
+ ptr = build_int_cst (reference_alias_ptr_type (ref),
+ get_object_alignment (ref));
+ /* Copy points-to info if possible. */
+ if (TREE_CODE (addr) == SSA_NAME && !SSA_NAME_PTR_INFO (addr))
+ copy_ref_info (build2 (MEM_REF, TREE_TYPE (ref), addr, ptr),
+ ref);
+ if (TREE_CODE (lhs) == SSA_NAME)
+ {
+ new_stmt
+ = gimple_build_call_internal (IFN_MASK_LOAD, 3, addr,
+ ptr, mask);
+ gimple_call_set_lhs (new_stmt, lhs);
+ gimple_set_vuse (new_stmt, gimple_vuse (stmt));
+ }
+ else
+ {
+ new_stmt
+ = gimple_build_call_internal (IFN_MASK_STORE, 4, addr, ptr,
+ mask, rhs);
+ gimple_set_vuse (new_stmt, gimple_vuse (stmt));
+ gimple_set_vdef (new_stmt, gimple_vdef (stmt));
+ SSA_NAME_DEF_STMT (gimple_vdef (new_stmt)) = new_stmt;
+ }
+ gimple_call_set_nothrow (new_stmt, true);
+
+ gsi_replace (&gsi, new_stmt, true);
+ }
+ else if (gimple_vdef (stmt))
+ {
+ tree lhs = gimple_assign_lhs (stmt);
+ tree rhs = gimple_assign_rhs1 (stmt);
+ tree type = TREE_TYPE (lhs);
+
+ lhs = ifc_temp_var (type, unshare_expr (lhs), &gsi);
+ rhs = ifc_temp_var (type, unshare_expr (rhs), &gsi);
+ if (swap)
+ std::swap (lhs, rhs);
+ cond = force_gimple_operand_gsi_1 (&gsi, unshare_expr (cond),
+ is_gimple_condexpr, NULL_TREE,
+ true, GSI_SAME_STMT);
+ rhs = fold_build_cond_expr (type, unshare_expr (cond), rhs, lhs);
+ gimple_assign_set_rhs1 (stmt, ifc_temp_var (type, rhs, &gsi));
+ update_stmt (stmt);
+ }
+ gsi_next (&gsi);
+ }
}
}
edge e;
edge_iterator ei;
- predicate_bbs (loop);
remove_conditions_and_labels (loop);
insert_gimplified_predicates (loop);
predicate_all_scalar_phis (loop);
if (exit_bb != loop->header)
{
/* Connect this node to loop header. */
- make_edge (loop->header, exit_bb, EDGE_FALLTHRU);
+ make_single_succ_edge (loop->header, exit_bb, EDGE_FALLTHRU);
set_immediate_dominator (CDI_DOMINATORS, exit_bb, loop->header);
}
}
merge_target_bb = loop->header;
+
+ /* Get at the virtual def valid for uses starting at the first block
+ we merge into the header. Without a virtual PHI the loop has the
+ same virtual use on all stmts. */
+ gphi *vphi = get_virtual_phi (loop->header);
+ tree last_vdef = NULL_TREE;
+ if (vphi)
+ {
+ last_vdef = gimple_phi_result (vphi);
+ for (gimple_stmt_iterator gsi = gsi_start_bb (loop->header);
+ ! gsi_end_p (gsi); gsi_next (&gsi))
+ if (gimple_vdef (gsi_stmt (gsi)))
+ last_vdef = gimple_vdef (gsi_stmt (gsi));
+ }
for (i = 1; i < orig_loop_num_nodes; i++)
{
gimple_stmt_iterator gsi;
if (bb == exit_bb || bb == loop->latch)
continue;
+ /* We release virtual PHIs late because we have to propagate them
+ out using the current VUSE. The def might be the one used
+ after the loop. */
+ vphi = get_virtual_phi (bb);
+ if (vphi)
+ {
+ imm_use_iterator iter;
+ use_operand_p use_p;
+ gimple *use_stmt;
+ FOR_EACH_IMM_USE_STMT (use_stmt, iter, gimple_phi_result (vphi))
+ {
+ FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
+ SET_USE (use_p, last_vdef);
+ }
+ gsi = gsi_for_stmt (vphi);
+ remove_phi_node (&gsi, true);
+ }
+
/* Make stmts member of loop->header and clear range info from all stmts
in BB which is now no longer executed conditional on a predicate we
could have derived it from. */
{
gimple *stmt = gsi_stmt (gsi);
gimple_set_bb (stmt, merge_target_bb);
+ /* Update virtual operands. */
+ if (last_vdef)
+ {
+ use_operand_p use_p = ssa_vuse_operand (stmt);
+ if (use_p
+ && USE_FROM_PTR (use_p) != last_vdef)
+ SET_USE (use_p, last_vdef);
+ if (gimple_vdef (stmt))
+ last_vdef = gimple_vdef (stmt);
+ }
if (predicated[i])
{
ssa_op_iter i;
/* Update stmt list. */
last = gsi_last_bb (merge_target_bb);
- gsi_insert_seq_after (&last, bb_seq (bb), GSI_NEW_STMT);
+ gsi_insert_seq_after_without_update (&last, bb_seq (bb), GSI_NEW_STMT);
set_bb_seq (bb, NULL);
delete_basic_block (bb);
This reduces the number of basic blocks to two, to please the
vectorizer that handles only loops with two nodes. */
if (exit_bb
- && exit_bb != loop->header
- && can_merge_blocks_p (loop->header, exit_bb))
- merge_blocks (loop->header, exit_bb);
+ && exit_bb != loop->header)
+ {
+ /* We release virtual PHIs late because we have to propagate them
+ out using the current VUSE. The def might be the one used
+ after the loop. */
+ vphi = get_virtual_phi (exit_bb);
+ if (vphi)
+ {
+ imm_use_iterator iter;
+ use_operand_p use_p;
+ gimple *use_stmt;
+ FOR_EACH_IMM_USE_STMT (use_stmt, iter, gimple_phi_result (vphi))
+ {
+ FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
+ SET_USE (use_p, last_vdef);
+ }
+ gimple_stmt_iterator gsi = gsi_for_stmt (vphi);
+ remove_phi_node (&gsi, true);
+ }
+
+ if (can_merge_blocks_p (loop->header, exit_bb))
+ merge_blocks (loop->header, exit_bb);
+ }
free (ifc_bbs);
ifc_bbs = NULL;
will be if-converted, the new copy of the loop will not,
and the LOOP_VECTORIZED internal call will be guarding which
loop to execute. The vectorizer pass will fold this
- internal call into either true or false. */
+ internal call into either true or false.
-static bool
+ Note that this function intentionally invalidates profile. Both edges
+ out of LOOP_VECTORIZED must have 100% probability so the profile remains
+ consistent after the condition is folded in the vectorizer. */
+
+static struct loop *
version_loop_for_if_conversion (struct loop *loop)
{
basic_block cond_bb;
struct loop *new_loop;
gimple *g;
gimple_stmt_iterator gsi;
+ unsigned int save_length;
g = gimple_build_call_internal (IFN_LOOP_VECTORIZED, 2,
build_int_cst (integer_type_node, loop->num),
integer_zero_node);
gimple_call_set_lhs (g, cond);
+ /* Save BB->aux around loop_version as that uses the same field. */
+ save_length = loop->inner ? loop->inner->num_nodes : loop->num_nodes;
+ void **saved_preds = XALLOCAVEC (void *, save_length);
+ for (unsigned i = 0; i < save_length; i++)
+ saved_preds[i] = ifc_bbs[i]->aux;
+
initialize_original_copy_tables ();
+ /* At this point we invalidate porfile confistency until IFN_LOOP_VECTORIZED
+ is re-merged in the vectorizer. */
new_loop = loop_version (loop, cond, &cond_bb,
- REG_BR_PROB_BASE, REG_BR_PROB_BASE,
- REG_BR_PROB_BASE, true);
+ profile_probability::always (),
+ profile_probability::always (),
+ profile_probability::always (),
+ profile_probability::always (), true);
free_original_copy_tables ();
+
+ for (unsigned i = 0; i < save_length; i++)
+ ifc_bbs[i]->aux = saved_preds[i];
+
if (new_loop == NULL)
- return false;
+ return NULL;
+
new_loop->dont_vectorize = true;
new_loop->force_vectorize = false;
gsi = gsi_last_bb (cond_bb);
gimple_call_set_arg (g, 1, build_int_cst (integer_type_node, new_loop->num));
gsi_insert_before (&gsi, g, GSI_SAME_STMT);
update_ssa (TODO_update_ssa);
+ return new_loop;
+}
+
+/* Return true when LOOP satisfies the follow conditions that will
+ allow it to be recognized by the vectorizer for outer-loop
+ vectorization:
+ - The loop is not the root node of the loop tree.
+ - The loop has exactly one inner loop.
+ - The loop has a single exit.
+ - The loop header has a single successor, which is the inner
+ loop header.
+ - Each of the inner and outer loop latches have a single
+ predecessor.
+ - The loop exit block has a single predecessor, which is the
+ inner loop's exit block. */
+
+static bool
+versionable_outer_loop_p (struct loop *loop)
+{
+ if (!loop_outer (loop)
+ || loop->dont_vectorize
+ || !loop->inner
+ || loop->inner->next
+ || !single_exit (loop)
+ || !single_succ_p (loop->header)
+ || single_succ (loop->header) != loop->inner->header
+ || !single_pred_p (loop->latch)
+ || !single_pred_p (loop->inner->latch))
+ return false;
+
+ basic_block outer_exit = single_pred (loop->latch);
+ basic_block inner_exit = single_pred (loop->inner->latch);
+
+ if (!single_pred_p (outer_exit) || single_pred (outer_exit) != inner_exit)
+ return false;
+
+ if (dump_file)
+ fprintf (dump_file, "Found vectorizable outer loop for versioning\n");
+
return true;
}
return true;
}
-/* Assumes that lhs of DEF_STMT have multiple uses.
- Delete one use by (1) creation of copy DEF_STMT with
- unique lhs; (2) change original use of lhs in one
- use statement with newly created lhs. */
-
-static void
-ifcvt_split_def_stmt (gimple *def_stmt, gimple *use_stmt)
-{
- tree var;
- tree lhs;
- gimple *copy_stmt;
- gimple_stmt_iterator gsi;
- use_operand_p use_p;
- imm_use_iterator imm_iter;
-
- var = gimple_assign_lhs (def_stmt);
- copy_stmt = gimple_copy (def_stmt);
- lhs = make_temp_ssa_name (TREE_TYPE (var), NULL, "_ifc_");
- gimple_assign_set_lhs (copy_stmt, lhs);
- SSA_NAME_DEF_STMT (lhs) = copy_stmt;
- /* Insert copy of DEF_STMT. */
- gsi = gsi_for_stmt (def_stmt);
- gsi_insert_after (&gsi, copy_stmt, GSI_SAME_STMT);
- /* Change use of var to lhs in use_stmt. */
- if (dump_file && (dump_flags & TDF_DETAILS))
- {
- fprintf (dump_file, "Change use of var ");
- print_generic_expr (dump_file, var, TDF_SLIM);
- fprintf (dump_file, " to ");
- print_generic_expr (dump_file, lhs, TDF_SLIM);
- fprintf (dump_file, "\n");
- }
- FOR_EACH_IMM_USE_FAST (use_p, imm_iter, var)
- {
- if (USE_STMT (use_p) != use_stmt)
- continue;
- SET_USE (use_p, lhs);
- break;
- }
-}
-
-/* Traverse bool pattern recursively starting from VAR.
- Save its def and use statements to defuse_list if VAR does
- not have single use. */
-
-static void
-ifcvt_walk_pattern_tree (tree var, vec<gimple *> *defuse_list,
- gimple *use_stmt)
-{
- tree rhs1, rhs2;
- enum tree_code code;
- gimple *def_stmt;
-
- if (TREE_CODE (var) != SSA_NAME)
- return;
-
- def_stmt = SSA_NAME_DEF_STMT (var);
- if (gimple_code (def_stmt) != GIMPLE_ASSIGN)
- return;
- if (!has_single_use (var))
- {
- /* Put def and use stmts into defuse_list. */
- defuse_list->safe_push (def_stmt);
- defuse_list->safe_push (use_stmt);
- if (dump_file && (dump_flags & TDF_DETAILS))
- {
- fprintf (dump_file, "Multiple lhs uses in stmt\n");
- print_gimple_stmt (dump_file, def_stmt, 0, TDF_SLIM);
- }
- }
- rhs1 = gimple_assign_rhs1 (def_stmt);
- code = gimple_assign_rhs_code (def_stmt);
- switch (code)
- {
- case SSA_NAME:
- ifcvt_walk_pattern_tree (rhs1, defuse_list, def_stmt);
- break;
- CASE_CONVERT:
- if ((TYPE_PRECISION (TREE_TYPE (rhs1)) != 1
- || !TYPE_UNSIGNED (TREE_TYPE (rhs1)))
- && TREE_CODE (TREE_TYPE (rhs1)) != BOOLEAN_TYPE)
- break;
- ifcvt_walk_pattern_tree (rhs1, defuse_list, def_stmt);
- break;
- case BIT_NOT_EXPR:
- ifcvt_walk_pattern_tree (rhs1, defuse_list, def_stmt);
- break;
- case BIT_AND_EXPR:
- case BIT_IOR_EXPR:
- case BIT_XOR_EXPR:
- ifcvt_walk_pattern_tree (rhs1, defuse_list, def_stmt);
- rhs2 = gimple_assign_rhs2 (def_stmt);
- ifcvt_walk_pattern_tree (rhs2, defuse_list, def_stmt);
- break;
- default:
- break;
- }
- return;
-}
-
-/* Returns true if STMT can be a root of bool pattern applied
- by vectorizer. */
-
-static bool
-stmt_is_root_of_bool_pattern (gimple *stmt)
-{
- enum tree_code code;
- tree lhs, rhs;
-
- code = gimple_assign_rhs_code (stmt);
- if (CONVERT_EXPR_CODE_P (code))
- {
- lhs = gimple_assign_lhs (stmt);
- rhs = gimple_assign_rhs1 (stmt);
- if (TREE_CODE (TREE_TYPE (rhs)) != BOOLEAN_TYPE)
- return false;
- if (TREE_CODE (TREE_TYPE (lhs)) == BOOLEAN_TYPE)
- return false;
- return true;
- }
- else if (code == COND_EXPR)
- {
- rhs = gimple_assign_rhs1 (stmt);
- if (TREE_CODE (rhs) != SSA_NAME)
- return false;
- return true;
- }
- return false;
-}
-
-/* Traverse all statements in BB which correspond to loop header to
- find out all statements which can start bool pattern applied by
- vectorizer and convert multiple uses in it to conform pattern
- restrictions. Such case can occur if the same predicate is used both
- for phi node conversion and load/store mask. */
-
-static void
-ifcvt_repair_bool_pattern (basic_block bb)
-{
- tree rhs;
- gimple *stmt;
- gimple_stmt_iterator gsi;
- vec<gimple *> defuse_list = vNULL;
- vec<gimple *> pattern_roots = vNULL;
- bool repeat = true;
- int niter = 0;
- unsigned int ix;
-
- /* Collect all root pattern statements. */
- for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
- {
- stmt = gsi_stmt (gsi);
- if (gimple_code (stmt) != GIMPLE_ASSIGN)
- continue;
- if (!stmt_is_root_of_bool_pattern (stmt))
- continue;
- pattern_roots.safe_push (stmt);
- }
-
- if (pattern_roots.is_empty ())
- return;
-
- /* Split all statements with multiple uses iteratively since splitting
- may create new multiple uses. */
- while (repeat)
- {
- repeat = false;
- niter++;
- FOR_EACH_VEC_ELT (pattern_roots, ix, stmt)
- {
- rhs = gimple_assign_rhs1 (stmt);
- ifcvt_walk_pattern_tree (rhs, &defuse_list, stmt);
- while (defuse_list.length () > 0)
- {
- repeat = true;
- gimple *def_stmt, *use_stmt;
- use_stmt = defuse_list.pop ();
- def_stmt = defuse_list.pop ();
- ifcvt_split_def_stmt (def_stmt, use_stmt);
- }
-
- }
- }
- if (dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file, "Repair bool pattern takes %d iterations. \n",
- niter);
-}
-
/* Delete redundant statements produced by predication which prevents
loop vectorization. */
profitability analysis. Returns non-zero todo flags when something
changed. */
-static unsigned int
+unsigned int
tree_if_conversion (struct loop *loop)
{
unsigned int todo = 0;
bool aggressive_if_conv;
+ struct loop *rloop;
+ again:
+ rloop = NULL;
ifc_bbs = NULL;
any_pred_load_store = false;
any_complicated_phi = false;
|| loop->dont_vectorize))
goto cleanup;
- if ((any_pred_load_store || any_complicated_phi)
- && !version_loop_for_if_conversion (loop))
- goto cleanup;
+ /* Since we have no cost model, always version loops unless the user
+ specified -ftree-loop-if-convert or unless versioning is required.
+ Either version this loop, or if the pattern is right for outer-loop
+ vectorization, version the outer loop. In the latter case we will
+ still if-convert the original inner loop. */
+ if (any_pred_load_store
+ || any_complicated_phi
+ || flag_tree_loop_if_convert != 1)
+ {
+ struct loop *vloop
+ = (versionable_outer_loop_p (loop_outer (loop))
+ ? loop_outer (loop) : loop);
+ struct loop *nloop = version_loop_for_if_conversion (vloop);
+ if (nloop == NULL)
+ goto cleanup;
+ if (vloop != loop)
+ {
+ /* If versionable_outer_loop_p decided to version the
+ outer loop, version also the inner loop of the non-vectorized
+ loop copy. So we transform:
+ loop1
+ loop2
+ into:
+ if (LOOP_VECTORIZED (1, 3))
+ {
+ loop1
+ loop2
+ }
+ else
+ loop3 (copy of loop1)
+ if (LOOP_VECTORIZED (4, 5))
+ loop4 (copy of loop2)
+ else
+ loop5 (copy of loop4) */
+ gcc_assert (nloop->inner && nloop->inner->next == NULL);
+ rloop = nloop->inner;
+ }
+ }
/* Now all statements are if-convertible. Combine all the basic
blocks into one huge basic block doing the if-conversion
on-the-fly. */
combine_blocks (loop);
- /* Delete dead predicate computations and repair tree correspondent
- to bool pattern to delete multiple uses of predicates. */
+ /* Delete dead predicate computations. */
ifcvt_local_dce (loop->header);
- ifcvt_repair_bool_pattern (loop->header);
todo |= TODO_cleanup_cfg;
- mark_virtual_operands_for_renaming (cfun);
- todo |= TODO_update_ssa_only_virtuals;
cleanup:
if (ifc_bbs)
free (ifc_bbs);
ifc_bbs = NULL;
}
- free_dominance_info (CDI_POST_DOMINATORS);
+ if (rloop != NULL)
+ {
+ loop = rloop;
+ goto again;
+ }
return todo;
}
GIMPLE_PASS, /* type */
"ifcvt", /* name */
OPTGROUP_NONE, /* optinfo_flags */
- TV_NONE, /* tv_id */
+ TV_TREE_LOOP_IFCVT, /* tv_id */
( PROP_cfg | PROP_ssa ), /* properties_required */
0, /* properties_provided */
0, /* properties_destroyed */
{
return (((flag_tree_loop_vectorize || fun->has_force_vectorize_loops)
&& flag_tree_loop_if_convert != 0)
- || flag_tree_loop_if_convert == 1
- || flag_tree_loop_if_convert_stores == 1);
+ || flag_tree_loop_if_convert == 1);
}
unsigned int
if (number_of_loops (fun) <= 1)
return 0;
- /* If there are infinite loops, during CDI_POST_DOMINATORS computation
- we can pick pretty much random bb inside of the infinite loop that
- has the fake edge. If we are unlucky enough, this can confuse the
- add_to_predicate_list post-dominator check to optimize as if that
- bb or some other one is a join block when it actually is not.
- See PR70916. */
- connect_infinite_loops_to_exit ();
-
FOR_EACH_LOOP (loop, 0)
if (flag_tree_loop_if_convert == 1
- || flag_tree_loop_if_convert_stores == 1
|| ((flag_tree_loop_vectorize || loop->force_vectorize)
&& !loop->dont_vectorize))
todo |= tree_if_conversion (loop);
- remove_fake_exit_edges ();
+ if (todo)
+ {
+ free_numbers_of_iterations_estimates (fun);
+ scev_reset ();
+ }
if (flag_checking)
{
projects / platform / upstream / gcc.git / blobdiff