cond_expr = expr;
}
+ tree cost_name = NULL_TREE;
+ profile_probability prob2 = profile_probability::uninitialized ();
+ if (cond_expr
+ && !integer_truep (cond_expr)
+ && (version_niter
+ || version_align
+ || version_alias
+ || version_simd_if_cond))
+ {
+ cost_name = cond_expr = force_gimple_operand_1 (unshare_expr (cond_expr),
+ &cond_expr_stmt_list,
+ is_gimple_val, NULL_TREE);
+ /* Split prob () into two so that the overall probability of passing
+ both the cost-model and versioning checks is the orig prob. */
+ prob2 = prob.split (prob);
+ }
+
if (version_niter)
vect_create_cond_for_niters_checks (loop_vinfo, &cond_expr);
if (cond_expr)
- cond_expr = force_gimple_operand_1 (unshare_expr (cond_expr),
- &cond_expr_stmt_list,
- is_gimple_condexpr, NULL_TREE);
+ {
+ gimple_seq tem = NULL;
+ cond_expr = force_gimple_operand_1 (unshare_expr (cond_expr),
+ &tem,
+ is_gimple_condexpr, NULL_TREE);
+ gimple_seq_add_seq (&cond_expr_stmt_list, tem);
+ }
if (version_align)
vect_create_cond_for_align_checks (loop_vinfo, &cond_expr,
update_ssa (TODO_update_ssa);
}
+ /* Split the cost model check off to a separate BB. Costing assumes
+ this is the only thing we perform when we enter the scalar loop
+ from a failed cost decision. */
+ if (cost_name && TREE_CODE (cost_name) == SSA_NAME)
+ {
+ gimple *def = SSA_NAME_DEF_STMT (cost_name);
+ /* All uses of the cost check are 'true' after the check we
+ are going to insert. */
+ replace_uses_by (cost_name, boolean_true_node);
+ /* And we're going to build the new single use of it. */
+ gcond *cond = gimple_build_cond (NE_EXPR, cost_name, boolean_false_node,
+ NULL_TREE, NULL_TREE);
+ edge e = split_block (gimple_bb (def), def);
+ gimple_stmt_iterator gsi = gsi_for_stmt (def);
+ gsi_insert_after (&gsi, cond, GSI_NEW_STMT);
+ edge true_e, false_e;
+ extract_true_false_edges_from_block (e->dest, &true_e, &false_e);
+ e->flags &= ~EDGE_FALLTHRU;
+ e->flags |= EDGE_TRUE_VALUE;
+ edge e2 = make_edge (e->src, false_e->dest, EDGE_FALSE_VALUE);
+ e->probability = prob2;
+ e2->probability = prob2.invert ();
+ set_immediate_dominator (CDI_DOMINATORS, false_e->dest, e->src);
+ auto_vec<basic_block, 3> adj;
+ for (basic_block son = first_dom_son (CDI_DOMINATORS, e->dest);
+ son;
+ son = next_dom_son (CDI_DOMINATORS, son))
+ if (EDGE_COUNT (son->preds) > 1)
+ adj.safe_push (son);
+ for (auto son : adj)
+ set_immediate_dominator (CDI_DOMINATORS, son, e->src);
+ }
+
if (version_niter)
{
/* The versioned loop could be infinite, we need to clear existing