return new_est_niter;
}
-/* Unroll LOOP FACTOR times. DESC describes number of iterations of LOOP.
- EXIT is the exit of the loop to that DESC corresponds.
+/* Unroll LOOP FACTOR times. LOOP is known to have a single exit edge
+ whose source block dominates the latch. DESC describes the number of
+ iterations of LOOP.
If N is number of iterations of the loop and MAY_BE_ZERO is the condition
under that loop exits in the first iteration even if N != 0,
void
tree_transform_and_unroll_loop (class loop *loop, unsigned factor,
- edge exit, class tree_niter_desc *desc,
+ class tree_niter_desc *desc,
transform_callback transform,
void *data)
{
gcond *exit_if = nullptr;
class loop *new_loop = nullptr;
- basic_block rest;
edge new_exit;
if (!single_loop_p)
{
+ edge exit = single_dom_exit (loop);
+
/* The values for scales should keep profile consistent, and somewhat
close to correct.
/* Prepare the cfg and update the phi nodes. Move the loop exit to the
loop latch (and make its condition dummy, for the moment). */
- rest = loop_preheader_edge (new_loop)->src;
+ basic_block rest = loop_preheader_edge (new_loop)->src;
edge precond_edge = single_pred_edge (rest);
split_edge (loop_latch_edge (loop));
basic_block exit_bb = single_pred (loop->latch);
remove_path (exit);
}
else
- {
- new_exit = exit;
- rest = exit->dest;
- }
+ new_exit = single_dom_exit (loop);
/* Transform the loop. */
if (transform)
}
update_ssa (TODO_update_ssa);
+ new_exit = single_dom_exit (loop);
if (!single_loop_p)
{
/* Ensure that the frequencies in the loop match the new estimated
freq_e = freq_e.force_nonzero ();
scale_loop_frequencies (loop, freq_e.probability_in (freq_h));
}
- }
- basic_block exit_bb = single_pred (loop->latch);
- new_exit = find_edge (exit_bb, rest);
- new_exit->probability = profile_probability::always ()
- .apply_scale (1, new_est_niter + 1);
+ basic_block rest = new_exit->dest;
+ new_exit->probability = profile_probability::always ()
+ .apply_scale (1, new_est_niter + 1);
- if (!single_loop_p)
- rest->count += new_exit->count ();
+ rest->count += new_exit->count ();
- edge new_nonexit = single_pred_edge (loop->latch);
- profile_probability prob = new_nonexit->probability;
- new_nonexit->probability = new_exit->probability.invert ();
- prob = new_nonexit->probability / prob;
- if (prob.initialized_p ())
- scale_bbs_frequencies (&loop->latch, 1, prob);
+ edge new_nonexit = single_pred_edge (loop->latch);
+ profile_probability prob = new_nonexit->probability;
+ new_nonexit->probability = new_exit->probability.invert ();
+ prob = new_nonexit->probability / prob;
+ if (prob.initialized_p ())
+ scale_bbs_frequencies (&loop->latch, 1, prob);
- if (!single_loop_p)
- {
/* Finally create the new counter for number of iterations and add
the new exit instruction. */
tree ctr_before, ctr_after;
- gimple_stmt_iterator bsi = gsi_last_nondebug_bb (exit_bb);
+ gimple_stmt_iterator bsi = gsi_last_nondebug_bb (new_exit->src);
exit_if = as_a <gcond *> (gsi_stmt (bsi));
create_iv (exit_base, exit_step, NULL_TREE, loop,
&bsi, false, &ctr_before, &ctr_after);
gimple_cond_set_rhs (exit_if, exit_bound);
update_stmt (exit_if);
}
+ else
+ {
+ /* gimple_duplicate_loop_to_header_edge has adjusted the loop body's
+ original profile counts in line with the unroll factor. However,
+ the old counts might not have been consistent with the old
+ iteration count.
+
+ Therefore, if the iteration count is known exactly, make sure that the
+ profile counts of the loop header (and any other blocks that might be
+ executed in the final iteration) are consistent with the combination
+ of (a) the incoming profile count and (b) the new iteration count. */
+ profile_count in_count = loop_preheader_edge (loop)->count ();
+ profile_count old_header_count = loop->header->count;
+ if (in_count.nonzero_p ()
+ && old_header_count.nonzero_p ()
+ && TREE_CODE (desc->niter) == INTEGER_CST)
+ {
+ /* The + 1 converts latch counts to iteration counts. */
+ profile_count new_header_count
+ = (in_count.apply_scale (new_est_niter + 1, 1));
+ basic_block *body = get_loop_body (loop);
+ scale_bbs_frequencies_profile_count (body, loop->num_nodes,
+ new_header_count,
+ old_header_count);
+ free (body);
+ }
+
+ /* gimple_duplicate_loop_to_header_edge discarded FACTOR - 1
+ exit edges and adjusted the loop body's profile counts for the
+ new probabilities of the remaining non-exit edges. However,
+ the remaining exit edge still has the same probability as it
+ did before, even though it is now more likely.
+
+ Therefore, all blocks executed after a failed exit test now have
+ a profile count that is too high, and the sum of the profile counts
+ for the header's incoming edges is greater than the profile count
+ of the header itself.
+
+ Adjust the profile counts of all code in the loop body after
+ the exit test so that the sum of the counts on entry to the
+ header agree. */
+ profile_count old_latch_count = loop_latch_edge (loop)->count ();
+ profile_count new_latch_count = loop->header->count - in_count;
+ if (old_latch_count.nonzero_p () && new_latch_count.nonzero_p ())
+ scale_dominated_blocks_in_loop (loop, new_exit->src, new_latch_count,
+ old_latch_count);
+
+ /* Set the probability of the exit edge based on NEW_EST_NITER
+ (which estimates latch counts rather than iteration counts).
+ Update the probabilities of other edges to match.
+
+ If the profile counts are large enough to give the required
+ precision, the updates above will have made
+
+ e->dest->count / e->src->count ~= new e->probability
+
+ for every outgoing edge e of NEW_EXIT->src. */
+ profile_probability new_exit_prob = profile_probability::always ()
+ .apply_scale (1, new_est_niter + 1);
+ change_edge_frequency (new_exit, new_exit_prob);
+ }
checking_verify_flow_info ();
checking_verify_loop_structure ();
void
tree_unroll_loop (class loop *loop, unsigned factor,
- edge exit, class tree_niter_desc *desc)
+ class tree_niter_desc *desc)
{
- tree_transform_and_unroll_loop (loop, factor, exit, desc,
- NULL, NULL);
+ tree_transform_and_unroll_loop (loop, factor, desc, NULL, NULL);
}
/* Rewrite the phi node at position PSI in function of the main
projects / platform / upstream / gcc.git / commitdiff