From: Richard Biener Date: Fri, 18 Feb 2022 10:50:44 +0000 (+0100) Subject: tree-optimization/104582 - make SLP node available in vector cost hook X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=f24dfc76177b3994434c8beb287cde1a9976b5ce;p=test_jj.git tree-optimization/104582 - make SLP node available in vector cost hook This adjusts the vectorizer costing API to allow passing down the SLP node the vector stmt is created from. 2022-02-18 Richard Biener PR tree-optimization/104582 * tree-vectorizer.h (stmt_info_for_cost::node): New field. (vector_costs::add_stmt_cost): Add SLP node parameter. (dump_stmt_cost): Likewise. (add_stmt_cost): Likewise, new overload and adjust. (add_stmt_costs): Adjust. (record_stmt_cost): New overload. * tree-vectorizer.cc (dump_stmt_cost): Dump the SLP node. (vector_costs::add_stmt_cost): Adjust. * tree-vect-loop.cc (vect_estimate_min_profitable_iters): Adjust. * tree-vect-slp.cc (vect_prologue_cost_for_slp): Record the SLP node for costing. (vectorizable_slp_permutation): Likewise. * tree-vect-stmts.cc (record_stmt_cost): Adjust and add new overloads. * config/i386/i386.cc (ix86_vector_costs::add_stmt_cost): Adjust. * config/aarch64/aarch64.cc (aarch64_vector_costs::add_stmt_cost): Adjust. * config/rs6000/rs6000.cc (rs6000_vector_costs::add_stmt_cost): Adjust. (rs6000_cost_data::adjust_vect_cost_per_loop): Likewise. --- diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc index 8bcee8b..dbeaaf4 100644 --- a/gcc/config/aarch64/aarch64.cc +++ b/gcc/config/aarch64/aarch64.cc @@ -15058,7 +15058,7 @@ public: aarch64_vector_costs (vec_info *, bool); unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind, - stmt_vec_info stmt_info, tree vectype, + stmt_vec_info stmt_info, slp_tree, tree vectype, int misalign, vect_cost_model_location where) override; void finish_cost (const vector_costs *) override; @@ -16003,8 +16003,8 @@ aarch64_stp_sequence_cost (unsigned int count, vect_cost_for_stmt kind, unsigned aarch64_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind, - stmt_vec_info stmt_info, tree vectype, - int misalign, + stmt_vec_info stmt_info, slp_tree, + tree vectype, int misalign, vect_cost_model_location where) { fractional_cost stmt_cost diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc index e4b42fb..0830dbd 100644 --- a/gcc/config/i386/i386.cc +++ b/gcc/config/i386/i386.cc @@ -22982,8 +22982,8 @@ class ix86_vector_costs : public vector_costs using vector_costs::vector_costs; unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind, - stmt_vec_info stmt_info, tree vectype, - int misalign, + stmt_vec_info stmt_info, slp_tree node, + tree vectype, int misalign, vect_cost_model_location where) override; }; @@ -22997,8 +22997,9 @@ ix86_vectorize_create_costs (vec_info *vinfo, bool costing_for_scalar) unsigned ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind, - stmt_vec_info stmt_info, tree vectype, - int misalign, vect_cost_model_location where) + stmt_vec_info stmt_info, slp_tree, + tree vectype, int misalign, + vect_cost_model_location where) { unsigned retval = 0; bool scalar_p diff --git a/gcc/config/rs6000/rs6000.cc b/gcc/config/rs6000/rs6000.cc index d7a7cfe..ca9e7b8 100644 --- a/gcc/config/rs6000/rs6000.cc +++ b/gcc/config/rs6000/rs6000.cc @@ -5212,7 +5212,7 @@ public: using vector_costs::vector_costs; unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind, - stmt_vec_info stmt_info, tree vectype, + stmt_vec_info stmt_info, slp_tree, tree vectype, int misalign, vect_cost_model_location where) override; void finish_cost (const vector_costs *) override; @@ -5428,8 +5428,9 @@ rs6000_cost_data::update_target_cost_per_stmt (vect_cost_for_stmt kind, unsigned rs6000_cost_data::add_stmt_cost (int count, vect_cost_for_stmt kind, - stmt_vec_info stmt_info, tree vectype, - int misalign, vect_cost_model_location where) + stmt_vec_info stmt_info, slp_tree, + tree vectype, int misalign, + vect_cost_model_location where) { unsigned retval = 0; @@ -5470,7 +5471,8 @@ rs6000_cost_data::adjust_vect_cost_per_loop (loop_vec_info loop_vinfo) /* Each length needs one shift to fill into bits 0-7. */ shift_cnt += num_vectors_m1 + 1; - add_stmt_cost (shift_cnt, scalar_stmt, NULL, NULL_TREE, 0, vect_body); + add_stmt_cost (shift_cnt, scalar_stmt, NULL, NULL, + NULL_TREE, 0, vect_body); } } diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc index 5c7b163..1f30fc8 100644 --- a/gcc/tree-vect-loop.cc +++ b/gcc/tree-vect-loop.cc @@ -3982,7 +3982,7 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo, { /* FIXME: Make cost depend on complexity of individual check. */ (void) add_stmt_cost (target_cost_data, 1, vector_stmt, - NULL, NULL_TREE, 0, vect_prologue); + NULL, NULL, NULL_TREE, 0, vect_prologue); if (dump_enabled_p ()) dump_printf (MSG_NOTE, "cost model: Adding cost of checks for loop " @@ -4079,8 +4079,8 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo, { (void) add_stmt_cost (target_cost_data, si->count * peel_iters_prologue, si->kind, - si->stmt_info, si->vectype, si->misalign, - vect_prologue); + si->stmt_info, si->node, si->vectype, + si->misalign, vect_prologue); } /* Add costs associated with peel_iters_epilogue. */ @@ -4089,8 +4089,8 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo, { (void) add_stmt_cost (target_cost_data, si->count * peel_iters_epilogue, si->kind, - si->stmt_info, si->vectype, si->misalign, - vect_epilogue); + si->stmt_info, si->node, si->vectype, + si->misalign, vect_epilogue); } /* Add possible cond_branch_taken/cond_branch_not_taken cost. */ @@ -4136,9 +4136,11 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo, being the tie-breaker between vectorizing or not, then it's probably better not to vectorize. */ (void) add_stmt_cost (target_cost_data, num_masks, - vector_stmt, NULL, NULL_TREE, 0, vect_prologue); + vector_stmt, NULL, NULL, NULL_TREE, 0, + vect_prologue); (void) add_stmt_cost (target_cost_data, num_masks - 1, - vector_stmt, NULL, NULL_TREE, 0, vect_body); + vector_stmt, NULL, NULL, NULL_TREE, 0, + vect_body); } else if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)) { diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc index c6b5a06..9188d72 100644 --- a/gcc/tree-vect-slp.cc +++ b/gcc/tree-vect-slp.cc @@ -4612,7 +4612,7 @@ vect_prologue_cost_for_slp (slp_tree node, kind = scalar_to_vec; else kind = vec_construct; - record_stmt_cost (cost_vec, 1, kind, NULL, vectype, 0, vect_prologue); + record_stmt_cost (cost_vec, 1, kind, node, vectype, 0, vect_prologue); } } @@ -7120,7 +7120,7 @@ vectorizable_slp_permutation (vec_info *vinfo, gimple_stmt_iterator *gsi, } if (!gsi) - record_stmt_cost (cost_vec, nperms, vec_perm, NULL, vectype, 0, vect_body); + record_stmt_cost (cost_vec, nperms, vec_perm, node, vectype, 0, vect_body); return true; } diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc index 6b3dde1..5c9e8cf 100644 --- a/gcc/tree-vect-stmts.cc +++ b/gcc/tree-vect-stmts.cc @@ -89,9 +89,10 @@ stmt_in_inner_loop_p (vec_info *vinfo, class _stmt_vec_info *stmt_info) target model or by saving it in a vector for later processing. Return a preliminary estimate of the statement's cost. */ -unsigned +static unsigned record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count, - enum vect_cost_for_stmt kind, stmt_vec_info stmt_info, + enum vect_cost_for_stmt kind, + stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, enum vect_cost_model_location where) { @@ -102,7 +103,8 @@ record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count, && (stmt_info && STMT_VINFO_GATHER_SCATTER_P (stmt_info))) kind = vector_scatter_store; - stmt_info_for_cost si = { count, kind, where, stmt_info, vectype, misalign }; + stmt_info_for_cost si + = { count, kind, where, stmt_info, node, vectype, misalign }; body_cost_vec->safe_push (si); return (unsigned) @@ -111,12 +113,32 @@ record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count, unsigned record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count, + enum vect_cost_for_stmt kind, stmt_vec_info stmt_info, + tree vectype, int misalign, + enum vect_cost_model_location where) +{ + return record_stmt_cost (body_cost_vec, count, kind, stmt_info, NULL, + vectype, misalign, where); +} + +unsigned +record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count, + enum vect_cost_for_stmt kind, slp_tree node, + tree vectype, int misalign, + enum vect_cost_model_location where) +{ + return record_stmt_cost (body_cost_vec, count, kind, NULL, node, + vectype, misalign, where); +} + +unsigned +record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count, enum vect_cost_for_stmt kind, enum vect_cost_model_location where) { gcc_assert (kind == cond_branch_taken || kind == cond_branch_not_taken || kind == scalar_stmt); - return record_stmt_cost (body_cost_vec, count, kind, NULL, + return record_stmt_cost (body_cost_vec, count, kind, NULL, NULL, NULL_TREE, 0, where); } diff --git a/gcc/tree-vectorizer.cc b/gcc/tree-vectorizer.cc index 344dc41..a63fa39 100644 --- a/gcc/tree-vectorizer.cc +++ b/gcc/tree-vectorizer.cc @@ -99,7 +99,8 @@ auto_purge_vect_location::~auto_purge_vect_location () void dump_stmt_cost (FILE *f, int count, enum vect_cost_for_stmt kind, - stmt_vec_info stmt_info, tree, int misalign, unsigned cost, + stmt_vec_info stmt_info, slp_tree node, tree, + int misalign, unsigned cost, enum vect_cost_model_location where) { if (stmt_info) @@ -107,6 +108,8 @@ dump_stmt_cost (FILE *f, int count, enum vect_cost_for_stmt kind, print_gimple_expr (f, STMT_VINFO_STMT (stmt_info), 0, TDF_SLIM); fprintf (f, " "); } + else if (node) + fprintf (f, "node %p ", (void *)node); else fprintf (f, " "); fprintf (f, "%d times ", count); @@ -1766,8 +1769,9 @@ scalar_cond_masked_key::get_cond_ops_from_tree (tree t) unsigned int vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind, - stmt_vec_info stmt_info, tree vectype, - int misalign, vect_cost_model_location where) + stmt_vec_info stmt_info, slp_tree, + tree vectype, int misalign, + vect_cost_model_location where) { unsigned int cost = builtin_vectorization_cost (kind, vectype, misalign) * count; diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index da99f28..642eb0a 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -22,6 +22,7 @@ along with GCC; see the file COPYING3. If not see #define GCC_TREE_VECTORIZER_H typedef class _stmt_vec_info *stmt_vec_info; +typedef struct _slp_tree *slp_tree; #include "tree-data-ref.h" #include "tree-hash-traits.h" @@ -101,6 +102,7 @@ struct stmt_info_for_cost { enum vect_cost_for_stmt kind; enum vect_cost_model_location where; stmt_vec_info stmt_info; + slp_tree node; tree vectype; int misalign; }; @@ -151,7 +153,6 @@ struct vect_scalar_ops_slice_hash : typed_noop_remove /************************************************************************ SLP ************************************************************************/ -typedef struct _slp_tree *slp_tree; typedef vec > lane_permutation_t; typedef vec load_permutation_t; @@ -1462,7 +1463,7 @@ public: - WHERE specifies whether the cost occurs in the loop prologue, the loop body, or the loop epilogue. - KIND is the kind of statement, which is always meaningful. - - STMT_INFO, if nonnull, describes the statement that will be + - STMT_INFO or NODE, if nonnull, describe the statement that will be vectorized. - VECTYPE, if nonnull, is the vector type that the vectorized statement will operate on. Note that this should be used in @@ -1476,8 +1477,9 @@ public: Return the calculated cost as well as recording it. The return value is used for dumping purposes. */ virtual unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind, - stmt_vec_info stmt_info, tree vectype, - int misalign, + stmt_vec_info stmt_info, + slp_tree node, + tree vectype, int misalign, vect_cost_model_location where); /* Finish calculating the cost of the code. The results can be @@ -1743,7 +1745,7 @@ init_cost (vec_info *vinfo, bool costing_for_scalar) } extern void dump_stmt_cost (FILE *, int, enum vect_cost_for_stmt, - stmt_vec_info, tree, int, unsigned, + stmt_vec_info, slp_tree, tree, int, unsigned, enum vect_cost_model_location); /* Alias targetm.vectorize.add_stmt_cost. */ @@ -1751,13 +1753,14 @@ extern void dump_stmt_cost (FILE *, int, enum vect_cost_for_stmt, static inline unsigned add_stmt_cost (vector_costs *costs, int count, enum vect_cost_for_stmt kind, - stmt_vec_info stmt_info, tree vectype, int misalign, + stmt_vec_info stmt_info, slp_tree node, + tree vectype, int misalign, enum vect_cost_model_location where) { - unsigned cost = costs->add_stmt_cost (count, kind, stmt_info, vectype, + unsigned cost = costs->add_stmt_cost (count, kind, stmt_info, node, vectype, misalign, where); if (dump_file && (dump_flags & TDF_DETAILS)) - dump_stmt_cost (dump_file, count, kind, stmt_info, vectype, misalign, + dump_stmt_cost (dump_file, count, kind, stmt_info, node, vectype, misalign, cost, where); return cost; } @@ -1768,7 +1771,7 @@ add_stmt_cost (vector_costs *costs, int count, enum vect_cost_for_stmt kind, { gcc_assert (kind == cond_branch_taken || kind == cond_branch_not_taken || kind == scalar_stmt); - return add_stmt_cost (costs, count, kind, NULL, NULL_TREE, 0, where); + return add_stmt_cost (costs, count, kind, NULL, NULL, NULL_TREE, 0, where); } /* Alias targetm.vectorize.add_stmt_cost. */ @@ -1776,7 +1779,7 @@ add_stmt_cost (vector_costs *costs, int count, enum vect_cost_for_stmt kind, static inline unsigned add_stmt_cost (vector_costs *costs, stmt_info_for_cost *i) { - return add_stmt_cost (costs, i->count, i->kind, i->stmt_info, + return add_stmt_cost (costs, i->count, i->kind, i->stmt_info, i->node, i->vectype, i->misalign, i->where); } @@ -1802,7 +1805,7 @@ add_stmt_costs (vector_costs *costs, stmt_vector_for_cost *cost_vec) unsigned i; FOR_EACH_VEC_ELT (*cost_vec, i, cost) add_stmt_cost (costs, cost->count, cost->kind, cost->stmt_info, - cost->vectype, cost->misalign, cost->where); + cost->node, cost->vectype, cost->misalign, cost->where); } /*-----------------------------------------------------------------*/ @@ -2130,6 +2133,9 @@ extern unsigned record_stmt_cost (stmt_vector_for_cost *, int, enum vect_cost_for_stmt, stmt_vec_info, tree, int, enum vect_cost_model_location); extern unsigned record_stmt_cost (stmt_vector_for_cost *, int, + enum vect_cost_for_stmt, slp_tree, + tree, int, enum vect_cost_model_location); +extern unsigned record_stmt_cost (stmt_vector_for_cost *, int, enum vect_cost_for_stmt, enum vect_cost_model_location);