tree-optimization/104582 - make SLP node available in vector cost hook

This adjusts the vectorizer costing API to allow passing down the
SLP node the vector stmt is created from.

2022-02-18  Richard Biener  <rguenther@suse.de>

	PR tree-optimization/104582
	* tree-vectorizer.h (stmt_info_for_cost::node): New field.
	(vector_costs::add_stmt_cost): Add SLP node parameter.
	(dump_stmt_cost): Likewise.
	(add_stmt_cost): Likewise, new overload and adjust.
	(add_stmt_costs): Adjust.
	(record_stmt_cost): New overload.
	* tree-vectorizer.cc (dump_stmt_cost): Dump the SLP node.
	(vector_costs::add_stmt_cost): Adjust.
	* tree-vect-loop.cc (vect_estimate_min_profitable_iters):
	Adjust.
	* tree-vect-slp.cc (vect_prologue_cost_for_slp): Record
	the SLP node for costing.
	(vectorizable_slp_permutation): Likewise.
	* tree-vect-stmts.cc (record_stmt_cost): Adjust and add
	new overloads.
	* config/i386/i386.cc (ix86_vector_costs::add_stmt_cost):
	Adjust.
	* config/aarch64/aarch64.cc (aarch64_vector_costs::add_stmt_cost):
	Adjust.
	* config/rs6000/rs6000.cc (rs6000_vector_costs::add_stmt_cost):
	Adjust.
	(rs6000_cost_data::adjust_vect_cost_per_loop): Likewise.

diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index 8bcee8be9eb91aebebac9504f9538cb5d906437a..dbeaaf484dbc070ae3fcc08530ec9bd20b8ab651 100644 (file)
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -15058,7 +15058,7 @@ public:
   aarch64_vector_costs (vec_info *, bool);
 
   unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind,
-                             stmt_vec_info stmt_info, tree vectype,
+                             stmt_vec_info stmt_info, slp_tree, tree vectype,
                              int misalign,
                              vect_cost_model_location where) override;
   void finish_cost (const vector_costs *) override;
@@ -16003,8 +16003,8 @@ aarch64_stp_sequence_cost (unsigned int count, vect_cost_for_stmt kind,
 
 unsigned
 aarch64_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
-                                    stmt_vec_info stmt_info, tree vectype,
-                                    int misalign,
+                                    stmt_vec_info stmt_info, slp_tree,
+                                    tree vectype, int misalign,
                                     vect_cost_model_location where)
 {
   fractional_cost stmt_cost

diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
index e4b42fbba6fdbe4b1fc70449a096fad47b053c18..0830dbd7dcabc075f61d5b2e2f5ee7f5fd55fefe 100644 (file)
--- a/gcc/config/i386/i386.cc
+++ b/gcc/config/i386/i386.cc
@@ -22982,8 +22982,8 @@ class ix86_vector_costs : public vector_costs
   using vector_costs::vector_costs;
 
   unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind,
-                             stmt_vec_info stmt_info, tree vectype,
-                             int misalign,
+                             stmt_vec_info stmt_info, slp_tree node,
+                             tree vectype, int misalign,
                              vect_cost_model_location where) override;
 };
 
@@ -22997,8 +22997,9 @@ ix86_vectorize_create_costs (vec_info *vinfo, bool costing_for_scalar)
 
 unsigned
 ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
-                                 stmt_vec_info stmt_info, tree vectype,
-                                 int misalign, vect_cost_model_location where)
+                                 stmt_vec_info stmt_info, slp_tree,
+                                 tree vectype, int misalign,
+                                 vect_cost_model_location where)
 {
   unsigned retval = 0;
   bool scalar_p

diff --git a/gcc/config/rs6000/rs6000.cc b/gcc/config/rs6000/rs6000.cc
index d7a7cfe860f15b3c59339a6e46aa5e4ed3aba7c2..ca9e7b80dcbfe61a2775ab51d93b14b3dd0123e1 100644 (file)
--- a/gcc/config/rs6000/rs6000.cc
+++ b/gcc/config/rs6000/rs6000.cc
@@ -5212,7 +5212,7 @@ public:
   using vector_costs::vector_costs;
 
   unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind,
-                             stmt_vec_info stmt_info, tree vectype,
+                             stmt_vec_info stmt_info, slp_tree, tree vectype,
                              int misalign,
                              vect_cost_model_location where) override;
   void finish_cost (const vector_costs *) override;
@@ -5428,8 +5428,9 @@ rs6000_cost_data::update_target_cost_per_stmt (vect_cost_for_stmt kind,
 
 unsigned
 rs6000_cost_data::add_stmt_cost (int count, vect_cost_for_stmt kind,
-                                stmt_vec_info stmt_info, tree vectype,
-                                int misalign, vect_cost_model_location where)
+                                stmt_vec_info stmt_info, slp_tree,
+                                tree vectype, int misalign,
+                                vect_cost_model_location where)
 {
   unsigned retval = 0;
 
@@ -5470,7 +5471,8 @@ rs6000_cost_data::adjust_vect_cost_per_loop (loop_vec_info loop_vinfo)
          /* Each length needs one shift to fill into bits 0-7.  */
          shift_cnt += num_vectors_m1 + 1;
 
-      add_stmt_cost (shift_cnt, scalar_stmt, NULL, NULL_TREE, 0, vect_body);
+      add_stmt_cost (shift_cnt, scalar_stmt, NULL, NULL,
+                    NULL_TREE, 0, vect_body);
     }
 }
 

diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 5c7b163f01cb438cef87aa2049f91863e67fc601..1f30fc82ca14822c821806bd66d9f83f0c178e8c 100644 (file)
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -3982,7 +3982,7 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
     {
       /*  FIXME: Make cost depend on complexity of individual check.  */
       (void) add_stmt_cost (target_cost_data, 1, vector_stmt,
-                           NULL, NULL_TREE, 0, vect_prologue);
+                           NULL, NULL, NULL_TREE, 0, vect_prologue);
       if (dump_enabled_p ())
        dump_printf (MSG_NOTE,
                     "cost model: Adding cost of checks for loop "
@@ -4079,8 +4079,8 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
       {
        (void) add_stmt_cost (target_cost_data,
                              si->count * peel_iters_prologue, si->kind,
-                             si->stmt_info, si->vectype, si->misalign,
-                             vect_prologue);
+                             si->stmt_info, si->node, si->vectype,
+                             si->misalign, vect_prologue);
       }
 
   /* Add costs associated with peel_iters_epilogue.  */
@@ -4089,8 +4089,8 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
       {
        (void) add_stmt_cost (target_cost_data,
                              si->count * peel_iters_epilogue, si->kind,
-                             si->stmt_info, si->vectype, si->misalign,
-                             vect_epilogue);
+                             si->stmt_info, si->node, si->vectype,
+                             si->misalign, vect_epilogue);
       }
 
   /* Add possible cond_branch_taken/cond_branch_not_taken cost.  */
@@ -4136,9 +4136,11 @@ vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo,
         being the tie-breaker between vectorizing or not, then it's
         probably better not to vectorize.  */
       (void) add_stmt_cost (target_cost_data, num_masks,
-                           vector_stmt, NULL, NULL_TREE, 0, vect_prologue);
+                           vector_stmt, NULL, NULL, NULL_TREE, 0,
+                           vect_prologue);
       (void) add_stmt_cost (target_cost_data, num_masks - 1,
-                           vector_stmt, NULL, NULL_TREE, 0, vect_body);
+                           vector_stmt, NULL, NULL, NULL_TREE, 0,
+                           vect_body);
     }
   else if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
     {

diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index c6b5a0696a2ef2d04e027abd2f665eced439f2ff..9188d727e335f683d504aab4c6edddf4fb19b41b 100644 (file)
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -4612,7 +4612,7 @@ vect_prologue_cost_for_slp (slp_tree node,
        kind = scalar_to_vec;
       else
        kind = vec_construct;
-      record_stmt_cost (cost_vec, 1, kind, NULL, vectype, 0, vect_prologue);
+      record_stmt_cost (cost_vec, 1, kind, node, vectype, 0, vect_prologue);
     }
 }
 
@@ -7120,7 +7120,7 @@ vectorizable_slp_permutation (vec_info *vinfo, gimple_stmt_iterator *gsi,
     }
 
   if (!gsi)
-    record_stmt_cost (cost_vec, nperms, vec_perm, NULL, vectype, 0, vect_body);
+    record_stmt_cost (cost_vec, nperms, vec_perm, node, vectype, 0, vect_body);
 
   return true;
 }

diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 6b3dde1c8969f498b3561abed174e8dfe89c2895..5c9e8cfefa5032d39a11696b06cff9ae50f4d46a 100644 (file)
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -89,9 +89,10 @@ stmt_in_inner_loop_p (vec_info *vinfo, class _stmt_vec_info *stmt_info)
    target model or by saving it in a vector for later processing.
    Return a preliminary estimate of the statement's cost.  */
 
-unsigned
+static unsigned
 record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
-                 enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
+                 enum vect_cost_for_stmt kind,
+                 stmt_vec_info stmt_info, slp_tree node,
                  tree vectype, int misalign,
                  enum vect_cost_model_location where)
 {
@@ -102,13 +103,34 @@ record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
       && (stmt_info && STMT_VINFO_GATHER_SCATTER_P (stmt_info)))
     kind = vector_scatter_store;
 
-  stmt_info_for_cost si = { count, kind, where, stmt_info, vectype, misalign };
+  stmt_info_for_cost si
+    = { count, kind, where, stmt_info, node, vectype, misalign };
   body_cost_vec->safe_push (si);
 
   return (unsigned)
       (builtin_vectorization_cost (kind, vectype, misalign) * count);
 }
 
+unsigned
+record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
+                 enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
+                 tree vectype, int misalign,
+                 enum vect_cost_model_location where)
+{
+  return record_stmt_cost (body_cost_vec, count, kind, stmt_info, NULL,
+                          vectype, misalign, where);
+}
+
+unsigned
+record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
+                 enum vect_cost_for_stmt kind, slp_tree node,
+                 tree vectype, int misalign,
+                 enum vect_cost_model_location where)
+{
+  return record_stmt_cost (body_cost_vec, count, kind, NULL, node,
+                          vectype, misalign, where);
+}
+
 unsigned
 record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
                  enum vect_cost_for_stmt kind,
@@ -116,7 +138,7 @@ record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
 {
   gcc_assert (kind == cond_branch_taken || kind == cond_branch_not_taken
              || kind == scalar_stmt);
-  return record_stmt_cost (body_cost_vec, count, kind, NULL,
+  return record_stmt_cost (body_cost_vec, count, kind, NULL, NULL,
                           NULL_TREE, 0, where);
 }
 

diff --git a/gcc/tree-vectorizer.cc b/gcc/tree-vectorizer.cc
index 344dc419d8cadf27fd605afbfdf5d87cd3a26cbb..a63fa391273923c351c78d65e0e1fed6030fb510 100644 (file)
--- a/gcc/tree-vectorizer.cc
+++ b/gcc/tree-vectorizer.cc
@@ -99,7 +99,8 @@ auto_purge_vect_location::~auto_purge_vect_location ()
 
 void
 dump_stmt_cost (FILE *f, int count, enum vect_cost_for_stmt kind,
-               stmt_vec_info stmt_info, tree, int misalign, unsigned cost,
+               stmt_vec_info stmt_info, slp_tree node, tree,
+               int misalign, unsigned cost,
                enum vect_cost_model_location where)
 {
   if (stmt_info)
@@ -107,6 +108,8 @@ dump_stmt_cost (FILE *f, int count, enum vect_cost_for_stmt kind,
       print_gimple_expr (f, STMT_VINFO_STMT (stmt_info), 0, TDF_SLIM);
       fprintf (f, " ");
     }
+  else if (node)
+    fprintf (f, "node %p ", (void *)node);
   else
     fprintf (f, "<unknown> ");
   fprintf (f, "%d times ", count);
@@ -1766,8 +1769,9 @@ scalar_cond_masked_key::get_cond_ops_from_tree (tree t)
 
 unsigned int
 vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
-                            stmt_vec_info stmt_info, tree vectype,
-                            int misalign, vect_cost_model_location where)
+                            stmt_vec_info stmt_info, slp_tree,
+                            tree vectype, int misalign,
+                            vect_cost_model_location where)
 {
   unsigned int cost
     = builtin_vectorization_cost (kind, vectype, misalign) * count;

diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index da99f28c0dcc9ffc172d59f46815359f7ff51b3f..642eb0aeb21264cd736a479b1ec25357abef29cd 100644 (file)
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -22,6 +22,7 @@ along with GCC; see the file COPYING3.  If not see
 #define GCC_TREE_VECTORIZER_H
 
 typedef class _stmt_vec_info *stmt_vec_info;
+typedef struct _slp_tree *slp_tree;
 
 #include "tree-data-ref.h"
 #include "tree-hash-traits.h"
@@ -101,6 +102,7 @@ struct stmt_info_for_cost {
   enum vect_cost_for_stmt kind;
   enum vect_cost_model_location where;
   stmt_vec_info stmt_info;
+  slp_tree node;
   tree vectype;
   int misalign;
 };
@@ -151,7 +153,6 @@ struct vect_scalar_ops_slice_hash : typed_noop_remove<vect_scalar_ops_slice>
 /************************************************************************
   SLP
  ************************************************************************/
-typedef struct _slp_tree *slp_tree;
 typedef vec<std::pair<unsigned, unsigned> > lane_permutation_t;
 typedef vec<unsigned> load_permutation_t;
 
@@ -1462,7 +1463,7 @@ public:
      - WHERE specifies whether the cost occurs in the loop prologue,
        the loop body, or the loop epilogue.
      - KIND is the kind of statement, which is always meaningful.
-     - STMT_INFO, if nonnull, describes the statement that will be
+     - STMT_INFO or NODE, if nonnull, describe the statement that will be
        vectorized.
      - VECTYPE, if nonnull, is the vector type that the vectorized
        statement will operate on.  Note that this should be used in
@@ -1476,8 +1477,9 @@ public:
      Return the calculated cost as well as recording it.  The return
      value is used for dumping purposes.  */
   virtual unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind,
-                                     stmt_vec_info stmt_info, tree vectype,
-                                     int misalign,
+                                     stmt_vec_info stmt_info,
+                                     slp_tree node,
+                                     tree vectype, int misalign,
                                      vect_cost_model_location where);
 
   /* Finish calculating the cost of the code.  The results can be
@@ -1743,7 +1745,7 @@ init_cost (vec_info *vinfo, bool costing_for_scalar)
 }
 
 extern void dump_stmt_cost (FILE *, int, enum vect_cost_for_stmt,
-                           stmt_vec_info, tree, int, unsigned,
+                           stmt_vec_info, slp_tree, tree, int, unsigned,
                            enum vect_cost_model_location);
 
 /* Alias targetm.vectorize.add_stmt_cost.  */
@@ -1751,13 +1753,14 @@ extern void dump_stmt_cost (FILE *, int, enum vect_cost_for_stmt,
 static inline unsigned
 add_stmt_cost (vector_costs *costs, int count,
               enum vect_cost_for_stmt kind,
-              stmt_vec_info stmt_info, tree vectype, int misalign,
+              stmt_vec_info stmt_info, slp_tree node,
+              tree vectype, int misalign,
               enum vect_cost_model_location where)
 {
-  unsigned cost = costs->add_stmt_cost (count, kind, stmt_info, vectype,
+  unsigned cost = costs->add_stmt_cost (count, kind, stmt_info, node, vectype,
                                        misalign, where);
   if (dump_file && (dump_flags & TDF_DETAILS))
-    dump_stmt_cost (dump_file, count, kind, stmt_info, vectype, misalign,
+    dump_stmt_cost (dump_file, count, kind, stmt_info, node, vectype, misalign,
                    cost, where);
   return cost;
 }
@@ -1768,7 +1771,7 @@ add_stmt_cost (vector_costs *costs, int count, enum vect_cost_for_stmt kind,
 {
   gcc_assert (kind == cond_branch_taken || kind == cond_branch_not_taken
              || kind == scalar_stmt);
-  return add_stmt_cost (costs, count, kind, NULL, NULL_TREE, 0, where);
+  return add_stmt_cost (costs, count, kind, NULL, NULL, NULL_TREE, 0, where);
 }
 
 /* Alias targetm.vectorize.add_stmt_cost.  */
@@ -1776,7 +1779,7 @@ add_stmt_cost (vector_costs *costs, int count, enum vect_cost_for_stmt kind,
 static inline unsigned
 add_stmt_cost (vector_costs *costs, stmt_info_for_cost *i)
 {
-  return add_stmt_cost (costs, i->count, i->kind, i->stmt_info,
+  return add_stmt_cost (costs, i->count, i->kind, i->stmt_info, i->node,
                        i->vectype, i->misalign, i->where);
 }
 
@@ -1802,7 +1805,7 @@ add_stmt_costs (vector_costs *costs, stmt_vector_for_cost *cost_vec)
   unsigned i;
   FOR_EACH_VEC_ELT (*cost_vec, i, cost)
     add_stmt_cost (costs, cost->count, cost->kind, cost->stmt_info,
-                  cost->vectype, cost->misalign, cost->where);
+                  cost->node, cost->vectype, cost->misalign, cost->where);
 }
 
 /*-----------------------------------------------------------------*/
@@ -2129,6 +2132,9 @@ extern bool supportable_narrowing_operation (enum tree_code, tree, tree,
 extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
                                  enum vect_cost_for_stmt, stmt_vec_info,
                                  tree, int, enum vect_cost_model_location);
+extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
+                                 enum vect_cost_for_stmt, slp_tree,
+                                 tree, int, enum vect_cost_model_location);
 extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
                                  enum vect_cost_for_stmt,
                                  enum vect_cost_model_location);