tree-vect-loop.c (vectorizable_reduction): Verify STMT_VINFO_REDUC_IDX on the to...

2019-10-25  Richard Biener  <rguenther@suse.de>

	* tree-vect-loop.c (vectorizable_reduction): Verify
	STMT_VINFO_REDUC_IDX on the to be vectorized stmts is set up
	correctly.
	* tree-vect-patterns.c (vect_mark_pattern_stmts): Transfer
	STMT_VINFO_REDUC_IDX from the original stmts to the pattern
	stmts.

From-SVN: r277437

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 4d9e470..bb64e93 100644 (file)
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,12 @@
+2019-10-25  Richard Biener  <rguenther@suse.de>
+
+       * tree-vect-loop.c (vectorizable_reduction): Verify
+       STMT_VINFO_REDUC_IDX on the to be vectorized stmts is set up
+       correctly.
+       * tree-vect-patterns.c (vect_mark_pattern_stmts): Transfer
+       STMT_VINFO_REDUC_IDX from the original stmts to the pattern
+       stmts.
+
 2019-10-24  Jakub Jelinek  <jakub@redhat.com>
 
        * gimplify.h (omp_construct_selector_matches): Declare.

diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c
index 9c5ce47..d0fd7bd 100644 (file)
--- a/gcc/tree-vect-loop.c
+++ b/gcc/tree-vect-loop.c
@@ -5719,7 +5719,19 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node,
   /* PHIs should not participate in patterns.  */
   gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info));
   gphi *reduc_def_phi = as_a <gphi *> (phi_info->stmt);
-  tree reduc_def = PHI_RESULT (reduc_def_phi);
+
+  /* Verify following REDUC_IDX from the latch def leads us back to the PHI.  */
+  tree reduc_def = PHI_ARG_DEF_FROM_EDGE (reduc_def_phi,
+                                         loop_latch_edge (loop));
+  while (reduc_def != PHI_RESULT (reduc_def_phi))
+    {
+      stmt_vec_info def = loop_vinfo->lookup_def (reduc_def);
+      def = vect_stmt_to_vectorize (def);
+      gcc_assert (STMT_VINFO_REDUC_IDX (def) != -1);
+      reduc_def = gimple_op (def->stmt, 1 + STMT_VINFO_REDUC_IDX (def));
+    }
+
+  reduc_def = PHI_RESULT (reduc_def_phi);
   int reduc_index = -1;
   for (i = 0; i < op_type; i++)
     {

diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c
index b497ec9..31e9e2a 100644 (file)
--- a/gcc/tree-vect-patterns.c
+++ b/gcc/tree-vect-patterns.c
@@ -5075,6 +5075,7 @@ static inline void
 vect_mark_pattern_stmts (stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
                          tree pattern_vectype)
 {
+  stmt_vec_info orig_stmt_info_saved = orig_stmt_info;
   gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
 
   gimple *orig_pattern_stmt = NULL;
@@ -5134,6 +5135,57 @@ vect_mark_pattern_stmts (stmt_vec_info orig_stmt_info, gimple *pattern_stmt,
     }
   else
     vect_set_pattern_stmt (pattern_stmt, orig_stmt_info, pattern_vectype);
+
+  /* Transfer reduction path info to the pattern.  */
+  if (STMT_VINFO_REDUC_IDX (orig_stmt_info_saved) != -1)
+    {
+      vec_info *vinfo = orig_stmt_info_saved->vinfo;
+      tree lookfor = gimple_op (orig_stmt_info_saved->stmt,
+                               1 + STMT_VINFO_REDUC_IDX (orig_stmt_info));
+      /* Search the pattern def sequence and the main pattern stmt.  Note
+         we may have inserted all into a containing pattern def sequence
+        so the following is a bit awkward.  */
+      gimple_stmt_iterator si;
+      gimple *s;
+      if (def_seq)
+       {
+         si = gsi_start (def_seq);
+         s = gsi_stmt (si);
+         gsi_next (&si);
+       }
+      else
+       {
+         si = gsi_none ();
+         s = pattern_stmt;
+       }
+      do
+       {
+         bool found = false;
+         for (unsigned i = 1; i < gimple_num_ops (s); ++i)
+           if (gimple_op (s, i) == lookfor)
+             {
+               STMT_VINFO_REDUC_IDX (vinfo->lookup_stmt (s)) = i - 1;
+               lookfor = gimple_get_lhs (s);
+               found = true;
+               break;
+             }
+         if (found && s == pattern_stmt)
+           break;
+         if (s == pattern_stmt)
+           gcc_unreachable ();
+         if (gsi_end_p (si))
+           s = pattern_stmt;
+         else
+           {
+             s = gsi_stmt (si);
+             if (s == pattern_stmt)
+               /* Found the end inside a bigger pattern def seq.  */
+               si = gsi_none ();
+             else
+               gsi_next (&si);
+           }
+       } while (1);
+    }
 }
 
 /* Function vect_pattern_recog_1