+2007-02-06 Dorit Nuzman <dorit@il.ibm.com>
+ Victor Kaplansky <victork@il.ibm.com>
+
+ * tree-vectorizer.c (vect_is_simple_use): Support induction.
+ (vect_is_simple_reduction): Support reduction with induction as
+ one of the operands.
+ (vect_is_simple_iv_evolution): Fix formatting.
+ * tree-vect-analyze.c (vect_mark_stmts_to_be_vectorized): Fix
+ formatting. Don't mark induction phis for vectorization.
+ (vect_analyze_scalar_cycles): Analyze all inductions, then reductions.
+ * tree-vect-transform.c (get_initial_def_for_induction): New function.
+ (vect_get_vec_def_for_operand): Support induction.
+ (vect_get_vec_def_for_stmt_copy): Fix formatting and add check for
+ induction case.
+ (vectorizable_reduction): Support reduction with induction as one of
+ the operands.
+ (vectorizable_type_demotion): Use def-type of stmt argument rather
+ than dummy def-type.
+
+ * tree-ssa-loop.c (gate_scev_const_prop): Return the value of
+ flag_tree_scev_cprop.
+ * common.opt (tree-scev-cprop): New flag.
+
+ * tree-vect-transform.c (vect_create_destination_var): Use 'kind' in
+ call to vect_get_new_vect_var.
+
2007-02-06 Ira Rosen <irar@il.ibm.com>
* tree-vect-patterns.c (vect_recog_widen_mult_pattern): Check that
Common RejectNegative Joined
-ftree-vectorizer-verbose=<number> Set the verbosity level of the vectorizer
+ftree-scev-cprop
+Common Report Var(flag_tree_scev_cprop) Init(1)
+Enable copy propagation of scalar-evolution information.
+
; -fverbose-asm causes extra commentary information to be produced in
; the generated assembly code (to make it more readable). This option
; is generally only of use to those who actually need to read the
+2007-02-06 Dorit Nuzman <dorit@il.ibm.com>
+
+ * gcc.dg/vect/vect.exp: Add support for -fno-tree-scev-cprop tests.
+ * gcc.dg/vect/vect-iv-1.c: New test.
+ * gcc.dg/vect/vect-iv-2.c: New test.
+ * gcc.dg/vect/vect-iv-3.c: New test.
+ * gcc.dg/vect/vect-iv-4.c: New test.
+ * gcc.dg/vect/vect-iv-5.c: New test.
+ * gcc.dg/vect/vect-iv-6.c: New test.
+ * gcc.dg/vect/vect-iv-7.c: New test.
+ * gcc.dg/vect/vect-iv-8.c: New test.
+ * gcc.dg/vect/vect-iv-9.c: New test.
+ * gcc.dg/vect/vect-iv-10.c: New test.
+ * gcc.dg/vect/vect-iv-11.c: New test.
+ * gcc.dg/vect/no-tree-scev-cprop-vect-iv-1.c: New test.
+ * gcc.dg/vect/no-tree-scev-cprop-vect-iv-2.c: New test.
+ * gcc.dg/vect/vect-14.c: Now vectorizable.
+ * gcc.dg/vect/pr21591.c: Additional loop vectorized (initilization loop).
+ * gcc.dg/vect/vect-27.c: Likewise.
+ * gcc.dg/vect/vect-29.c Likewise.
+ * gcc.dg/vect/vect-dv-2.c: Likewise.
+ * gcc.dg/vect/vect-reduc-dot-u16a.c: Likewise.
+ * gcc.dg/vect/vect-reduc-dot-u16b.c: Likewise.
+ * gcc.dg/vect/vect-widen-mult-u16.c: Likewise.
+
2007-02-06 Ira Rosen <irar@il.ibm.com>
* gcc.dg/vect/fast-math-vect-pow-2.c: New test.
--- /dev/null
+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 26
+
+int main1 (int X)
+{
+ int s = X;
+ int i;
+
+ /* vectorization of reduction with induction.
+ Need -fno-tree-scev-cprop or else the loop is eliminated. */
+ for (i = 0; i < N; i++)
+ s += i;
+
+ return s;
+}
+
+int main (void)
+{
+ int s;
+ check_vect ();
+
+ s = main1 (3);
+ if (s != 328)
+ abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
--- /dev/null
+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+
+int main1 ()
+{
+ int arr1[N];
+ int k = 0;
+ int m = 3, i = 0;
+
+ /* Vectorization of induction that is used after the loop.
+ Currently vectorizable because scev_ccp disconnects the
+ use-after-the-loop from the iv def inside the loop. */
+
+ do {
+ k = k + 2;
+ arr1[i] = k;
+ m = m + k;
+ i++;
+ } while (i < N);
+
+ /* check results: */
+ for (i = 0; i < N; i++)
+ {
+ if (arr1[i] != 2+2*i)
+ abort ();
+ }
+
+ return m + k;
+}
+
+int main (void)
+{
+ int res;
+
+ check_vect ();
+
+ res = main1 ();
+ if (res != 32 + 275)
+ abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail *-*-* } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
free(c);
}
-/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" } } */
/* { dg-final { cleanup-tree-dump "vect" } } */
int i;
int ia[N];
-
- /* Not vectorizable yet (induction). */
+ /* Induction. */
for ( i = 0; i < N; i++) {
ia[i] = i;
}
return main1 ();
}
-/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
/* { dg-final { cleanup-tree-dump "vect" } } */
return main1 ();
}
-/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail vect_no_align } } } */
+/* The initialization induction loop (with aligned access) is also vectorized. */
+/* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" { xfail vect_no_align } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { target vect_no_align } } } */
/* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 1 "vect" { xfail vect_no_align } } } */
/* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 0 "vect" } } */
/* { dg-final { cleanup-tree-dump "vect" } } */
/* For targets that don't support misaligned loads we version for the load.
(The store is aligned). */
-/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
+/* The initialization induction loop (with aligned access) is also vectorized. */
+/* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" } } */
/* { dg-final { scan-tree-dump-times "Vectorizing an unaligned access" 1 "vect" { xfail vect_no_align } } } */
/* { dg-final { scan-tree-dump-times "Alignment of access forced using peeling" 0 "vect" } } */
/* { dg-final { scan-tree-dump-times "Alignment of access forced using versioning." 1 "vect" {target vect_no_align } } } */
}
-
-/* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" } } */
+/* The initialization induction loop (with aligned access) is also vectorized. */
+/* { dg-final { scan-tree-dump-times "vectorized 3 loops" 1 "vect" } } */
/* { dg-final { scan-tree-dump-times "accesses have the same alignment." 2 "vect" } } */
/* { dg-final { cleanup-tree-dump "vect" } } */
--- /dev/null
+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+int result[N] = {8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38};
+
+int main1 (int X)
+{
+ int arr[N];
+ int k = X;
+ int m, i=0;
+
+ /* vectorization of induction. */
+
+ do {
+ m = k + 5;
+ arr[i] = m;
+ k = k + 2;
+ i++;
+ } while (i < N);
+
+ /* check results: */
+ for (i = 0; i < N; i++)
+ {
+ if (arr[i] != result[i])
+ abort ();
+ }
+
+ return 0;
+}
+
+int main (void)
+{
+ check_vect ();
+
+ return main1 (3);
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
--- /dev/null
+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+
+int main1 ()
+{
+ int i,j;
+ int ia[N];
+
+ /* Induction. */
+ for (j=0,i=N; j<N,i>0; i--,j++) {
+ ia[j] = i;
+ }
+
+ /* check results: */
+ for (j=0,i=N; j<N,i>0; i--,j++) {
+ if (ia[j] != i)
+ abort ();
+ }
+
+ return 0;
+}
+
+int main (void)
+{
+ check_vect();
+
+ return main1 ();
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
--- /dev/null
+/* { dg-require-effective-target vect_int } */
+
+#include <stdio.h>
+#include <stdarg.h>
+#include "tree-vect.h"
+
+int main1 (int len)
+{
+ int s = 0;
+ int i = len;
+
+ /* vectorization of induction with reduction. */
+ for ( ; i > 1; i -=2)
+ s += i;
+
+ return s;
+}
+
+int main (void)
+{
+ int s;
+ check_vect ();
+
+ s = main1 (26);
+ if (s != 182)
+ abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail *-*-* } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
--- /dev/null
+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+
+int main1 ()
+{
+ int arr1[N];
+ int k = 0;
+ int m = 3, i = 0;
+
+ /* Vectorization of induction that is used after the loop.
+ Currently vectorizable because scev_ccp disconnects the
+ use-after-the-loop from the iv def inside the loop. */
+
+ do {
+ k = k + 2;
+ arr1[i] = k;
+ m = m + k;
+ i++;
+ } while (i < N);
+
+ /* check results: */
+ for (i = 0; i < N; i++)
+ {
+ if (arr1[i] != 2+2*i)
+ abort ();
+ }
+
+ return m + k;
+}
+
+int main (void)
+{
+ int res;
+
+ check_vect ();
+
+ res = main1 ();
+ if (res != 32 + 275)
+ abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
--- /dev/null
+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+
+int main1 ()
+{
+ int arr1[N];
+ int arr2[N];
+ int k = 0;
+ int m = 3, i = 0;
+
+ /* vectorization of induction.
+ Peeling to align the store is also applied. */
+
+ do {
+ k = k + 2;
+ arr1[i] = k;
+ m = k + 3;
+ arr2[i] = m;
+ i++;
+ } while (i < N);
+
+ /* check results: */
+ for (i = 0; i < N; i++)
+ {
+ if (arr1[i] != 2+2*i || arr2[i] != 5 + 2*i)
+ abort ();
+ }
+
+ return 0;
+}
+
+int main (void)
+{
+ check_vect ();
+
+ return main1 ();
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
--- /dev/null
+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+
+int main1 ()
+{
+ unsigned int arr1[N];
+ unsigned short arr2[N];
+ unsigned int k = 0;
+ unsigned short m = 3;
+ int i = 0;
+
+ /* Vectorization of induction with multiple data types. */
+
+ do {
+ k = k + 2;
+ arr1[i] = k;
+ m = k + 3;
+ arr2[i] = m;
+ i++;
+ } while (i < N);
+
+ /* check results: */
+ for (i = 0; i < N; i++)
+ {
+ if (arr1[i] != 2+2*i || arr2[i] != 5 + 2*i)
+ abort ();
+ }
+
+ return 0;
+}
+
+int main (void)
+{
+ check_vect ();
+
+ return main1 ();
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
--- /dev/null
+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+
+int main1 ()
+{
+ float arr[N];
+ float f = 1.0;
+ int i;
+
+ /* Vectorization of fp induction. */
+
+ for (i=0; i<N; i++)
+ {
+ arr[i] = f;
+ f += 2.0;
+ }
+
+ /* check results: */
+ for (i = 0; i < N; i++)
+ {
+ if (arr[i] != 1.0 + 2.0*i)
+ abort ();
+ }
+
+ return 0;
+}
+
+int main (void)
+{
+ check_vect ();
+
+ return main1 ();
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail *-*-* } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
--- /dev/null
+/* { dg-require-effective-target vect_int } */
+#include <stdio.h>
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+
+int main1 (int X)
+{
+ int arr1[N+1];
+ int arr2[N+1];
+ int k = X;
+ int m, i=0;
+
+ /* Vectorization of induction with non-constant initial condition X.
+ Also we have here two uses of the induction-variable k as defined
+ by the loop-header phi (as opposed to the other uses of k that are
+ defined in the loop), in which case we exercise the fact that we
+ reuse the same vector def-use-cycle for both uses.
+ Peeling to align the store is also applied. */
+
+ do {
+ arr2[i+1] = 2*k;
+ k = k + 2;
+ arr1[i+1] = k;
+ k = k + 4;
+ i++;
+ } while (i < N);
+
+ /* check results: */
+ for (i = 0; i < N; i++)
+ {
+ if (arr1[i+1] != X+6*i+2
+ || arr2[i+1] != 2*(X+6*i))
+ abort ();
+ }
+
+ return 0;
+}
+
+int main (void)
+{
+ check_vect ();
+
+ return main1 (3);
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail vect_no_align } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
--- /dev/null
+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 16
+int result[N] = {8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38};
+
+int main1 (int X)
+{
+ int arr[N];
+ int k = 3;
+ int m, i=0;
+
+ /* Vectorization of induction with non-constant step X. */
+
+ do {
+ m = k + 5;
+ arr[i] = m;
+ k = k + X;
+ i++;
+ } while (i < N);
+
+ /* check results: */
+ for (i = 0; i < N; i++)
+ {
+ if (arr[i] != result[i])
+ abort ();
+ }
+
+ return 0;
+}
+
+int main (void)
+{
+ check_vect ();
+
+ return main1 (2);
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail *-*-* } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
--- /dev/null
+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 26
+
+int main1 (short X)
+{
+ unsigned char a[N];
+ unsigned short b[N];
+ unsigned int c[N];
+ int i;
+
+ /* vectorization of induction with type conversions. */
+ for (i = 0; i < N; i++)
+ {
+ a[i] = (unsigned char)X;
+ b[i] = X;
+ c[i] = (unsigned int)X;
+ }
+
+ /* check results: */
+ for (i = 0; i < N; i++)
+ {
+ if (a[i] != (char)X || b[i] != X || c[i] != (int)X)
+ abort ();
+ }
+
+ return 0;
+}
+
+int main (void)
+{
+ check_vect ();
+
+ return main1 (3);
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { target { vect_pack_mod && vect_unpack } } } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
--- /dev/null
+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+#define N 26
+int a[N];
+
+int main1 (int X)
+{
+ int s = X;
+ int i;
+
+ /* vectorization of reduction with induction. */
+ for (i = 0; i < N; i++)
+ s += (i + a[i]);
+
+ return s;
+}
+
+int main (void)
+{
+ int s, i;
+ check_vect ();
+
+ for (i = 0; i < N; i++)
+ a[i] = 2*i;
+
+ s = main1 (3);
+ if (s != 978)
+ abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" } } */
+/* { dg-final { cleanup-tree-dump "vect" } } */
return 0;
}
+/* The initialization loop in main also gets vectorized. */
/* { dg-final { scan-tree-dump-times "vect_recog_dot_prod_pattern: detected" 1 "vect" { xfail *-*-* } } } */
-/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { target { vect_short_mult && vect_widen_sum_hi_to_si } } } } */
-
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { target { vect_short_mult && vect_widen_sum_hi_to_si } } } } */
/* { dg-final { cleanup-tree-dump "vect" } } */
/* Once the dot-product pattern is detected, we expect
that loop to be vectorized on vect_udot_hi targets (targets that support
dot-product of unsigned shorts) and targets that support widening multiplication. */
-/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail *-*-* } } } */
+/* The induction loop in main is vectorized. */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
/* { dg-final { cleanup-tree-dump "vect" } } */
return 0;
}
-/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail *-*-* } } } */
+/*The induction loop is vectorized */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 2 "vect" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */
/* { dg-final { cleanup-tree-dump "vect" } } */
dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/no-trapping-math-*.\[cS\]]] \
"" $DEFAULT_VECTCFLAGS
+# -fno-tree-scev-cprop
+set DEFAULT_VECTCFLAGS $SAVED_DEFAULT_VECTCFLAGS
+lappend DEFAULT_VECTCFLAGS "-fno-tree-scev-cprop"
+dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/no-tree-scev-cprop-*.\[cS\]]] \
+ "" $DEFAULT_VECTCFLAGS
+
# With -Os
lappend DEFAULT_VECTCFLAGS "-Os"
dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/Os-vect-*.\[cS\]]] \
static bool
gate_scev_const_prop (void)
{
- return true;
+ return flag_tree_scev_cprop;
}
struct tree_opt_pass pass_scev_cprop =
tree phi;
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
basic_block bb = loop->header;
- tree dummy;
+ tree dumy;
+ VEC(tree,heap) *worklist = VEC_alloc (tree, heap, 64);
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_analyze_scalar_cycles ===");
+ /* First - identify all inductions. */
for (phi = phi_nodes (bb); phi; phi = PHI_CHAIN (phi))
{
tree access_fn = NULL;
tree def = PHI_RESULT (phi);
stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi);
- tree reduc_stmt;
if (vect_print_dump_info (REPORT_DETAILS))
{
- fprintf (vect_dump, "Analyze phi: ");
- print_generic_expr (vect_dump, phi, TDF_SLIM);
+ fprintf (vect_dump, "Analyze phi: ");
+ print_generic_expr (vect_dump, phi, TDF_SLIM);
}
/* Skip virtual phi's. The data dependences that are associated with
virtual defs/uses (i.e., memory accesses) are analyzed elsewhere. */
-
if (!is_gimple_reg (SSA_NAME_VAR (def)))
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "virtual phi. skip.");
- continue;
- }
+ continue;
STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_unknown_def_type;
/* Analyze the evolution function. */
-
access_fn = analyze_scalar_evolution (loop, def);
+ if (access_fn && vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "Access function of PHI: ");
+ print_generic_expr (vect_dump, access_fn, TDF_SLIM);
+ }
- if (!access_fn)
- continue;
+ if (!access_fn
+ || !vect_is_simple_iv_evolution (loop->num, access_fn, &dumy, &dumy))
+ {
+ VEC_safe_push (tree, heap, worklist, phi);
+ continue;
+ }
if (vect_print_dump_info (REPORT_DETAILS))
- {
- fprintf (vect_dump, "Access function of PHI: ");
- print_generic_expr (vect_dump, access_fn, TDF_SLIM);
- }
+ fprintf (vect_dump, "Detected induction.");
+ STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_induction_def;
+ }
- if (vect_is_simple_iv_evolution (loop->num, access_fn, &dummy, &dummy))
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "Detected induction.");
- STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_induction_def;
- continue;
- }
- /* TODO: handle invariant phis */
+ /* Second - identify all reductions. */
+ while (VEC_length (tree, worklist) > 0)
+ {
+ tree phi = VEC_pop (tree, worklist);
+ tree def = PHI_RESULT (phi);
+ stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi);
+ tree reduc_stmt;
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "Analyze phi: ");
+ print_generic_expr (vect_dump, phi, TDF_SLIM);
+ }
+
+ gcc_assert (is_gimple_reg (SSA_NAME_VAR (def)));
+ gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_unknown_def_type);
reduc_stmt = vect_is_simple_reduction (loop, phi);
if (reduc_stmt)
else
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "Unknown def-use cycle pattern.");
-
}
+ VEC_free (tree, heap, worklist);
return;
}
tree scalar_type;
if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "=== vect_analyze_data_refs ===");
+ fprintf (vect_dump, "=== vect_analyze_data_refs ===\n");
compute_data_dependences_for_loop (loop, true,
&LOOP_VINFO_DATAREFS (loop_vinfo),
/* Update DR field in stmt_vec_info struct. */
stmt = DR_STMT (dr);
stmt_info = vinfo_for_stmt (stmt);
-
+
if (STMT_VINFO_DATA_REF (stmt_info))
{
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
/* case 2.2: */
if (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def)
- {
- gcc_assert (relevant == vect_unused_in_loop && live_p);
- relevant = vect_used_by_reduction;
- live_p = false;
- }
+ {
+ gcc_assert (relevant == vect_unused_in_loop && live_p);
+ relevant = vect_used_by_reduction;
+ live_p = false;
+ }
+ i = 0;
FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
{
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "worklist: examine use %d: ", i++);
+ print_generic_expr (vect_dump, use, TDF_SLIM);
+ }
+
/* case 1: we are only interested in uses that need to be vectorized.
Uses that are used for address computation are not considered
relevant.
continue;
if (!vect_is_simple_use (use, loop_vinfo, &def_stmt, &def, &dt))
- {
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
- fprintf (vect_dump, "not vectorized: unsupported use in stmt.");
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ fprintf (vect_dump, "not vectorized: unsupported use in stmt.");
VEC_free (tree, heap, worklist);
- return false;
+ return false;
}
if (!def_stmt || IS_EMPTY_STMT (def_stmt))
continue;
- if (vect_print_dump_info (REPORT_DETAILS))
- {
- fprintf (vect_dump, "worklist: examine use %d: ", i);
- print_generic_expr (vect_dump, use, TDF_SLIM);
- }
-
bb = bb_for_stmt (def_stmt);
- if (!flow_bb_inside_loop_p (loop, bb))
- continue;
+ if (!flow_bb_inside_loop_p (loop, bb))
+ continue;
/* case 2.1: the reduction-use does not mark the defining-phi
as relevant. */
&& TREE_CODE (def_stmt) == PHI_NODE)
continue;
+ if (dt == vect_induction_def && TREE_CODE (def_stmt) == PHI_NODE)
+ continue;
+
vect_mark_relevant (&worklist, def_stmt, relevant, live_p);
}
} /* while worklist */
new_name = get_name (scalar_dest);
if (!new_name)
new_name = "var_";
- vec_dest = vect_get_new_vect_var (type, vect_simple_var, new_name);
+ vec_dest = vect_get_new_vect_var (type, kind, new_name);
add_referenced_var (vec_dest);
return vec_dest;
}
+/* Function get_initial_def_for_induction
+
+ Input:
+ STMT - a stmt that performs an induction operation in the loop.
+ IV_PHI - the initial value of the induction variable
+
+ Output:
+ Return a vector variable, initialized with the first VF values of
+ the induction variable. E.g., for an iv with IV_PHI='X' and
+ evolution S, for a vector of 4 units, we want to return:
+ [X, X + S, X + 2*S, X + 3*S]. */
+
+static tree
+get_initial_def_for_induction (tree stmt, tree iv_phi)
+{
+ stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ tree scalar_type = TREE_TYPE (iv_phi);
+ tree vectype = get_vectype_for_scalar_type (scalar_type);
+ int nunits = GET_MODE_NUNITS (TYPE_MODE (vectype));
+ edge pe = loop_preheader_edge (loop);
+ basic_block new_bb;
+ block_stmt_iterator bsi;
+ tree vec, vec_init, vec_step, t;
+ tree access_fn;
+ tree new_var;
+ tree new_name;
+ tree init_stmt;
+ tree induction_phi, induc_def, new_stmt, vec_def, vec_dest;
+ tree init_expr, step_expr;
+ int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ int i;
+ bool ok;
+ int ncopies = vf / nunits;
+ tree expr;
+ stmt_vec_info phi_info = vinfo_for_stmt (iv_phi);
+
+ gcc_assert (phi_info);
+
+ if (STMT_VINFO_VEC_STMT (phi_info))
+ {
+ induction_phi = STMT_VINFO_VEC_STMT (phi_info);
+ gcc_assert (TREE_CODE (induction_phi) == PHI_NODE);
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "induction already vectorized:");
+ print_generic_expr (vect_dump, iv_phi, TDF_SLIM);
+ fprintf (vect_dump, "\n");
+ print_generic_expr (vect_dump, induction_phi, TDF_SLIM);
+ }
+
+ return PHI_RESULT (induction_phi);
+ }
+
+ gcc_assert (ncopies >= 1);
+
+ access_fn = analyze_scalar_evolution (loop, PHI_RESULT (iv_phi));
+ gcc_assert (access_fn);
+ ok = vect_is_simple_iv_evolution (loop->num, access_fn, &init_expr, &step_expr);
+ gcc_assert (ok);
+
+ /* Create the vector that holds the initial_value of the induction. */
+ new_name = init_expr;
+ t = NULL_TREE;
+ t = tree_cons (NULL_TREE, init_expr, t);
+ for (i = 1; i < nunits; i++)
+ {
+ /* Create: new_name = new_name + step_expr */
+ new_var = vect_get_new_vect_var (scalar_type, vect_scalar_var, "var_");
+ add_referenced_var (new_var);
+ init_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, new_var,
+ fold_build2 (PLUS_EXPR, scalar_type, new_name, step_expr));
+ new_name = make_ssa_name (new_var, init_stmt);
+ GIMPLE_STMT_OPERAND (init_stmt, 0) = new_name;
+
+ new_bb = bsi_insert_on_edge_immediate (pe, init_stmt);
+ gcc_assert (!new_bb);
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "created new init_stmt: ");
+ print_generic_expr (vect_dump, init_stmt, TDF_SLIM);
+ }
+ t = tree_cons (NULL_TREE, new_name, t);
+ }
+ vec = build_constructor_from_list (vectype, nreverse (t));
+ vec_init = vect_init_vector (stmt, vec, vectype);
+
+
+ /* Create the vector that holds the step of the induction. */
+ expr = build_int_cst (scalar_type, vf);
+ new_name = fold_build2 (MULT_EXPR, scalar_type, expr, step_expr);
+ t = NULL_TREE;
+ for (i = 0; i < nunits; i++)
+ t = tree_cons (NULL_TREE, unshare_expr (new_name), t);
+ vec = build_constructor_from_list (vectype, t);
+ vec_step = vect_init_vector (stmt, vec, vectype);
+
+
+ /* Create the following def-use cycle:
+ loop prolog:
+ vec_init = [X, X+S, X+2*S, X+3*S]
+ vec_step = [VF*S, VF*S, VF*S, VF*S]
+ loop:
+ vec_iv = PHI <vec_init, vec_loop>
+ ...
+ STMT
+ ...
+ vec_loop = vec_iv + vec_step; */
+
+ /* Create the induction-phi that defines the induction-operand. */
+ vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_");
+ add_referenced_var (vec_dest);
+ induction_phi = create_phi_node (vec_dest, loop->header);
+ set_stmt_info (get_stmt_ann (induction_phi),
+ new_stmt_vec_info (induction_phi, loop_vinfo));
+ induc_def = PHI_RESULT (induction_phi);
+
+ /* Create the iv update inside the loop */
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, NULL_TREE,
+ build2 (PLUS_EXPR, vectype, induc_def, vec_step));
+ vec_def = make_ssa_name (vec_dest, new_stmt);
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = vec_def;
+ bsi = bsi_for_stmt (stmt);
+ vect_finish_stmt_generation (stmt, new_stmt, &bsi);
+
+ /* Set the arguments of the phi node: */
+ add_phi_arg (induction_phi, vec_init, loop_preheader_edge (loop));
+ add_phi_arg (induction_phi, vec_def, loop_latch_edge (loop));
+
+
+ /* In case the vectorization factor (VF) is bigger than the number
+ of elements that we can fit in a vectype (nunits), we have to generate
+ more than one vector stmt - i.e - we need to "unroll" the
+ vector stmt by a factor VF/nunits. For more details see documentation
+ in vectorizable_operation. */
+
+ if (ncopies > 1)
+ {
+ stmt_vec_info prev_stmt_vinfo;
+
+ /* Create the vector that holds the step of the induction. */
+ expr = build_int_cst (scalar_type, nunits);
+ new_name = fold_build2 (MULT_EXPR, scalar_type, expr, step_expr);
+ t = NULL_TREE;
+ for (i = 0; i < nunits; i++)
+ t = tree_cons (NULL_TREE, unshare_expr (new_name), t);
+ vec = build_constructor_from_list (vectype, t);
+ vec_step = vect_init_vector (stmt, vec, vectype);
+
+ vec_def = induc_def;
+ prev_stmt_vinfo = vinfo_for_stmt (induction_phi);
+ for (i = 1; i < ncopies; i++)
+ {
+ /* vec_i = vec_prev + vec_{step*nunits} */
+
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, NULL_TREE,
+ build2 (PLUS_EXPR, vectype, vec_def, vec_step));
+ vec_def = make_ssa_name (vec_dest, new_stmt);
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = vec_def;
+ bsi = bsi_for_stmt (stmt);
+ vect_finish_stmt_generation (stmt, new_stmt, &bsi);
+
+ STMT_VINFO_RELATED_STMT (prev_stmt_vinfo) = new_stmt;
+ prev_stmt_vinfo = vinfo_for_stmt (new_stmt);
+ }
+ }
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "transform induction: created def-use cycle:");
+ print_generic_expr (vect_dump, induction_phi, TDF_SLIM);
+ fprintf (vect_dump, "\n");
+ print_generic_expr (vect_dump, SSA_NAME_DEF_STMT (vec_def), TDF_SLIM);
+ }
+
+ STMT_VINFO_VEC_STMT (phi_info) = induction_phi;
+ return induc_def;
+}
+
+
/* Function vect_get_vec_def_for_operand.
OP is an operand in STMT. This function returns a (vector) def that will be
/* Case 5: operand is defined by loop-header phi - induction. */
case vect_induction_def:
{
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "induction - unsupported.");
- internal_error ("no support for induction"); /* FORNOW */
+ gcc_assert (TREE_CODE (def_stmt) == PHI_NODE);
+
+ /* Get the def before the loop */
+ return get_initial_def_for_induction (stmt, def_stmt);
}
default:
tree vec_stmt_for_operand;
stmt_vec_info def_stmt_info;
- if (dt == vect_invariant_def || dt == vect_constant_def)
- {
- /* Do nothing; can reuse same def. */ ;
- return vec_oprnd;
- }
+ /* Do nothing; can reuse same def. */
+ if (dt == vect_invariant_def || dt == vect_constant_def )
+ return vec_oprnd;
vec_stmt_for_operand = SSA_NAME_DEF_STMT (vec_oprnd);
def_stmt_info = vinfo_for_stmt (vec_stmt_for_operand);
+ if (dt == vect_induction_def)
+ gcc_assert (TREE_CODE (vec_stmt_for_operand) == PHI_NODE);
gcc_assert (def_stmt_info);
vec_stmt_for_operand = STMT_VINFO_RELATED_STMT (def_stmt_info);
gcc_assert (vec_stmt_for_operand);
op = TREE_OPERAND (operation, i);
is_simple_use = vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt);
gcc_assert (is_simple_use);
- gcc_assert (dt == vect_loop_def || dt == vect_invariant_def ||
- dt == vect_constant_def);
+ if (dt != vect_loop_def
+ && dt != vect_invariant_def
+ && dt != vect_constant_def
+ && dt != vect_induction_def)
+ return false;
}
op = TREE_OPERAND (operation, i);
/* Handle uses. */
if (j == 0)
{
- enum vect_def_type dt = vect_unknown_def_type; /* Dummy */
vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
- vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt, vec_oprnd0);
+ vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt0, vec_oprnd0);
}
else
{
return false;
}
- if (*dt == vect_induction_def)
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "induction not supported.");
- return false;
- }
-
return true;
}
*/
def1 = SSA_NAME_DEF_STMT (op1);
def2 = SSA_NAME_DEF_STMT (op2);
- if (!def1 || !def2)
+ if (!def1 || !def2 || IS_EMPTY_STMT (def1) || IS_EMPTY_STMT (def2))
{
if (vect_print_dump_info (REPORT_DETAILS))
{
return NULL_TREE;
}
- if (TREE_CODE (def1) == GIMPLE_MODIFY_STMT
+
+ /* Check that one def is the reduction def, defined by PHI,
+ the other def is either defined in the loop by a GIMPLE_MODIFY_STMT,
+ or it's an induction (defined by some phi node). */
+
+ if (def2 == phi
&& flow_bb_inside_loop_p (loop, bb_for_stmt (def1))
- && def2 == phi)
+ && (TREE_CODE (def1) == GIMPLE_MODIFY_STMT
+ || STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1)) == vect_induction_def))
{
if (vect_print_dump_info (REPORT_DETAILS))
{
}
return def_stmt;
}
- else if (TREE_CODE (def2) == GIMPLE_MODIFY_STMT
- && flow_bb_inside_loop_p (loop, bb_for_stmt (def2))
- && def1 == phi)
+ else if (def1 == phi
+ && flow_bb_inside_loop_p (loop, bb_for_stmt (def2))
+ && (TREE_CODE (def2) == GIMPLE_MODIFY_STMT
+ || STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2)) == vect_induction_def))
{
/* Swap operands (just for simplicity - so that the rest of the code
can assume that the reduction variable is always the last (second)
{
tree init_expr;
tree step_expr;
-
tree evolution_part = evolution_part_in_loop_num (access_fn, loop_nb);
/* When there is no evolution in this loop, the evolution function
return false;
step_expr = evolution_part;
- init_expr = unshare_expr (initial_condition_in_loop_num (access_fn,
- loop_nb));
+ init_expr = unshare_expr (initial_condition_in_loop_num (access_fn, loop_nb));
if (vect_print_dump_info (REPORT_DETAILS))
{
*step = step_expr;
if (TREE_CODE (step_expr) != INTEGER_CST)
- {
+ {
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "step unknown.");
return false;
projects / platform / upstream / gcc.git / commitdiff