expand_ASSUME (internal_fn, gcall *)
{
}
+
+void
+expand_MASK_CALL (internal_fn, gcall *)
+{
+ /* This IFN should only exist between ifcvt and vect passes. */
+ gcc_unreachable ();
+}
DEF_INTERNAL_FN (ASSUME, ECF_CONST | ECF_LEAF | ECF_NOTHROW
| ECF_LOOPING_CONST_OR_PURE, NULL)
+/* For if-conversion of inbranch SIMD clones. */
+DEF_INTERNAL_FN (MASK_CALL, ECF_NOVOPS, NULL)
+
#undef DEF_INTERNAL_INT_FN
#undef DEF_INTERNAL_FLT_FN
#undef DEF_INTERNAL_FLT_FLOATN_FN
extern void expand_SPACESHIP (internal_fn, gcall *);
extern void expand_TRAP (internal_fn, gcall *);
extern void expand_ASSUME (internal_fn, gcall *);
+extern void expand_MASK_CALL (internal_fn, gcall *);
extern bool vectorized_internal_fn_supported_p (internal_fn, tree);
}
sc->args[i].orig_type = base_type;
sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
+ sc->args[i].vector_type = adj.type;
}
if (node->definition)
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+/* Test that simd inbranch clones work correctly. */
+
+#ifndef TYPE
+#define TYPE int
+#endif
+
+/* A simple function that will be cloned. */
+#pragma omp declare simd
+TYPE __attribute__((noinline))
+foo (TYPE a)
+{
+ return a + 1;
+}
+
+/* Check that "inbranch" clones are called correctly. */
+
+void __attribute__((noipa))
+masked (TYPE * __restrict a, TYPE * __restrict b, int size)
+{
+ #pragma omp simd
+ for (int i = 0; i < size; i++)
+ b[i] = a[i]<1 ? foo(a[i]) : a[i];
+}
+
+/* Check that "inbranch" works when there might be unrolling. */
+
+void __attribute__((noipa))
+masked_fixed (TYPE * __restrict a, TYPE * __restrict b)
+{
+ #pragma omp simd
+ for (int i = 0; i < 128; i++)
+ b[i] = a[i]<1 ? foo(a[i]) : a[i];
+}
+
+/* Validate the outputs. */
+
+void
+check_masked (TYPE *b, int size)
+{
+ for (int i = 0; i < size; i++)
+ if (((TYPE)i < 1 && b[i] != (TYPE)(i + 1))
+ || ((TYPE)i >= 1 && b[i] != (TYPE)i))
+ {
+ __builtin_printf ("error at %d\n", i);
+ __builtin_exit (1);
+ }
+}
+
+int
+main ()
+{
+ TYPE a[1024];
+ TYPE b[1024];
+
+ for (int i = 0; i < 1024; i++)
+ a[i] = i;
+
+ masked_fixed (a, b);
+ check_masked (b, 128);
+
+ /* Test various sizes to cover machines with different vectorization
+ factors. */
+ for (int size = 8; size <= 1024; size *= 2)
+ {
+ masked (a, b, size);
+ check_masked (b, size);
+ }
+
+ /* Test sizes that might exercise the partial vector code-path. */
+ for (int size = 8; size <= 1024; size *= 2)
+ {
+ masked (a, b, size-4);
+ check_masked (b, size-4);
+ }
+
+ return 0;
+}
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use another call for the epilogue loops. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { ! aarch64*-*-* } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 3 "vect" { target aarch64*-*-* } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+#define TYPE float
+#include "vect-simd-clone-16.c"
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use another call for the epilogue loops. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { ! { avx_runtime || aarch64*-*-* } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 3 "vect" { target { avx_runtime || aarch64*-*-* } } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+#define TYPE short
+#include "vect-simd-clone-16.c"
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use another call for the epilogue loops. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { ! { x86_64*-*-* || { i686*-*-* || aarch64*-*-* } } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 3 "vect" { target { aarch64*-*-* } } } } */
+
+/* x86_64 fails to use in-branch clones for TYPE=short. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 0 "vect" { target x86_64*-*-* i686*-*-* } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+#define TYPE char
+#include "vect-simd-clone-16.c"
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use another call for the epilogue loops. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { ! { x86_64*-*-* || { i686*-*-* || aarch64*-*-* } } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 3 "vect" { target { aarch64*-*-* } } } } */
+
+/* x86_64 fails to use in-branch clones for TYPE=char. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 0 "vect" { target x86_64*-*-* i686*-*-* } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+#define TYPE double
+#include "vect-simd-clone-16.c"
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use another call for the epilogue loops.
+ Some targets use pairs of vectors and do twice the calls. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { { ! avx_runtime } && { ! { i686*-*-* && { ! lp64 } } } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 3 "vect" { target { avx_runtime && { ! { i686*-*-* && { ! lp64 } } } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 4 "vect" { target { i686*-*-* && { ! lp64 } } } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+#define TYPE __INT64_TYPE__
+#include "vect-simd-clone-16.c"
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use pairs of vectors and do twice the calls. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { ! { i686*-*-* && { ! lp64 } } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 4 "vect" { target { i686*-*-* && { ! lp64 } } } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+/* Test that simd inbranch clones work correctly. */
+
+#ifndef TYPE
+#define TYPE int
+#endif
+
+/* A simple function that will be cloned. */
+#pragma omp declare simd uniform(b)
+TYPE __attribute__((noinline))
+foo (TYPE a, TYPE b)
+{
+ return a + b;
+}
+
+/* Check that "inbranch" clones are called correctly. */
+
+void __attribute__((noipa))
+masked (TYPE * __restrict a, TYPE * __restrict b, int size)
+{
+ #pragma omp simd
+ for (int i = 0; i < size; i++)
+ b[i] = a[i]<1 ? foo(a[i], 1) : a[i];
+}
+
+/* Check that "inbranch" works when there might be unrolling. */
+
+void __attribute__((noipa))
+masked_fixed (TYPE * __restrict a, TYPE * __restrict b)
+{
+ #pragma omp simd
+ for (int i = 0; i < 128; i++)
+ b[i] = a[i]<1 ? foo(a[i], 1) : a[i];
+}
+
+/* Validate the outputs. */
+
+void
+check_masked (TYPE *b, int size)
+{
+ for (int i = 0; i < size; i++)
+ if (((TYPE)i < 1 && b[i] != (TYPE)(i + 1))
+ || ((TYPE)i >= 1 && b[i] != (TYPE)i))
+ {
+ __builtin_printf ("error at %d\n", i);
+ __builtin_exit (1);
+ }
+}
+
+int
+main ()
+{
+ TYPE a[1024];
+ TYPE b[1024];
+
+ for (int i = 0; i < 1024; i++)
+ a[i] = i;
+
+ masked_fixed (a, b);
+ check_masked (b, 128);
+
+ /* Test various sizes to cover machines with different vectorization
+ factors. */
+ for (int size = 8; size <= 1024; size *= 2)
+ {
+ masked (a, b, size);
+ check_masked (b, size);
+ }
+
+ /* Test sizes that might exercise the partial vector code-path. */
+ for (int size = 8; size <= 1024; size *= 2)
+ {
+ masked (a, b, size-4);
+ check_masked (b, size-4);
+ }
+
+ return 0;
+}
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use another call for the epilogue loops. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { ! aarch64*-*-* } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 3 "vect" { target aarch64*-*-* } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+#define TYPE float
+#include "vect-simd-clone-17.c"
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use another call for the epilogue loops. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { ! { avx_runtime || aarch64*-*-* } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 3 "vect" { target { avx_runtime || aarch64*-*-* } } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+#define TYPE short
+#include "vect-simd-clone-17.c"
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use another call for the epilogue loops. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { ! { x86_64*-*-* || { i686*-*-* || aarch64*-*-* } } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 3 "vect" { target { aarch64*-*-* } } } } */
+
+/* x86_64 fails to use in-branch clones for TYPE=short. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 0 "vect" { target x86_64*-*-* i686*-*-* } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd -fdump-tree-optimized" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+#define TYPE char
+#include "vect-simd-clone-17.c"
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use another call for the epilogue loops. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { ! { x86_64*-*-* || { i686*-*-* || aarch64*-*-* } } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 3 "vect" { target { aarch64*-*-* } } } } */
+
+/* x86_64 fails to use in-branch clones for TYPE=char. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 0 "vect" { target x86_64*-*-* i686*-*-* } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd -fdump-tree-optimized" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+#define TYPE double
+#include "vect-simd-clone-17.c"
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use another call for the epilogue loops.
+ Some targets use pairs of vectors and do twice the calls. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { { ! avx_runtime } && { ! { i686*-*-* && { ! lp64 } } } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 3 "vect" { target { avx_runtime && { ! { i686*-*-* && { ! lp64 } } } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 4 "vect" { target { i686*-*-* && { ! lp64 } } } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd -fdump-tree-optimized" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+#define TYPE __INT64_TYPE__
+#include "vect-simd-clone-17.c"
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use pairs of vectors and do twice the calls. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { ! { i686*-*-* && { ! lp64 } } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 4 "vect" { target { i686*-*-* && { ! lp64 } } } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+/* Test that simd inbranch clones work correctly. */
+
+#ifndef TYPE
+#define TYPE int
+#endif
+
+/* A simple function that will be cloned. */
+#pragma omp declare simd uniform(b)
+TYPE __attribute__((noinline))
+foo (TYPE b, TYPE a)
+{
+ return a + b;
+}
+
+/* Check that "inbranch" clones are called correctly. */
+
+void __attribute__((noipa))
+masked (TYPE * __restrict a, TYPE * __restrict b, int size)
+{
+ #pragma omp simd
+ for (int i = 0; i < size; i++)
+ b[i] = a[i]<1 ? foo(1, a[i]) : a[i];
+}
+
+/* Check that "inbranch" works when there might be unrolling. */
+
+void __attribute__((noipa))
+masked_fixed (TYPE * __restrict a, TYPE * __restrict b)
+{
+ #pragma omp simd
+ for (int i = 0; i < 128; i++)
+ b[i] = a[i]<1 ? foo(1, a[i]) : a[i];
+}
+
+/* Validate the outputs. */
+
+void
+check_masked (TYPE *b, int size)
+{
+ for (int i = 0; i < size; i++)
+ if (((TYPE)i < 1 && b[i] != (TYPE)(i + 1))
+ || ((TYPE)i >= 1 && b[i] != (TYPE)i))
+ {
+ __builtin_printf ("error at %d\n", i);
+ __builtin_exit (1);
+ }
+}
+
+int
+main ()
+{
+ TYPE a[1024];
+ TYPE b[1024];
+
+ for (int i = 0; i < 1024; i++)
+ a[i] = i;
+
+ masked_fixed (a, b);
+ check_masked (b, 128);
+
+ /* Test various sizes to cover machines with different vectorization
+ factors. */
+ for (int size = 8; size <= 1024; size *= 2)
+ {
+ masked (a, b, size);
+ check_masked (b, size);
+ }
+
+ /* Test sizes that might exercise the partial vector code-path. */
+ for (int size = 8; size <= 1024; size *= 2)
+ {
+ masked (a, b, size-4);
+ check_masked (b, size-4);
+ }
+
+ return 0;
+}
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use another call for the epilogue loops. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { ! aarch64*-*-* } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 3 "vect" { target aarch64*-*-* } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+#define TYPE float
+#include "vect-simd-clone-18.c"
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use another call for the epilogue loops. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { ! { avx_runtime || aarch64*-*-* } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 3 "vect" { target { avx_runtime || aarch64*-*-* } } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+#define TYPE short
+#include "vect-simd-clone-18.c"
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use another call for the epilogue loops. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { ! { x86_64*-*-* || { i686*-*-* || aarch64*-*-* } } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 3 "vect" { target { aarch64*-*-* } } } } */
+
+/* x86_64 fails to use in-branch clones for TYPE=short. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 0 "vect" { target x86_64*-*-* i686*-*-* } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+#define TYPE char
+#include "vect-simd-clone-18.c"
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use another call for the epilogue loops. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { ! { x86_64*-*-* || { i686*-*-* || aarch64*-*-* } } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 3 "vect" { target { aarch64*-*-* } } } } */
+
+/* x86_64 fails to use in-branch clones for TYPE=char. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 0 "vect" { target x86_64*-*-* i686*-*-* } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+#define TYPE double
+#include "vect-simd-clone-18.c"
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use another call for the epilogue loops.
+ Some targets use pairs of vectors and do twice the calls. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { { ! avx_runtime } && { ! { i686*-*-* && { ! lp64 } } } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 3 "vect" { target { avx_runtime && { ! { i686*-*-* && { ! lp64 } } } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 4 "vect" { target { i686*-*-* && { ! lp64 } } } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
--- /dev/null
+/* { dg-require-effective-target vect_simd_clones } */
+/* { dg-additional-options "-fopenmp-simd" } */
+/* { dg-additional-options "-mavx" { target avx_runtime } } */
+
+#define TYPE __INT64_TYPE__
+#include "vect-simd-clone-18.c"
+
+/* Ensure the the in-branch simd clones are used on targets that support them.
+ Some targets use pairs of vectors and do twice the calls. */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 2 "vect" { target { ! { i686*-*-* && { ! lp64 } } } } } } */
+/* { dg-final { scan-tree-dump-times {[\n\r] [^\n]* = foo\.simdclone} 4 "vect" { target { i686*-*-* && { ! lp64 } } } } } */
+
+/* The LTO test produces two dump files and we scan the wrong one. */
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
#include "tree-ssa-dse.h"
#include "tree-vectorizer.h"
#include "tree-eh.h"
+#include "cgraph.h"
/* For lang_hooks.types.type_for_mode. */
#include "langhooks.h"
A statement is if-convertible if:
- it is an if-convertible GIMPLE_ASSIGN,
- it is a GIMPLE_LABEL or a GIMPLE_COND,
- - it is builtins call. */
+ - it is builtins call,
+ - it is a call to a function with a SIMD clone. */
static bool
if_convertible_stmt_p (gimple *stmt, vec<data_reference_p> refs)
tree fndecl = gimple_call_fndecl (stmt);
if (fndecl)
{
+ /* We can vectorize some builtins and functions with SIMD
+ "inbranch" clones. */
int flags = gimple_call_flags (stmt);
+ struct cgraph_node *node = cgraph_node::get (fndecl);
if ((flags & ECF_CONST)
&& !(flags & ECF_LOOPING_CONST_OR_PURE)
- /* We can only vectorize some builtins at the moment,
- so restrict if-conversion to those. */
&& fndecl_built_in_p (fndecl))
return true;
+ if (node && node->simd_clones != NULL)
+ /* Ensure that at least one clone can be "inbranch". */
+ for (struct cgraph_node *n = node->simd_clones; n != NULL;
+ n = n->simdclone->next_clone)
+ if (n->simdclone->inbranch)
+ {
+ need_to_predicate = true;
+ return true;
+ }
}
return false;
}
gimple_assign_set_rhs1 (stmt, ifc_temp_var (type, rhs, &gsi));
update_stmt (stmt);
}
+
+ /* Convert functions that have a SIMD clone to IFN_MASK_CALL. This
+ will cause the vectorizer to match the "in branch" clone variants,
+ and serves to build the mask vector in a natural way. */
+ gcall *call = dyn_cast <gcall *> (gsi_stmt (gsi));
+ if (call && !gimple_call_internal_p (call))
+ {
+ tree orig_fn = gimple_call_fn (call);
+ int orig_nargs = gimple_call_num_args (call);
+ auto_vec<tree> args;
+ args.safe_push (orig_fn);
+ for (int i = 0; i < orig_nargs; i++)
+ args.safe_push (gimple_call_arg (call, i));
+ args.safe_push (cond);
+
+ /* Replace the call with a IFN_MASK_CALL that has the extra
+ condition parameter. */
+ gcall *new_call = gimple_build_call_internal_vec (IFN_MASK_CALL,
+ args);
+ gimple_call_set_lhs (new_call, gimple_call_lhs (call));
+ gsi_replace (&gsi, new_call, true);
+ }
+
lhs = gimple_get_lhs (gsi_stmt (gsi));
if (lhs && TREE_CODE (lhs) == SSA_NAME)
ssa_names.add (lhs);
if (is_gimple_call (stmt) && loop->safelen)
{
tree fndecl = gimple_call_fndecl (stmt), op;
+ if (fndecl == NULL_TREE
+ && gimple_call_internal_p (stmt, IFN_MASK_CALL))
+ {
+ fndecl = gimple_call_arg (stmt, 0);
+ gcc_checking_assert (TREE_CODE (fndecl) == ADDR_EXPR);
+ fndecl = TREE_OPERAND (fndecl, 0);
+ gcc_checking_assert (TREE_CODE (fndecl) == FUNCTION_DECL);
+ }
if (fndecl != NULL_TREE)
{
cgraph_node *node = cgraph_node::get (fndecl);
size_t i, nargs;
tree lhs, rtype, ratype;
vec<constructor_elt, va_gc> *ret_ctor_elts = NULL;
+ int arg_offset = 0;
/* Is STMT a vectorizable call? */
gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt);
return false;
fndecl = gimple_call_fndecl (stmt);
+ if (fndecl == NULL_TREE
+ && gimple_call_internal_p (stmt, IFN_MASK_CALL))
+ {
+ fndecl = gimple_call_arg (stmt, 0);
+ gcc_checking_assert (TREE_CODE (fndecl) == ADDR_EXPR);
+ fndecl = TREE_OPERAND (fndecl, 0);
+ gcc_checking_assert (TREE_CODE (fndecl) == FUNCTION_DECL);
+ arg_offset = 1;
+ }
if (fndecl == NULL_TREE)
return false;
return false;
/* Process function arguments. */
- nargs = gimple_call_num_args (stmt);
+ nargs = gimple_call_num_args (stmt) - arg_offset;
/* Bail out if the function has zero arguments. */
if (nargs == 0)
thisarginfo.op = NULL_TREE;
thisarginfo.simd_lane_linear = false;
- op = gimple_call_arg (stmt, i);
+ op = gimple_call_arg (stmt, i + arg_offset);
if (!vect_is_simple_use (op, vinfo, &thisarginfo.dt,
&thisarginfo.vectype)
|| thisarginfo.dt == vect_uninitialized_def)
|| thisarginfo.dt == vect_external_def)
gcc_assert (thisarginfo.vectype == NULL_TREE);
else
- {
- gcc_assert (thisarginfo.vectype != NULL_TREE);
- if (VECTOR_BOOLEAN_TYPE_P (thisarginfo.vectype))
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "vector mask arguments are not supported\n");
- return false;
- }
- }
+ gcc_assert (thisarginfo.vectype != NULL_TREE);
/* For linear arguments, the analyze phase should have saved
the base and step in STMT_VINFO_SIMD_CLONE_INFO. */
if (target_badness < 0)
continue;
this_badness += target_badness * 512;
- /* FORNOW: Have to add code to add the mask argument. */
- if (n->simdclone->inbranch)
- continue;
for (i = 0; i < nargs; i++)
{
switch (n->simdclone->args[i].arg_type)
case SIMD_CLONE_ARG_TYPE_VECTOR:
if (!useless_type_conversion_p
(n->simdclone->args[i].orig_type,
- TREE_TYPE (gimple_call_arg (stmt, i))))
+ TREE_TYPE (gimple_call_arg (stmt, i + arg_offset))))
i = -1;
else if (arginfo[i].dt == vect_constant_def
|| arginfo[i].dt == vect_external_def
i = -1;
break;
case SIMD_CLONE_ARG_TYPE_MASK:
- gcc_unreachable ();
+ break;
}
if (i == (size_t) -1)
break;
return false;
for (i = 0; i < nargs; i++)
- if ((arginfo[i].dt == vect_constant_def
- || arginfo[i].dt == vect_external_def)
- && bestn->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
- {
- tree arg_type = TREE_TYPE (gimple_call_arg (stmt, i));
- arginfo[i].vectype = get_vectype_for_scalar_type (vinfo, arg_type,
- slp_node);
- if (arginfo[i].vectype == NULL
- || !constant_multiple_p (bestn->simdclone->simdlen,
- simd_clone_subparts (arginfo[i].vectype)))
+ {
+ if ((arginfo[i].dt == vect_constant_def
+ || arginfo[i].dt == vect_external_def)
+ && bestn->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
+ {
+ tree arg_type = TREE_TYPE (gimple_call_arg (stmt, i + arg_offset));
+ arginfo[i].vectype = get_vectype_for_scalar_type (vinfo, arg_type,
+ slp_node);
+ if (arginfo[i].vectype == NULL
+ || !constant_multiple_p (bestn->simdclone->simdlen,
+ simd_clone_subparts (arginfo[i].vectype)))
+ return false;
+ }
+
+ if (bestn->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR
+ && VECTOR_BOOLEAN_TYPE_P (bestn->simdclone->args[i].vector_type))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "vector mask arguments are not supported.\n");
return false;
- }
+ }
+
+ if (bestn->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK
+ && bestn->simdclone->mask_mode == VOIDmode
+ && (simd_clone_subparts (bestn->simdclone->args[i].vector_type)
+ != simd_clone_subparts (arginfo[i].vectype)))
+ {
+ /* FORNOW we only have partial support for vector-type masks that
+ can't hold all of simdlen. */
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION,
+ vect_location,
+ "in-branch vector clones are not yet"
+ " supported for mismatched vector sizes.\n");
+ return false;
+ }
+ if (bestn->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK
+ && bestn->simdclone->mask_mode != VOIDmode)
+ {
+ /* FORNOW don't support integer-type masks. */
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION,
+ vect_location,
+ "in-branch vector clones are not yet"
+ " supported for integer mask modes.\n");
+ return false;
+ }
+ }
fndecl = bestn->decl;
nunits = bestn->simdclone->simdlen;
{
unsigned int k, l, m, o;
tree atype;
- op = gimple_call_arg (stmt, i);
+ op = gimple_call_arg (stmt, i + arg_offset);
switch (bestn->simdclone->args[i].arg_type)
{
case SIMD_CLONE_ARG_TYPE_VECTOR:
}
}
break;
+ case SIMD_CLONE_ARG_TYPE_MASK:
+ atype = bestn->simdclone->args[i].vector_type;
+ if (bestn->simdclone->mask_mode != VOIDmode)
+ {
+ /* FORNOW: this is disabled above. */
+ gcc_unreachable ();
+ }
+ else
+ {
+ tree elt_type = TREE_TYPE (atype);
+ tree one = fold_convert (elt_type, integer_one_node);
+ tree zero = fold_convert (elt_type, integer_zero_node);
+ o = vector_unroll_factor (nunits,
+ simd_clone_subparts (atype));
+ for (m = j * o; m < (j + 1) * o; m++)
+ {
+ if (simd_clone_subparts (atype)
+ < simd_clone_subparts (arginfo[i].vectype))
+ {
+ /* The mask type has fewer elements than simdlen. */
+
+ /* FORNOW */
+ gcc_unreachable ();
+ }
+ else if (simd_clone_subparts (atype)
+ == simd_clone_subparts (arginfo[i].vectype))
+ {
+ /* The SIMD clone function has the same number of
+ elements as the current function. */
+ if (m == 0)
+ {
+ vect_get_vec_defs_for_operand (vinfo, stmt_info,
+ o * ncopies,
+ op,
+ &vec_oprnds[i]);
+ vec_oprnds_i[i] = 0;
+ }
+ vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
+ vec_oprnd0
+ = build3 (VEC_COND_EXPR, atype, vec_oprnd0,
+ build_vector_from_val (atype, one),
+ build_vector_from_val (atype, zero));
+ gassign *new_stmt
+ = gimple_build_assign (make_ssa_name (atype),
+ vec_oprnd0);
+ vect_finish_stmt_generation (vinfo, stmt_info,
+ new_stmt, gsi);
+ vargs.safe_push (gimple_assign_lhs (new_stmt));
+ }
+ else
+ {
+ /* The mask type has more elements than simdlen. */
+
+ /* FORNOW */
+ gcc_unreachable ();
+ }
+ }
+ }
+ break;
case SIMD_CLONE_ARG_TYPE_UNIFORM:
vargs.safe_push (op);
break;
projects / platform / upstream / gcc.git / commitdiff