+2016-04-15 Thomas Schwinge <thomas@codesourcery.com>
+
+ * omp-low.c (simd_clone_struct_alloc, simd_clone_struct_copy)
+ (simd_clone_vector_of_formal_parm_types)
+ (simd_clone_clauses_extract, simd_clone_compute_base_data_type)
+ (simd_clone_mangle, simd_clone_create)
+ (simd_clone_adjust_return_type, create_tmp_simd_array)
+ (simd_clone_adjust_argument_types, simd_clone_init_simd_arrays)
+ (struct modify_stmt_info, ipa_simd_modify_stmt_ops)
+ (ipa_simd_modify_function_body, simd_clone_linear_addend)
+ (simd_clone_adjust, expand_simd_clones, ipa_omp_simd_clone)
+ (pass_data_omp_simd_clone, class pass_omp_simd_clone)
+ (pass_omp_simd_clone::gate, make_pass_omp_simd_clone): Move
+ into...
+ * omp-simd-clone.c: ... this new file.
+ (simd_clone_vector_of_formal_parm_types): Make it static.
+ * Makefile.in (OBJS): Add omp-simd-clone.o.
+
2016-04-15 Kirill Yukhin <kirill.yukhin@intel.com>
PR target/70662
modulo-sched.o \
multiple_target.o \
omp-low.o \
+ omp-simd-clone.o \
optabs.o \
optabs-libfuncs.o \
optabs-query.o \
return new pass_diagnose_omp_blocks (ctxt);
}
\f
-/* SIMD clone supporting code. */
-
-/* Allocate a fresh `simd_clone' and return it. NARGS is the number
- of arguments to reserve space for. */
-
-static struct cgraph_simd_clone *
-simd_clone_struct_alloc (int nargs)
-{
- struct cgraph_simd_clone *clone_info;
- size_t len = (sizeof (struct cgraph_simd_clone)
- + nargs * sizeof (struct cgraph_simd_clone_arg));
- clone_info = (struct cgraph_simd_clone *)
- ggc_internal_cleared_alloc (len);
- return clone_info;
-}
-
-/* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
-
-static inline void
-simd_clone_struct_copy (struct cgraph_simd_clone *to,
- struct cgraph_simd_clone *from)
-{
- memcpy (to, from, (sizeof (struct cgraph_simd_clone)
- + ((from->nargs - from->inbranch)
- * sizeof (struct cgraph_simd_clone_arg))));
-}
-
-/* Return vector of parameter types of function FNDECL. This uses
- TYPE_ARG_TYPES if available, otherwise falls back to types of
- DECL_ARGUMENTS types. */
-
-vec<tree>
-simd_clone_vector_of_formal_parm_types (tree fndecl)
-{
- if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
- return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl));
- vec<tree> args = ipa_get_vector_of_formal_parms (fndecl);
- unsigned int i;
- tree arg;
- FOR_EACH_VEC_ELT (args, i, arg)
- args[i] = TREE_TYPE (args[i]);
- return args;
-}
-
-/* Given a simd function in NODE, extract the simd specific
- information from the OMP clauses passed in CLAUSES, and return
- the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
- is set to TRUE if the `inbranch' or `notinbranch' clause specified,
- otherwise set to FALSE. */
-
-static struct cgraph_simd_clone *
-simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
- bool *inbranch_specified)
-{
- vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl);
- tree t;
- int n;
- *inbranch_specified = false;
-
- n = args.length ();
- if (n > 0 && args.last () == void_type_node)
- n--;
-
- /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
- be cloned have a distinctive artificial label in addition to "omp
- declare simd". */
- bool cilk_clone
- = (flag_cilkplus
- && lookup_attribute ("cilk simd function",
- DECL_ATTRIBUTES (node->decl)));
-
- /* Allocate one more than needed just in case this is an in-branch
- clone which will require a mask argument. */
- struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
- clone_info->nargs = n;
- clone_info->cilk_elemental = cilk_clone;
-
- if (!clauses)
- {
- args.release ();
- return clone_info;
- }
- clauses = TREE_VALUE (clauses);
- if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
- return clone_info;
-
- for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
- {
- switch (OMP_CLAUSE_CODE (t))
- {
- case OMP_CLAUSE_INBRANCH:
- clone_info->inbranch = 1;
- *inbranch_specified = true;
- break;
- case OMP_CLAUSE_NOTINBRANCH:
- clone_info->inbranch = 0;
- *inbranch_specified = true;
- break;
- case OMP_CLAUSE_SIMDLEN:
- clone_info->simdlen
- = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
- break;
- case OMP_CLAUSE_LINEAR:
- {
- tree decl = OMP_CLAUSE_DECL (t);
- tree step = OMP_CLAUSE_LINEAR_STEP (t);
- int argno = TREE_INT_CST_LOW (decl);
- if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
- {
- enum cgraph_simd_clone_arg_type arg_type;
- if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
- switch (OMP_CLAUSE_LINEAR_KIND (t))
- {
- case OMP_CLAUSE_LINEAR_REF:
- arg_type
- = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP;
- break;
- case OMP_CLAUSE_LINEAR_UVAL:
- arg_type
- = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP;
- break;
- case OMP_CLAUSE_LINEAR_VAL:
- case OMP_CLAUSE_LINEAR_DEFAULT:
- arg_type
- = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP;
- break;
- default:
- gcc_unreachable ();
- }
- else
- arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
- clone_info->args[argno].arg_type = arg_type;
- clone_info->args[argno].linear_step = tree_to_shwi (step);
- gcc_assert (clone_info->args[argno].linear_step >= 0
- && clone_info->args[argno].linear_step < n);
- }
- else
- {
- if (POINTER_TYPE_P (args[argno]))
- step = fold_convert (ssizetype, step);
- if (!tree_fits_shwi_p (step))
- {
- warning_at (OMP_CLAUSE_LOCATION (t), 0,
- "ignoring large linear step");
- args.release ();
- return NULL;
- }
- else if (integer_zerop (step))
- {
- warning_at (OMP_CLAUSE_LOCATION (t), 0,
- "ignoring zero linear step");
- args.release ();
- return NULL;
- }
- else
- {
- enum cgraph_simd_clone_arg_type arg_type;
- if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
- switch (OMP_CLAUSE_LINEAR_KIND (t))
- {
- case OMP_CLAUSE_LINEAR_REF:
- arg_type
- = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP;
- break;
- case OMP_CLAUSE_LINEAR_UVAL:
- arg_type
- = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP;
- break;
- case OMP_CLAUSE_LINEAR_VAL:
- case OMP_CLAUSE_LINEAR_DEFAULT:
- arg_type
- = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP;
- break;
- default:
- gcc_unreachable ();
- }
- else
- arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
- clone_info->args[argno].arg_type = arg_type;
- clone_info->args[argno].linear_step = tree_to_shwi (step);
- }
- }
- break;
- }
- case OMP_CLAUSE_UNIFORM:
- {
- tree decl = OMP_CLAUSE_DECL (t);
- int argno = tree_to_uhwi (decl);
- clone_info->args[argno].arg_type
- = SIMD_CLONE_ARG_TYPE_UNIFORM;
- break;
- }
- case OMP_CLAUSE_ALIGNED:
- {
- tree decl = OMP_CLAUSE_DECL (t);
- int argno = tree_to_uhwi (decl);
- clone_info->args[argno].alignment
- = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
- break;
- }
- default:
- break;
- }
- }
- args.release ();
- return clone_info;
-}
-
-/* Given a SIMD clone in NODE, calculate the characteristic data
- type and return the coresponding type. The characteristic data
- type is computed as described in the Intel Vector ABI. */
-
-static tree
-simd_clone_compute_base_data_type (struct cgraph_node *node,
- struct cgraph_simd_clone *clone_info)
-{
- tree type = integer_type_node;
- tree fndecl = node->decl;
-
- /* a) For non-void function, the characteristic data type is the
- return type. */
- if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
- type = TREE_TYPE (TREE_TYPE (fndecl));
-
- /* b) If the function has any non-uniform, non-linear parameters,
- then the characteristic data type is the type of the first
- such parameter. */
- else
- {
- vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl);
- for (unsigned int i = 0; i < clone_info->nargs; ++i)
- if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
- {
- type = map[i];
- break;
- }
- map.release ();
- }
-
- /* c) If the characteristic data type determined by a) or b) above
- is struct, union, or class type which is pass-by-value (except
- for the type that maps to the built-in complex data type), the
- characteristic data type is int. */
- if (RECORD_OR_UNION_TYPE_P (type)
- && !aggregate_value_p (type, NULL)
- && TREE_CODE (type) != COMPLEX_TYPE)
- return integer_type_node;
-
- /* d) If none of the above three classes is applicable, the
- characteristic data type is int. */
-
- return type;
-
- /* e) For Intel Xeon Phi native and offload compilation, if the
- resulting characteristic data type is 8-bit or 16-bit integer
- data type, the characteristic data type is int. */
- /* Well, we don't handle Xeon Phi yet. */
-}
-
-static tree
-simd_clone_mangle (struct cgraph_node *node,
- struct cgraph_simd_clone *clone_info)
-{
- char vecsize_mangle = clone_info->vecsize_mangle;
- char mask = clone_info->inbranch ? 'M' : 'N';
- unsigned int simdlen = clone_info->simdlen;
- unsigned int n;
- pretty_printer pp;
-
- gcc_assert (vecsize_mangle && simdlen);
-
- pp_string (&pp, "_ZGV");
- pp_character (&pp, vecsize_mangle);
- pp_character (&pp, mask);
- pp_decimal_int (&pp, simdlen);
-
- for (n = 0; n < clone_info->nargs; ++n)
- {
- struct cgraph_simd_clone_arg arg = clone_info->args[n];
-
- switch (arg.arg_type)
- {
- case SIMD_CLONE_ARG_TYPE_UNIFORM:
- pp_character (&pp, 'u');
- break;
- case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
- pp_character (&pp, 'l');
- goto mangle_linear;
- case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
- pp_character (&pp, 'R');
- goto mangle_linear;
- case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
- pp_character (&pp, 'L');
- goto mangle_linear;
- case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
- pp_character (&pp, 'U');
- goto mangle_linear;
- mangle_linear:
- gcc_assert (arg.linear_step != 0);
- if (arg.linear_step > 1)
- pp_unsigned_wide_integer (&pp, arg.linear_step);
- else if (arg.linear_step < 0)
- {
- pp_character (&pp, 'n');
- pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
- arg.linear_step));
- }
- break;
- case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
- pp_string (&pp, "ls");
- pp_unsigned_wide_integer (&pp, arg.linear_step);
- break;
- case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
- pp_string (&pp, "Rs");
- pp_unsigned_wide_integer (&pp, arg.linear_step);
- break;
- case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
- pp_string (&pp, "Ls");
- pp_unsigned_wide_integer (&pp, arg.linear_step);
- break;
- case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
- pp_string (&pp, "Us");
- pp_unsigned_wide_integer (&pp, arg.linear_step);
- break;
- default:
- pp_character (&pp, 'v');
- }
- if (arg.alignment)
- {
- pp_character (&pp, 'a');
- pp_decimal_int (&pp, arg.alignment);
- }
- }
-
- pp_underscore (&pp);
- const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
- if (*str == '*')
- ++str;
- pp_string (&pp, str);
- str = pp_formatted_text (&pp);
-
- /* If there already is a SIMD clone with the same mangled name, don't
- add another one. This can happen e.g. for
- #pragma omp declare simd
- #pragma omp declare simd simdlen(8)
- int foo (int, int);
- if the simdlen is assumed to be 8 for the first one, etc. */
- for (struct cgraph_node *clone = node->simd_clones; clone;
- clone = clone->simdclone->next_clone)
- if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone->decl)),
- str) == 0)
- return NULL_TREE;
-
- return get_identifier (str);
-}
-
-/* Create a simd clone of OLD_NODE and return it. */
-
-static struct cgraph_node *
-simd_clone_create (struct cgraph_node *old_node)
-{
- struct cgraph_node *new_node;
- if (old_node->definition)
- {
- if (!old_node->has_gimple_body_p ())
- return NULL;
- old_node->get_body ();
- new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
- false, NULL, NULL,
- "simdclone");
- }
- else
- {
- tree old_decl = old_node->decl;
- tree new_decl = copy_node (old_node->decl);
- DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone");
- SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
- SET_DECL_RTL (new_decl, NULL);
- DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
- DECL_STATIC_DESTRUCTOR (new_decl) = 0;
- new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
- if (old_node->in_other_partition)
- new_node->in_other_partition = 1;
- }
- if (new_node == NULL)
- return new_node;
-
- TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
-
- /* The function cgraph_function_versioning () will force the new
- symbol local. Undo this, and inherit external visability from
- the old node. */
- new_node->local.local = old_node->local.local;
- new_node->externally_visible = old_node->externally_visible;
-
- return new_node;
-}
-
-/* Adjust the return type of the given function to its appropriate
- vector counterpart. Returns a simd array to be used throughout the
- function as a return value. */
-
-static tree
-simd_clone_adjust_return_type (struct cgraph_node *node)
-{
- tree fndecl = node->decl;
- tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
- unsigned int veclen;
- tree t;
-
- /* Adjust the function return type. */
- if (orig_rettype == void_type_node)
- return NULL_TREE;
- TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl));
- t = TREE_TYPE (TREE_TYPE (fndecl));
- if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t))
- veclen = node->simdclone->vecsize_int;
- else
- veclen = node->simdclone->vecsize_float;
- veclen /= GET_MODE_BITSIZE (TYPE_MODE (t));
- if (veclen > node->simdclone->simdlen)
- veclen = node->simdclone->simdlen;
- if (POINTER_TYPE_P (t))
- t = pointer_sized_int_node;
- if (veclen == node->simdclone->simdlen)
- t = build_vector_type (t, node->simdclone->simdlen);
- else
- {
- t = build_vector_type (t, veclen);
- t = build_array_type_nelts (t, node->simdclone->simdlen / veclen);
- }
- TREE_TYPE (TREE_TYPE (fndecl)) = t;
- if (!node->definition)
- return NULL_TREE;
-
- t = DECL_RESULT (fndecl);
- /* Adjust the DECL_RESULT. */
- gcc_assert (TREE_TYPE (t) != void_type_node);
- TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
- relayout_decl (t);
-
- tree atype = build_array_type_nelts (orig_rettype,
- node->simdclone->simdlen);
- if (veclen != node->simdclone->simdlen)
- return build1 (VIEW_CONVERT_EXPR, atype, t);
-
- /* Set up a SIMD array to use as the return value. */
- tree retval = create_tmp_var_raw (atype, "retval");
- gimple_add_tmp_var (retval);
- return retval;
-}
-
-/* Each vector argument has a corresponding array to be used locally
- as part of the eventual loop. Create such temporary array and
- return it.
-
- PREFIX is the prefix to be used for the temporary.
-
- TYPE is the inner element type.
-
- SIMDLEN is the number of elements. */
-
-static tree
-create_tmp_simd_array (const char *prefix, tree type, int simdlen)
-{
- tree atype = build_array_type_nelts (type, simdlen);
- tree avar = create_tmp_var_raw (atype, prefix);
- gimple_add_tmp_var (avar);
- return avar;
-}
-
-/* Modify the function argument types to their corresponding vector
- counterparts if appropriate. Also, create one array for each simd
- argument to be used locally when using the function arguments as
- part of the loop.
-
- NODE is the function whose arguments are to be adjusted.
-
- Returns an adjustment vector that will be filled describing how the
- argument types will be adjusted. */
-
-static ipa_parm_adjustment_vec
-simd_clone_adjust_argument_types (struct cgraph_node *node)
-{
- vec<tree> args;
- ipa_parm_adjustment_vec adjustments;
-
- if (node->definition)
- args = ipa_get_vector_of_formal_parms (node->decl);
- else
- args = simd_clone_vector_of_formal_parm_types (node->decl);
- adjustments.create (args.length ());
- unsigned i, j, veclen;
- struct ipa_parm_adjustment adj;
- struct cgraph_simd_clone *sc = node->simdclone;
-
- for (i = 0; i < sc->nargs; ++i)
- {
- memset (&adj, 0, sizeof (adj));
- tree parm = args[i];
- tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
- adj.base_index = i;
- adj.base = parm;
-
- sc->args[i].orig_arg = node->definition ? parm : NULL_TREE;
- sc->args[i].orig_type = parm_type;
-
- switch (sc->args[i].arg_type)
- {
- default:
- /* No adjustment necessary for scalar arguments. */
- adj.op = IPA_PARM_OP_COPY;
- break;
- case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
- case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
- if (node->definition)
- sc->args[i].simd_array
- = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
- TREE_TYPE (parm_type),
- sc->simdlen);
- adj.op = IPA_PARM_OP_COPY;
- break;
- case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
- case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
- case SIMD_CLONE_ARG_TYPE_VECTOR:
- if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
- veclen = sc->vecsize_int;
- else
- veclen = sc->vecsize_float;
- veclen /= GET_MODE_BITSIZE (TYPE_MODE (parm_type));
- if (veclen > sc->simdlen)
- veclen = sc->simdlen;
- adj.arg_prefix = "simd";
- if (POINTER_TYPE_P (parm_type))
- adj.type = build_vector_type (pointer_sized_int_node, veclen);
- else
- adj.type = build_vector_type (parm_type, veclen);
- sc->args[i].vector_type = adj.type;
- for (j = veclen; j < sc->simdlen; j += veclen)
- {
- adjustments.safe_push (adj);
- if (j == veclen)
- {
- memset (&adj, 0, sizeof (adj));
- adj.op = IPA_PARM_OP_NEW;
- adj.arg_prefix = "simd";
- adj.base_index = i;
- adj.type = sc->args[i].vector_type;
- }
- }
-
- if (node->definition)
- sc->args[i].simd_array
- = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
- parm_type, sc->simdlen);
- }
- adjustments.safe_push (adj);
- }
-
- if (sc->inbranch)
- {
- tree base_type = simd_clone_compute_base_data_type (sc->origin, sc);
-
- memset (&adj, 0, sizeof (adj));
- adj.op = IPA_PARM_OP_NEW;
- adj.arg_prefix = "mask";
-
- adj.base_index = i;
- if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
- veclen = sc->vecsize_int;
- else
- veclen = sc->vecsize_float;
- veclen /= GET_MODE_BITSIZE (TYPE_MODE (base_type));
- if (veclen > sc->simdlen)
- veclen = sc->simdlen;
- if (sc->mask_mode != VOIDmode)
- adj.type
- = lang_hooks.types.type_for_mode (sc->mask_mode, 1);
- else if (POINTER_TYPE_P (base_type))
- adj.type = build_vector_type (pointer_sized_int_node, veclen);
- else
- adj.type = build_vector_type (base_type, veclen);
- adjustments.safe_push (adj);
-
- for (j = veclen; j < sc->simdlen; j += veclen)
- adjustments.safe_push (adj);
-
- /* We have previously allocated one extra entry for the mask. Use
- it and fill it. */
- sc->nargs++;
- if (sc->mask_mode != VOIDmode)
- base_type = boolean_type_node;
- if (node->definition)
- {
- sc->args[i].orig_arg
- = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
- if (sc->mask_mode == VOIDmode)
- sc->args[i].simd_array
- = create_tmp_simd_array ("mask", base_type, sc->simdlen);
- else if (veclen < sc->simdlen)
- sc->args[i].simd_array
- = create_tmp_simd_array ("mask", adj.type, sc->simdlen / veclen);
- else
- sc->args[i].simd_array = NULL_TREE;
- }
- sc->args[i].orig_type = base_type;
- sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
- }
-
- if (node->definition)
- ipa_modify_formal_parameters (node->decl, adjustments);
- else
- {
- tree new_arg_types = NULL_TREE, new_reversed;
- bool last_parm_void = false;
- if (args.length () > 0 && args.last () == void_type_node)
- last_parm_void = true;
-
- gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
- j = adjustments.length ();
- for (i = 0; i < j; i++)
- {
- struct ipa_parm_adjustment *adj = &adjustments[i];
- tree ptype;
- if (adj->op == IPA_PARM_OP_COPY)
- ptype = args[adj->base_index];
- else
- ptype = adj->type;
- new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
- }
- new_reversed = nreverse (new_arg_types);
- if (last_parm_void)
- {
- if (new_reversed)
- TREE_CHAIN (new_arg_types) = void_list_node;
- else
- new_reversed = void_list_node;
- }
-
- tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl));
- TYPE_ARG_TYPES (new_type) = new_reversed;
- TREE_TYPE (node->decl) = new_type;
-
- adjustments.release ();
- }
- args.release ();
- return adjustments;
-}
-
-/* Initialize and copy the function arguments in NODE to their
- corresponding local simd arrays. Returns a fresh gimple_seq with
- the instruction sequence generated. */
-
-static gimple_seq
-simd_clone_init_simd_arrays (struct cgraph_node *node,
- ipa_parm_adjustment_vec adjustments)
-{
- gimple_seq seq = NULL;
- unsigned i = 0, j = 0, k;
-
- for (tree arg = DECL_ARGUMENTS (node->decl);
- arg;
- arg = DECL_CHAIN (arg), i++, j++)
- {
- if (adjustments[j].op == IPA_PARM_OP_COPY
- || POINTER_TYPE_P (TREE_TYPE (arg)))
- continue;
-
- node->simdclone->args[i].vector_arg = arg;
-
- tree array = node->simdclone->args[i].simd_array;
- if (node->simdclone->mask_mode != VOIDmode
- && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK)
- {
- if (array == NULL_TREE)
- continue;
- unsigned int l
- = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array))));
- for (k = 0; k <= l; k++)
- {
- if (k)
- {
- arg = DECL_CHAIN (arg);
- j++;
- }
- tree t = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (array)),
- array, size_int (k), NULL, NULL);
- t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
- gimplify_and_add (t, &seq);
- }
- continue;
- }
- if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)) == node->simdclone->simdlen)
- {
- tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
- tree ptr = build_fold_addr_expr (array);
- tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
- build_int_cst (ptype, 0));
- t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
- gimplify_and_add (t, &seq);
- }
- else
- {
- unsigned int simdlen = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg));
- tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
- for (k = 0; k < node->simdclone->simdlen; k += simdlen)
- {
- tree ptr = build_fold_addr_expr (array);
- int elemsize;
- if (k)
- {
- arg = DECL_CHAIN (arg);
- j++;
- }
- elemsize
- = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg))));
- tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
- build_int_cst (ptype, k * elemsize));
- t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
- gimplify_and_add (t, &seq);
- }
- }
- }
- return seq;
-}
-
-/* Callback info for ipa_simd_modify_stmt_ops below. */
-
-struct modify_stmt_info {
- ipa_parm_adjustment_vec adjustments;
- gimple *stmt;
- /* True if the parent statement was modified by
- ipa_simd_modify_stmt_ops. */
- bool modified;
-};
-
-/* Callback for walk_gimple_op.
-
- Adjust operands from a given statement as specified in the
- adjustments vector in the callback data. */
-
-static tree
-ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
-{
- struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
- struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
- tree *orig_tp = tp;
- if (TREE_CODE (*tp) == ADDR_EXPR)
- tp = &TREE_OPERAND (*tp, 0);
- struct ipa_parm_adjustment *cand = NULL;
- if (TREE_CODE (*tp) == PARM_DECL)
- cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true);
- else
- {
- if (TYPE_P (*tp))
- *walk_subtrees = 0;
- }
-
- tree repl = NULL_TREE;
- if (cand)
- repl = unshare_expr (cand->new_decl);
- else
- {
- if (tp != orig_tp)
- {
- *walk_subtrees = 0;
- bool modified = info->modified;
- info->modified = false;
- walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
- if (!info->modified)
- {
- info->modified = modified;
- return NULL_TREE;
- }
- info->modified = modified;
- repl = *tp;
- }
- else
- return NULL_TREE;
- }
-
- if (tp != orig_tp)
- {
- repl = build_fold_addr_expr (repl);
- gimple *stmt;
- if (is_gimple_debug (info->stmt))
- {
- tree vexpr = make_node (DEBUG_EXPR_DECL);
- stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
- DECL_ARTIFICIAL (vexpr) = 1;
- TREE_TYPE (vexpr) = TREE_TYPE (repl);
- DECL_MODE (vexpr) = TYPE_MODE (TREE_TYPE (repl));
- repl = vexpr;
- }
- else
- {
- stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
- repl = gimple_assign_lhs (stmt);
- }
- gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt);
- gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
- *orig_tp = repl;
- }
- else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
- {
- tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
- *tp = vce;
- }
- else
- *tp = repl;
-
- info->modified = true;
- return NULL_TREE;
-}
-
-/* Traverse the function body and perform all modifications as
- described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
- modified such that the replacement/reduction value will now be an
- offset into the corresponding simd_array.
-
- This function will replace all function argument uses with their
- corresponding simd array elements, and ajust the return values
- accordingly. */
-
-static void
-ipa_simd_modify_function_body (struct cgraph_node *node,
- ipa_parm_adjustment_vec adjustments,
- tree retval_array, tree iter)
-{
- basic_block bb;
- unsigned int i, j, l;
-
- /* Re-use the adjustments array, but this time use it to replace
- every function argument use to an offset into the corresponding
- simd_array. */
- for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
- {
- if (!node->simdclone->args[i].vector_arg)
- continue;
-
- tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
- tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
- adjustments[j].new_decl
- = build4 (ARRAY_REF,
- basetype,
- node->simdclone->args[i].simd_array,
- iter,
- NULL_TREE, NULL_TREE);
- if (adjustments[j].op == IPA_PARM_OP_NONE
- && TYPE_VECTOR_SUBPARTS (vectype) < node->simdclone->simdlen)
- j += node->simdclone->simdlen / TYPE_VECTOR_SUBPARTS (vectype) - 1;
- }
-
- l = adjustments.length ();
- for (i = 1; i < num_ssa_names; i++)
- {
- tree name = ssa_name (i);
- if (name
- && SSA_NAME_VAR (name)
- && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL)
- {
- for (j = 0; j < l; j++)
- if (SSA_NAME_VAR (name) == adjustments[j].base
- && adjustments[j].new_decl)
- {
- tree base_var;
- if (adjustments[j].new_ssa_base == NULL_TREE)
- {
- base_var
- = copy_var_decl (adjustments[j].base,
- DECL_NAME (adjustments[j].base),
- TREE_TYPE (adjustments[j].base));
- adjustments[j].new_ssa_base = base_var;
- }
- else
- base_var = adjustments[j].new_ssa_base;
- if (SSA_NAME_IS_DEFAULT_DEF (name))
- {
- bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
- gimple_stmt_iterator gsi = gsi_after_labels (bb);
- tree new_decl = unshare_expr (adjustments[j].new_decl);
- set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE);
- SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
- SSA_NAME_IS_DEFAULT_DEF (name) = 0;
- gimple *stmt = gimple_build_assign (name, new_decl);
- gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
- }
- else
- SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
- }
- }
- }
-
- struct modify_stmt_info info;
- info.adjustments = adjustments;
-
- FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
- {
- gimple_stmt_iterator gsi;
-
- gsi = gsi_start_bb (bb);
- while (!gsi_end_p (gsi))
- {
- gimple *stmt = gsi_stmt (gsi);
- info.stmt = stmt;
- struct walk_stmt_info wi;
-
- memset (&wi, 0, sizeof (wi));
- info.modified = false;
- wi.info = &info;
- walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
-
- if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
- {
- tree retval = gimple_return_retval (return_stmt);
- if (!retval)
- {
- gsi_remove (&gsi, true);
- continue;
- }
-
- /* Replace `return foo' with `retval_array[iter] = foo'. */
- tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
- retval_array, iter, NULL, NULL);
- stmt = gimple_build_assign (ref, retval);
- gsi_replace (&gsi, stmt, true);
- info.modified = true;
- }
-
- if (info.modified)
- {
- update_stmt (stmt);
- if (maybe_clean_eh_stmt (stmt))
- gimple_purge_dead_eh_edges (gimple_bb (stmt));
- }
- gsi_next (&gsi);
- }
- }
-}
-
-/* Helper function of simd_clone_adjust, return linear step addend
- of Ith argument. */
-
-static tree
-simd_clone_linear_addend (struct cgraph_node *node, unsigned int i,
- tree addtype, basic_block entry_bb)
-{
- tree ptype = NULL_TREE;
- switch (node->simdclone->args[i].arg_type)
- {
- case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
- case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
- case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
- case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
- return build_int_cst (addtype, node->simdclone->args[i].linear_step);
- case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
- case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
- ptype = TREE_TYPE (node->simdclone->args[i].orig_arg);
- break;
- case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
- case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
- ptype = TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg));
- break;
- default:
- gcc_unreachable ();
- }
-
- unsigned int idx = node->simdclone->args[i].linear_step;
- tree arg = node->simdclone->args[idx].orig_arg;
- gcc_assert (is_gimple_reg_type (TREE_TYPE (arg)));
- gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
- gimple *g;
- tree ret;
- if (is_gimple_reg (arg))
- ret = get_or_create_ssa_default_def (cfun, arg);
- else
- {
- g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg);
- gsi_insert_before (&gsi, g, GSI_SAME_STMT);
- ret = gimple_assign_lhs (g);
- }
- if (TREE_CODE (TREE_TYPE (arg)) == REFERENCE_TYPE)
- {
- g = gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg))),
- build_simple_mem_ref (ret));
- gsi_insert_before (&gsi, g, GSI_SAME_STMT);
- ret = gimple_assign_lhs (g);
- }
- if (!useless_type_conversion_p (addtype, TREE_TYPE (ret)))
- {
- g = gimple_build_assign (make_ssa_name (addtype), NOP_EXPR, ret);
- gsi_insert_before (&gsi, g, GSI_SAME_STMT);
- ret = gimple_assign_lhs (g);
- }
- if (POINTER_TYPE_P (ptype))
- {
- tree size = TYPE_SIZE_UNIT (TREE_TYPE (ptype));
- if (size && TREE_CODE (size) == INTEGER_CST)
- {
- g = gimple_build_assign (make_ssa_name (addtype), MULT_EXPR,
- ret, fold_convert (addtype, size));
- gsi_insert_before (&gsi, g, GSI_SAME_STMT);
- ret = gimple_assign_lhs (g);
- }
- }
- return ret;
-}
-
-/* Adjust the argument types in NODE to their appropriate vector
- counterparts. */
-
-static void
-simd_clone_adjust (struct cgraph_node *node)
-{
- push_cfun (DECL_STRUCT_FUNCTION (node->decl));
-
- targetm.simd_clone.adjust (node);
-
- tree retval = simd_clone_adjust_return_type (node);
- ipa_parm_adjustment_vec adjustments
- = simd_clone_adjust_argument_types (node);
-
- push_gimplify_context ();
-
- gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
-
- /* Adjust all uses of vector arguments accordingly. Adjust all
- return values accordingly. */
- tree iter = create_tmp_var (unsigned_type_node, "iter");
- tree iter1 = make_ssa_name (iter);
- tree iter2 = make_ssa_name (iter);
- ipa_simd_modify_function_body (node, adjustments, retval, iter1);
-
- /* Initialize the iteration variable. */
- basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
- basic_block body_bb = split_block_after_labels (entry_bb)->dest;
- gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
- /* Insert the SIMD array and iv initialization at function
- entry. */
- gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
-
- pop_gimplify_context (NULL);
-
- /* Create a new BB right before the original exit BB, to hold the
- iteration increment and the condition/branch. */
- basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
- basic_block incr_bb = create_empty_bb (orig_exit);
- add_bb_to_loop (incr_bb, body_bb->loop_father);
- /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
- flag. Set it now to be a FALLTHRU_EDGE. */
- gcc_assert (EDGE_COUNT (orig_exit->succs) == 1);
- EDGE_SUCC (orig_exit, 0)->flags |= EDGE_FALLTHRU;
- for (unsigned i = 0;
- i < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds); ++i)
- {
- edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), i);
- redirect_edge_succ (e, incr_bb);
- }
- edge e = make_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
- e->probability = REG_BR_PROB_BASE;
- gsi = gsi_last_bb (incr_bb);
- gimple *g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
- build_int_cst (unsigned_type_node, 1));
- gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
-
- /* Mostly annotate the loop for the vectorizer (the rest is done below). */
- struct loop *loop = alloc_loop ();
- cfun->has_force_vectorize_loops = true;
- loop->safelen = node->simdclone->simdlen;
- loop->force_vectorize = true;
- loop->header = body_bb;
-
- /* Branch around the body if the mask applies. */
- if (node->simdclone->inbranch)
- {
- gimple_stmt_iterator gsi = gsi_last_bb (loop->header);
- tree mask_array
- = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
- tree mask;
- if (node->simdclone->mask_mode != VOIDmode)
- {
- tree shift_cnt;
- if (mask_array == NULL_TREE)
- {
- tree arg = node->simdclone->args[node->simdclone->nargs
- - 1].vector_arg;
- mask = get_or_create_ssa_default_def (cfun, arg);
- shift_cnt = iter1;
- }
- else
- {
- tree maskt = TREE_TYPE (mask_array);
- int c = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt)));
- c = node->simdclone->simdlen / (c + 1);
- int s = exact_log2 (c);
- gcc_assert (s > 0);
- c--;
- tree idx = make_ssa_name (TREE_TYPE (iter1));
- g = gimple_build_assign (idx, RSHIFT_EXPR, iter1,
- build_int_cst (NULL_TREE, s));
- gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
- mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
- tree aref = build4 (ARRAY_REF,
- TREE_TYPE (TREE_TYPE (mask_array)),
- mask_array, idx, NULL, NULL);
- g = gimple_build_assign (mask, aref);
- gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
- shift_cnt = make_ssa_name (TREE_TYPE (iter1));
- g = gimple_build_assign (shift_cnt, BIT_AND_EXPR, iter1,
- build_int_cst (TREE_TYPE (iter1), c));
- gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
- }
- g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
- RSHIFT_EXPR, mask, shift_cnt);
- gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
- mask = gimple_assign_lhs (g);
- g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
- BIT_AND_EXPR, mask,
- build_int_cst (TREE_TYPE (mask), 1));
- gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
- mask = gimple_assign_lhs (g);
- }
- else
- {
- mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
- tree aref = build4 (ARRAY_REF,
- TREE_TYPE (TREE_TYPE (mask_array)),
- mask_array, iter1, NULL, NULL);
- g = gimple_build_assign (mask, aref);
- gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
- int bitsize = GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref)));
- if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
- {
- aref = build1 (VIEW_CONVERT_EXPR,
- build_nonstandard_integer_type (bitsize, 0),
- mask);
- mask = make_ssa_name (TREE_TYPE (aref));
- g = gimple_build_assign (mask, aref);
- gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
- }
- }
-
- g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
- NULL, NULL);
- gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
- make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
- FALLTHRU_EDGE (loop->header)->flags = EDGE_FALSE_VALUE;
- }
-
- /* Generate the condition. */
- g = gimple_build_cond (LT_EXPR,
- iter2,
- build_int_cst (unsigned_type_node,
- node->simdclone->simdlen),
- NULL, NULL);
- gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
- e = split_block (incr_bb, gsi_stmt (gsi));
- basic_block latch_bb = e->dest;
- basic_block new_exit_bb;
- new_exit_bb = split_block_after_labels (latch_bb)->dest;
- loop->latch = latch_bb;
-
- redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
-
- make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
- /* The successor of incr_bb is already pointing to latch_bb; just
- change the flags.
- make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
- FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
-
- gphi *phi = create_phi_node (iter1, body_bb);
- edge preheader_edge = find_edge (entry_bb, body_bb);
- edge latch_edge = single_succ_edge (latch_bb);
- add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
- UNKNOWN_LOCATION);
- add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
-
- /* Generate the new return. */
- gsi = gsi_last_bb (new_exit_bb);
- if (retval
- && TREE_CODE (retval) == VIEW_CONVERT_EXPR
- && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
- retval = TREE_OPERAND (retval, 0);
- else if (retval)
- {
- retval = build1 (VIEW_CONVERT_EXPR,
- TREE_TYPE (TREE_TYPE (node->decl)),
- retval);
- retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
- false, GSI_CONTINUE_LINKING);
- }
- g = gimple_build_return (retval);
- gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
-
- /* Handle aligned clauses by replacing default defs of the aligned
- uniform args with __builtin_assume_aligned (arg_N(D), alignment)
- lhs. Handle linear by adding PHIs. */
- for (unsigned i = 0; i < node->simdclone->nargs; i++)
- if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
- && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg)
- || !is_gimple_reg_type
- (TREE_TYPE (node->simdclone->args[i].orig_arg))))
- {
- tree orig_arg = node->simdclone->args[i].orig_arg;
- if (is_gimple_reg_type (TREE_TYPE (orig_arg)))
- iter1 = make_ssa_name (TREE_TYPE (orig_arg));
- else
- {
- iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg));
- gimple_add_tmp_var (iter1);
- }
- gsi = gsi_after_labels (entry_bb);
- g = gimple_build_assign (iter1, orig_arg);
- gsi_insert_before (&gsi, g, GSI_NEW_STMT);
- gsi = gsi_after_labels (body_bb);
- g = gimple_build_assign (orig_arg, iter1);
- gsi_insert_before (&gsi, g, GSI_NEW_STMT);
- }
- else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
- && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg)
- && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
- == REFERENCE_TYPE
- && TREE_ADDRESSABLE
- (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg))))
- {
- tree orig_arg = node->simdclone->args[i].orig_arg;
- tree def = ssa_default_def (cfun, orig_arg);
- if (def && !has_zero_uses (def))
- {
- iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg)));
- gimple_add_tmp_var (iter1);
- gsi = gsi_after_labels (entry_bb);
- g = gimple_build_assign (iter1, build_simple_mem_ref (def));
- gsi_insert_before (&gsi, g, GSI_NEW_STMT);
- gsi = gsi_after_labels (body_bb);
- g = gimple_build_assign (build_simple_mem_ref (def), iter1);
- gsi_insert_before (&gsi, g, GSI_NEW_STMT);
- }
- }
- else if (node->simdclone->args[i].alignment
- && node->simdclone->args[i].arg_type
- == SIMD_CLONE_ARG_TYPE_UNIFORM
- && (node->simdclone->args[i].alignment
- & (node->simdclone->args[i].alignment - 1)) == 0
- && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
- == POINTER_TYPE)
- {
- unsigned int alignment = node->simdclone->args[i].alignment;
- tree orig_arg = node->simdclone->args[i].orig_arg;
- tree def = ssa_default_def (cfun, orig_arg);
- if (def && !has_zero_uses (def))
- {
- tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
- gimple_seq seq = NULL;
- bool need_cvt = false;
- gcall *call
- = gimple_build_call (fn, 2, def, size_int (alignment));
- g = call;
- if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
- ptr_type_node))
- need_cvt = true;
- tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg);
- gimple_call_set_lhs (g, t);
- gimple_seq_add_stmt_without_update (&seq, g);
- if (need_cvt)
- {
- t = make_ssa_name (orig_arg);
- g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g));
- gimple_seq_add_stmt_without_update (&seq, g);
- }
- gsi_insert_seq_on_edge_immediate
- (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
-
- entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
- int freq = compute_call_stmt_bb_frequency (current_function_decl,
- entry_bb);
- node->create_edge (cgraph_node::get_create (fn),
- call, entry_bb->count, freq);
-
- imm_use_iterator iter;
- use_operand_p use_p;
- gimple *use_stmt;
- tree repl = gimple_get_lhs (g);
- FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
- if (is_gimple_debug (use_stmt) || use_stmt == call)
- continue;
- else
- FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
- SET_USE (use_p, repl);
- }
- }
- else if ((node->simdclone->args[i].arg_type
- == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
- || (node->simdclone->args[i].arg_type
- == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
- || (node->simdclone->args[i].arg_type
- == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
- || (node->simdclone->args[i].arg_type
- == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP))
- {
- tree orig_arg = node->simdclone->args[i].orig_arg;
- gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
- || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
- tree def = NULL_TREE;
- if (TREE_ADDRESSABLE (orig_arg))
- {
- def = make_ssa_name (TREE_TYPE (orig_arg));
- iter1 = make_ssa_name (TREE_TYPE (orig_arg));
- iter2 = make_ssa_name (TREE_TYPE (orig_arg));
- gsi = gsi_after_labels (entry_bb);
- g = gimple_build_assign (def, orig_arg);
- gsi_insert_before (&gsi, g, GSI_NEW_STMT);
- }
- else
- {
- def = ssa_default_def (cfun, orig_arg);
- if (!def || has_zero_uses (def))
- def = NULL_TREE;
- else
- {
- iter1 = make_ssa_name (orig_arg);
- iter2 = make_ssa_name (orig_arg);
- }
- }
- if (def)
- {
- phi = create_phi_node (iter1, body_bb);
- add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
- add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
- enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
- ? PLUS_EXPR : POINTER_PLUS_EXPR;
- tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
- ? TREE_TYPE (orig_arg) : sizetype;
- tree addcst = simd_clone_linear_addend (node, i, addtype,
- entry_bb);
- gsi = gsi_last_bb (incr_bb);
- g = gimple_build_assign (iter2, code, iter1, addcst);
- gsi_insert_before (&gsi, g, GSI_SAME_STMT);
-
- imm_use_iterator iter;
- use_operand_p use_p;
- gimple *use_stmt;
- if (TREE_ADDRESSABLE (orig_arg))
- {
- gsi = gsi_after_labels (body_bb);
- g = gimple_build_assign (orig_arg, iter1);
- gsi_insert_before (&gsi, g, GSI_NEW_STMT);
- }
- else
- FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
- if (use_stmt == phi)
- continue;
- else
- FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
- SET_USE (use_p, iter1);
- }
- }
- else if (node->simdclone->args[i].arg_type
- == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
- || (node->simdclone->args[i].arg_type
- == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP))
- {
- tree orig_arg = node->simdclone->args[i].orig_arg;
- tree def = ssa_default_def (cfun, orig_arg);
- gcc_assert (!TREE_ADDRESSABLE (orig_arg)
- && TREE_CODE (TREE_TYPE (orig_arg)) == REFERENCE_TYPE);
- if (def && !has_zero_uses (def))
- {
- tree rtype = TREE_TYPE (TREE_TYPE (orig_arg));
- iter1 = make_ssa_name (orig_arg);
- iter2 = make_ssa_name (orig_arg);
- tree iter3 = make_ssa_name (rtype);
- tree iter4 = make_ssa_name (rtype);
- tree iter5 = make_ssa_name (rtype);
- gsi = gsi_after_labels (entry_bb);
- gimple *load
- = gimple_build_assign (iter3, build_simple_mem_ref (def));
- gsi_insert_before (&gsi, load, GSI_NEW_STMT);
-
- tree array = node->simdclone->args[i].simd_array;
- TREE_ADDRESSABLE (array) = 1;
- tree ptr = build_fold_addr_expr (array);
- phi = create_phi_node (iter1, body_bb);
- add_phi_arg (phi, ptr, preheader_edge, UNKNOWN_LOCATION);
- add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
- g = gimple_build_assign (iter2, POINTER_PLUS_EXPR, iter1,
- TYPE_SIZE_UNIT (TREE_TYPE (iter3)));
- gsi = gsi_last_bb (incr_bb);
- gsi_insert_before (&gsi, g, GSI_SAME_STMT);
-
- phi = create_phi_node (iter4, body_bb);
- add_phi_arg (phi, iter3, preheader_edge, UNKNOWN_LOCATION);
- add_phi_arg (phi, iter5, latch_edge, UNKNOWN_LOCATION);
- enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
- ? PLUS_EXPR : POINTER_PLUS_EXPR;
- tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
- ? TREE_TYPE (iter3) : sizetype;
- tree addcst = simd_clone_linear_addend (node, i, addtype,
- entry_bb);
- g = gimple_build_assign (iter5, code, iter4, addcst);
- gsi = gsi_last_bb (incr_bb);
- gsi_insert_before (&gsi, g, GSI_SAME_STMT);
-
- g = gimple_build_assign (build_simple_mem_ref (iter1), iter4);
- gsi = gsi_after_labels (body_bb);
- gsi_insert_before (&gsi, g, GSI_SAME_STMT);
-
- imm_use_iterator iter;
- use_operand_p use_p;
- gimple *use_stmt;
- FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
- if (use_stmt == load)
- continue;
- else
- FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
- SET_USE (use_p, iter1);
-
- if (!TYPE_READONLY (rtype))
- {
- tree v = make_ssa_name (rtype);
- tree aref = build4 (ARRAY_REF, rtype, array,
- size_zero_node, NULL_TREE,
- NULL_TREE);
- gsi = gsi_after_labels (new_exit_bb);
- g = gimple_build_assign (v, aref);
- gsi_insert_before (&gsi, g, GSI_SAME_STMT);
- g = gimple_build_assign (build_simple_mem_ref (def), v);
- gsi_insert_before (&gsi, g, GSI_SAME_STMT);
- }
- }
- }
-
- calculate_dominance_info (CDI_DOMINATORS);
- add_loop (loop, loop->header->loop_father);
- update_ssa (TODO_update_ssa);
-
- pop_cfun ();
-}
-
-/* If the function in NODE is tagged as an elemental SIMD function,
- create the appropriate SIMD clones. */
-
-static void
-expand_simd_clones (struct cgraph_node *node)
-{
- tree attr = lookup_attribute ("omp declare simd",
- DECL_ATTRIBUTES (node->decl));
- if (attr == NULL_TREE
- || node->global.inlined_to
- || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
- return;
-
- /* Ignore
- #pragma omp declare simd
- extern int foo ();
- in C, there we don't know the argument types at all. */
- if (!node->definition
- && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
- return;
-
- /* Call this before creating clone_info, as it might ggc_collect. */
- if (node->definition && node->has_gimple_body_p ())
- node->get_body ();
-
- do
- {
- /* Start with parsing the "omp declare simd" attribute(s). */
- bool inbranch_clause_specified;
- struct cgraph_simd_clone *clone_info
- = simd_clone_clauses_extract (node, TREE_VALUE (attr),
- &inbranch_clause_specified);
- if (clone_info == NULL)
- continue;
-
- int orig_simdlen = clone_info->simdlen;
- tree base_type = simd_clone_compute_base_data_type (node, clone_info);
- /* The target can return 0 (no simd clones should be created),
- 1 (just one ISA of simd clones should be created) or higher
- count of ISA variants. In that case, clone_info is initialized
- for the first ISA variant. */
- int count
- = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
- base_type, 0);
- if (count == 0)
- continue;
-
- /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
- also create one inbranch and one !inbranch clone of it. */
- for (int i = 0; i < count * 2; i++)
- {
- struct cgraph_simd_clone *clone = clone_info;
- if (inbranch_clause_specified && (i & 1) != 0)
- continue;
-
- if (i != 0)
- {
- clone = simd_clone_struct_alloc (clone_info->nargs
- + ((i & 1) != 0));
- simd_clone_struct_copy (clone, clone_info);
- /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
- and simd_clone_adjust_argument_types did to the first
- clone's info. */
- clone->nargs -= clone_info->inbranch;
- clone->simdlen = orig_simdlen;
- /* And call the target hook again to get the right ISA. */
- targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
- base_type,
- i / 2);
- if ((i & 1) != 0)
- clone->inbranch = 1;
- }
-
- /* simd_clone_mangle might fail if such a clone has been created
- already. */
- tree id = simd_clone_mangle (node, clone);
- if (id == NULL_TREE)
- continue;
-
- /* Only when we are sure we want to create the clone actually
- clone the function (or definitions) or create another
- extern FUNCTION_DECL (for prototypes without definitions). */
- struct cgraph_node *n = simd_clone_create (node);
- if (n == NULL)
- continue;
-
- n->simdclone = clone;
- clone->origin = node;
- clone->next_clone = NULL;
- if (node->simd_clones == NULL)
- {
- clone->prev_clone = n;
- node->simd_clones = n;
- }
- else
- {
- clone->prev_clone = node->simd_clones->simdclone->prev_clone;
- clone->prev_clone->simdclone->next_clone = n;
- node->simd_clones->simdclone->prev_clone = n;
- }
- symtab->change_decl_assembler_name (n->decl, id);
- /* And finally adjust the return type, parameters and for
- definitions also function body. */
- if (node->definition)
- simd_clone_adjust (n);
- else
- {
- simd_clone_adjust_return_type (n);
- simd_clone_adjust_argument_types (n);
- }
- }
- }
- while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
-}
-
-/* Entry point for IPA simd clone creation pass. */
-
-static unsigned int
-ipa_omp_simd_clone (void)
-{
- struct cgraph_node *node;
- FOR_EACH_FUNCTION (node)
- expand_simd_clones (node);
- return 0;
-}
-
-namespace {
-
-const pass_data pass_data_omp_simd_clone =
-{
- SIMPLE_IPA_PASS, /* type */
- "simdclone", /* name */
- OPTGROUP_NONE, /* optinfo_flags */
- TV_NONE, /* tv_id */
- ( PROP_ssa | PROP_cfg ), /* properties_required */
- 0, /* properties_provided */
- 0, /* properties_destroyed */
- 0, /* todo_flags_start */
- 0, /* todo_flags_finish */
-};
-
-class pass_omp_simd_clone : public simple_ipa_opt_pass
-{
-public:
- pass_omp_simd_clone(gcc::context *ctxt)
- : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
- {}
-
- /* opt_pass methods: */
- virtual bool gate (function *);
- virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
-};
-
-bool
-pass_omp_simd_clone::gate (function *)
-{
- return targetm.simd_clone.compute_vecsize_and_simdlen != NULL;
-}
-
-} // anon namespace
-
-simple_ipa_opt_pass *
-make_pass_omp_simd_clone (gcc::context *ctxt)
-{
- return new pass_omp_simd_clone (ctxt);
-}
-
/* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
adds their addresses and sizes to constructor-vector V_CTOR. */
static void
--- /dev/null
+/* OMP constructs' SIMD clone supporting code.
+
+Copyright (C) 2005-2016 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3. If not see
+<http://www.gnu.org/licenses/>. */
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "backend.h"
+#include "target.h"
+#include "tree.h"
+#include "gimple.h"
+#include "cfghooks.h"
+#include "alloc-pool.h"
+#include "tree-pass.h"
+#include "ssa.h"
+#include "cgraph.h"
+#include "pretty-print.h"
+#include "diagnostic-core.h"
+#include "fold-const.h"
+#include "stor-layout.h"
+#include "cfganal.h"
+#include "gimplify.h"
+#include "gimple-iterator.h"
+#include "gimplify-me.h"
+#include "gimple-walk.h"
+#include "langhooks.h"
+#include "tree-cfg.h"
+#include "tree-into-ssa.h"
+#include "tree-dfa.h"
+#include "cfgloop.h"
+#include "symbol-summary.h"
+#include "ipa-prop.h"
+#include "tree-eh.h"
+
+
+/* Allocate a fresh `simd_clone' and return it. NARGS is the number
+ of arguments to reserve space for. */
+
+static struct cgraph_simd_clone *
+simd_clone_struct_alloc (int nargs)
+{
+ struct cgraph_simd_clone *clone_info;
+ size_t len = (sizeof (struct cgraph_simd_clone)
+ + nargs * sizeof (struct cgraph_simd_clone_arg));
+ clone_info = (struct cgraph_simd_clone *)
+ ggc_internal_cleared_alloc (len);
+ return clone_info;
+}
+
+/* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
+
+static inline void
+simd_clone_struct_copy (struct cgraph_simd_clone *to,
+ struct cgraph_simd_clone *from)
+{
+ memcpy (to, from, (sizeof (struct cgraph_simd_clone)
+ + ((from->nargs - from->inbranch)
+ * sizeof (struct cgraph_simd_clone_arg))));
+}
+
+/* Return vector of parameter types of function FNDECL. This uses
+ TYPE_ARG_TYPES if available, otherwise falls back to types of
+ DECL_ARGUMENTS types. */
+
+static vec<tree>
+simd_clone_vector_of_formal_parm_types (tree fndecl)
+{
+ if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
+ return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl));
+ vec<tree> args = ipa_get_vector_of_formal_parms (fndecl);
+ unsigned int i;
+ tree arg;
+ FOR_EACH_VEC_ELT (args, i, arg)
+ args[i] = TREE_TYPE (args[i]);
+ return args;
+}
+
+/* Given a simd function in NODE, extract the simd specific
+ information from the OMP clauses passed in CLAUSES, and return
+ the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
+ is set to TRUE if the `inbranch' or `notinbranch' clause specified,
+ otherwise set to FALSE. */
+
+static struct cgraph_simd_clone *
+simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
+ bool *inbranch_specified)
+{
+ vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl);
+ tree t;
+ int n;
+ *inbranch_specified = false;
+
+ n = args.length ();
+ if (n > 0 && args.last () == void_type_node)
+ n--;
+
+ /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
+ be cloned have a distinctive artificial label in addition to "omp
+ declare simd". */
+ bool cilk_clone
+ = (flag_cilkplus
+ && lookup_attribute ("cilk simd function",
+ DECL_ATTRIBUTES (node->decl)));
+
+ /* Allocate one more than needed just in case this is an in-branch
+ clone which will require a mask argument. */
+ struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
+ clone_info->nargs = n;
+ clone_info->cilk_elemental = cilk_clone;
+
+ if (!clauses)
+ {
+ args.release ();
+ return clone_info;
+ }
+ clauses = TREE_VALUE (clauses);
+ if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
+ return clone_info;
+
+ for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
+ {
+ switch (OMP_CLAUSE_CODE (t))
+ {
+ case OMP_CLAUSE_INBRANCH:
+ clone_info->inbranch = 1;
+ *inbranch_specified = true;
+ break;
+ case OMP_CLAUSE_NOTINBRANCH:
+ clone_info->inbranch = 0;
+ *inbranch_specified = true;
+ break;
+ case OMP_CLAUSE_SIMDLEN:
+ clone_info->simdlen
+ = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
+ break;
+ case OMP_CLAUSE_LINEAR:
+ {
+ tree decl = OMP_CLAUSE_DECL (t);
+ tree step = OMP_CLAUSE_LINEAR_STEP (t);
+ int argno = TREE_INT_CST_LOW (decl);
+ if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
+ {
+ enum cgraph_simd_clone_arg_type arg_type;
+ if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
+ switch (OMP_CLAUSE_LINEAR_KIND (t))
+ {
+ case OMP_CLAUSE_LINEAR_REF:
+ arg_type
+ = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP;
+ break;
+ case OMP_CLAUSE_LINEAR_UVAL:
+ arg_type
+ = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP;
+ break;
+ case OMP_CLAUSE_LINEAR_VAL:
+ case OMP_CLAUSE_LINEAR_DEFAULT:
+ arg_type
+ = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ else
+ arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
+ clone_info->args[argno].arg_type = arg_type;
+ clone_info->args[argno].linear_step = tree_to_shwi (step);
+ gcc_assert (clone_info->args[argno].linear_step >= 0
+ && clone_info->args[argno].linear_step < n);
+ }
+ else
+ {
+ if (POINTER_TYPE_P (args[argno]))
+ step = fold_convert (ssizetype, step);
+ if (!tree_fits_shwi_p (step))
+ {
+ warning_at (OMP_CLAUSE_LOCATION (t), 0,
+ "ignoring large linear step");
+ args.release ();
+ return NULL;
+ }
+ else if (integer_zerop (step))
+ {
+ warning_at (OMP_CLAUSE_LOCATION (t), 0,
+ "ignoring zero linear step");
+ args.release ();
+ return NULL;
+ }
+ else
+ {
+ enum cgraph_simd_clone_arg_type arg_type;
+ if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
+ switch (OMP_CLAUSE_LINEAR_KIND (t))
+ {
+ case OMP_CLAUSE_LINEAR_REF:
+ arg_type
+ = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP;
+ break;
+ case OMP_CLAUSE_LINEAR_UVAL:
+ arg_type
+ = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP;
+ break;
+ case OMP_CLAUSE_LINEAR_VAL:
+ case OMP_CLAUSE_LINEAR_DEFAULT:
+ arg_type
+ = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ else
+ arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
+ clone_info->args[argno].arg_type = arg_type;
+ clone_info->args[argno].linear_step = tree_to_shwi (step);
+ }
+ }
+ break;
+ }
+ case OMP_CLAUSE_UNIFORM:
+ {
+ tree decl = OMP_CLAUSE_DECL (t);
+ int argno = tree_to_uhwi (decl);
+ clone_info->args[argno].arg_type
+ = SIMD_CLONE_ARG_TYPE_UNIFORM;
+ break;
+ }
+ case OMP_CLAUSE_ALIGNED:
+ {
+ tree decl = OMP_CLAUSE_DECL (t);
+ int argno = tree_to_uhwi (decl);
+ clone_info->args[argno].alignment
+ = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
+ break;
+ }
+ default:
+ break;
+ }
+ }
+ args.release ();
+ return clone_info;
+}
+
+/* Given a SIMD clone in NODE, calculate the characteristic data
+ type and return the coresponding type. The characteristic data
+ type is computed as described in the Intel Vector ABI. */
+
+static tree
+simd_clone_compute_base_data_type (struct cgraph_node *node,
+ struct cgraph_simd_clone *clone_info)
+{
+ tree type = integer_type_node;
+ tree fndecl = node->decl;
+
+ /* a) For non-void function, the characteristic data type is the
+ return type. */
+ if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
+ type = TREE_TYPE (TREE_TYPE (fndecl));
+
+ /* b) If the function has any non-uniform, non-linear parameters,
+ then the characteristic data type is the type of the first
+ such parameter. */
+ else
+ {
+ vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl);
+ for (unsigned int i = 0; i < clone_info->nargs; ++i)
+ if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
+ {
+ type = map[i];
+ break;
+ }
+ map.release ();
+ }
+
+ /* c) If the characteristic data type determined by a) or b) above
+ is struct, union, or class type which is pass-by-value (except
+ for the type that maps to the built-in complex data type), the
+ characteristic data type is int. */
+ if (RECORD_OR_UNION_TYPE_P (type)
+ && !aggregate_value_p (type, NULL)
+ && TREE_CODE (type) != COMPLEX_TYPE)
+ return integer_type_node;
+
+ /* d) If none of the above three classes is applicable, the
+ characteristic data type is int. */
+
+ return type;
+
+ /* e) For Intel Xeon Phi native and offload compilation, if the
+ resulting characteristic data type is 8-bit or 16-bit integer
+ data type, the characteristic data type is int. */
+ /* Well, we don't handle Xeon Phi yet. */
+}
+
+static tree
+simd_clone_mangle (struct cgraph_node *node,
+ struct cgraph_simd_clone *clone_info)
+{
+ char vecsize_mangle = clone_info->vecsize_mangle;
+ char mask = clone_info->inbranch ? 'M' : 'N';
+ unsigned int simdlen = clone_info->simdlen;
+ unsigned int n;
+ pretty_printer pp;
+
+ gcc_assert (vecsize_mangle && simdlen);
+
+ pp_string (&pp, "_ZGV");
+ pp_character (&pp, vecsize_mangle);
+ pp_character (&pp, mask);
+ pp_decimal_int (&pp, simdlen);
+
+ for (n = 0; n < clone_info->nargs; ++n)
+ {
+ struct cgraph_simd_clone_arg arg = clone_info->args[n];
+
+ switch (arg.arg_type)
+ {
+ case SIMD_CLONE_ARG_TYPE_UNIFORM:
+ pp_character (&pp, 'u');
+ break;
+ case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
+ pp_character (&pp, 'l');
+ goto mangle_linear;
+ case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
+ pp_character (&pp, 'R');
+ goto mangle_linear;
+ case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
+ pp_character (&pp, 'L');
+ goto mangle_linear;
+ case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
+ pp_character (&pp, 'U');
+ goto mangle_linear;
+ mangle_linear:
+ gcc_assert (arg.linear_step != 0);
+ if (arg.linear_step > 1)
+ pp_unsigned_wide_integer (&pp, arg.linear_step);
+ else if (arg.linear_step < 0)
+ {
+ pp_character (&pp, 'n');
+ pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
+ arg.linear_step));
+ }
+ break;
+ case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
+ pp_string (&pp, "ls");
+ pp_unsigned_wide_integer (&pp, arg.linear_step);
+ break;
+ case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
+ pp_string (&pp, "Rs");
+ pp_unsigned_wide_integer (&pp, arg.linear_step);
+ break;
+ case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
+ pp_string (&pp, "Ls");
+ pp_unsigned_wide_integer (&pp, arg.linear_step);
+ break;
+ case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
+ pp_string (&pp, "Us");
+ pp_unsigned_wide_integer (&pp, arg.linear_step);
+ break;
+ default:
+ pp_character (&pp, 'v');
+ }
+ if (arg.alignment)
+ {
+ pp_character (&pp, 'a');
+ pp_decimal_int (&pp, arg.alignment);
+ }
+ }
+
+ pp_underscore (&pp);
+ const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
+ if (*str == '*')
+ ++str;
+ pp_string (&pp, str);
+ str = pp_formatted_text (&pp);
+
+ /* If there already is a SIMD clone with the same mangled name, don't
+ add another one. This can happen e.g. for
+ #pragma omp declare simd
+ #pragma omp declare simd simdlen(8)
+ int foo (int, int);
+ if the simdlen is assumed to be 8 for the first one, etc. */
+ for (struct cgraph_node *clone = node->simd_clones; clone;
+ clone = clone->simdclone->next_clone)
+ if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone->decl)),
+ str) == 0)
+ return NULL_TREE;
+
+ return get_identifier (str);
+}
+
+/* Create a simd clone of OLD_NODE and return it. */
+
+static struct cgraph_node *
+simd_clone_create (struct cgraph_node *old_node)
+{
+ struct cgraph_node *new_node;
+ if (old_node->definition)
+ {
+ if (!old_node->has_gimple_body_p ())
+ return NULL;
+ old_node->get_body ();
+ new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
+ false, NULL, NULL,
+ "simdclone");
+ }
+ else
+ {
+ tree old_decl = old_node->decl;
+ tree new_decl = copy_node (old_node->decl);
+ DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone");
+ SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
+ SET_DECL_RTL (new_decl, NULL);
+ DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
+ DECL_STATIC_DESTRUCTOR (new_decl) = 0;
+ new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
+ if (old_node->in_other_partition)
+ new_node->in_other_partition = 1;
+ }
+ if (new_node == NULL)
+ return new_node;
+
+ TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
+
+ /* The function cgraph_function_versioning () will force the new
+ symbol local. Undo this, and inherit external visability from
+ the old node. */
+ new_node->local.local = old_node->local.local;
+ new_node->externally_visible = old_node->externally_visible;
+
+ return new_node;
+}
+
+/* Adjust the return type of the given function to its appropriate
+ vector counterpart. Returns a simd array to be used throughout the
+ function as a return value. */
+
+static tree
+simd_clone_adjust_return_type (struct cgraph_node *node)
+{
+ tree fndecl = node->decl;
+ tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
+ unsigned int veclen;
+ tree t;
+
+ /* Adjust the function return type. */
+ if (orig_rettype == void_type_node)
+ return NULL_TREE;
+ TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl));
+ t = TREE_TYPE (TREE_TYPE (fndecl));
+ if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t))
+ veclen = node->simdclone->vecsize_int;
+ else
+ veclen = node->simdclone->vecsize_float;
+ veclen /= GET_MODE_BITSIZE (TYPE_MODE (t));
+ if (veclen > node->simdclone->simdlen)
+ veclen = node->simdclone->simdlen;
+ if (POINTER_TYPE_P (t))
+ t = pointer_sized_int_node;
+ if (veclen == node->simdclone->simdlen)
+ t = build_vector_type (t, node->simdclone->simdlen);
+ else
+ {
+ t = build_vector_type (t, veclen);
+ t = build_array_type_nelts (t, node->simdclone->simdlen / veclen);
+ }
+ TREE_TYPE (TREE_TYPE (fndecl)) = t;
+ if (!node->definition)
+ return NULL_TREE;
+
+ t = DECL_RESULT (fndecl);
+ /* Adjust the DECL_RESULT. */
+ gcc_assert (TREE_TYPE (t) != void_type_node);
+ TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
+ relayout_decl (t);
+
+ tree atype = build_array_type_nelts (orig_rettype,
+ node->simdclone->simdlen);
+ if (veclen != node->simdclone->simdlen)
+ return build1 (VIEW_CONVERT_EXPR, atype, t);
+
+ /* Set up a SIMD array to use as the return value. */
+ tree retval = create_tmp_var_raw (atype, "retval");
+ gimple_add_tmp_var (retval);
+ return retval;
+}
+
+/* Each vector argument has a corresponding array to be used locally
+ as part of the eventual loop. Create such temporary array and
+ return it.
+
+ PREFIX is the prefix to be used for the temporary.
+
+ TYPE is the inner element type.
+
+ SIMDLEN is the number of elements. */
+
+static tree
+create_tmp_simd_array (const char *prefix, tree type, int simdlen)
+{
+ tree atype = build_array_type_nelts (type, simdlen);
+ tree avar = create_tmp_var_raw (atype, prefix);
+ gimple_add_tmp_var (avar);
+ return avar;
+}
+
+/* Modify the function argument types to their corresponding vector
+ counterparts if appropriate. Also, create one array for each simd
+ argument to be used locally when using the function arguments as
+ part of the loop.
+
+ NODE is the function whose arguments are to be adjusted.
+
+ Returns an adjustment vector that will be filled describing how the
+ argument types will be adjusted. */
+
+static ipa_parm_adjustment_vec
+simd_clone_adjust_argument_types (struct cgraph_node *node)
+{
+ vec<tree> args;
+ ipa_parm_adjustment_vec adjustments;
+
+ if (node->definition)
+ args = ipa_get_vector_of_formal_parms (node->decl);
+ else
+ args = simd_clone_vector_of_formal_parm_types (node->decl);
+ adjustments.create (args.length ());
+ unsigned i, j, veclen;
+ struct ipa_parm_adjustment adj;
+ struct cgraph_simd_clone *sc = node->simdclone;
+
+ for (i = 0; i < sc->nargs; ++i)
+ {
+ memset (&adj, 0, sizeof (adj));
+ tree parm = args[i];
+ tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
+ adj.base_index = i;
+ adj.base = parm;
+
+ sc->args[i].orig_arg = node->definition ? parm : NULL_TREE;
+ sc->args[i].orig_type = parm_type;
+
+ switch (sc->args[i].arg_type)
+ {
+ default:
+ /* No adjustment necessary for scalar arguments. */
+ adj.op = IPA_PARM_OP_COPY;
+ break;
+ case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
+ case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
+ if (node->definition)
+ sc->args[i].simd_array
+ = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
+ TREE_TYPE (parm_type),
+ sc->simdlen);
+ adj.op = IPA_PARM_OP_COPY;
+ break;
+ case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
+ case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
+ case SIMD_CLONE_ARG_TYPE_VECTOR:
+ if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
+ veclen = sc->vecsize_int;
+ else
+ veclen = sc->vecsize_float;
+ veclen /= GET_MODE_BITSIZE (TYPE_MODE (parm_type));
+ if (veclen > sc->simdlen)
+ veclen = sc->simdlen;
+ adj.arg_prefix = "simd";
+ if (POINTER_TYPE_P (parm_type))
+ adj.type = build_vector_type (pointer_sized_int_node, veclen);
+ else
+ adj.type = build_vector_type (parm_type, veclen);
+ sc->args[i].vector_type = adj.type;
+ for (j = veclen; j < sc->simdlen; j += veclen)
+ {
+ adjustments.safe_push (adj);
+ if (j == veclen)
+ {
+ memset (&adj, 0, sizeof (adj));
+ adj.op = IPA_PARM_OP_NEW;
+ adj.arg_prefix = "simd";
+ adj.base_index = i;
+ adj.type = sc->args[i].vector_type;
+ }
+ }
+
+ if (node->definition)
+ sc->args[i].simd_array
+ = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
+ parm_type, sc->simdlen);
+ }
+ adjustments.safe_push (adj);
+ }
+
+ if (sc->inbranch)
+ {
+ tree base_type = simd_clone_compute_base_data_type (sc->origin, sc);
+
+ memset (&adj, 0, sizeof (adj));
+ adj.op = IPA_PARM_OP_NEW;
+ adj.arg_prefix = "mask";
+
+ adj.base_index = i;
+ if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
+ veclen = sc->vecsize_int;
+ else
+ veclen = sc->vecsize_float;
+ veclen /= GET_MODE_BITSIZE (TYPE_MODE (base_type));
+ if (veclen > sc->simdlen)
+ veclen = sc->simdlen;
+ if (sc->mask_mode != VOIDmode)
+ adj.type
+ = lang_hooks.types.type_for_mode (sc->mask_mode, 1);
+ else if (POINTER_TYPE_P (base_type))
+ adj.type = build_vector_type (pointer_sized_int_node, veclen);
+ else
+ adj.type = build_vector_type (base_type, veclen);
+ adjustments.safe_push (adj);
+
+ for (j = veclen; j < sc->simdlen; j += veclen)
+ adjustments.safe_push (adj);
+
+ /* We have previously allocated one extra entry for the mask. Use
+ it and fill it. */
+ sc->nargs++;
+ if (sc->mask_mode != VOIDmode)
+ base_type = boolean_type_node;
+ if (node->definition)
+ {
+ sc->args[i].orig_arg
+ = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
+ if (sc->mask_mode == VOIDmode)
+ sc->args[i].simd_array
+ = create_tmp_simd_array ("mask", base_type, sc->simdlen);
+ else if (veclen < sc->simdlen)
+ sc->args[i].simd_array
+ = create_tmp_simd_array ("mask", adj.type, sc->simdlen / veclen);
+ else
+ sc->args[i].simd_array = NULL_TREE;
+ }
+ sc->args[i].orig_type = base_type;
+ sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
+ }
+
+ if (node->definition)
+ ipa_modify_formal_parameters (node->decl, adjustments);
+ else
+ {
+ tree new_arg_types = NULL_TREE, new_reversed;
+ bool last_parm_void = false;
+ if (args.length () > 0 && args.last () == void_type_node)
+ last_parm_void = true;
+
+ gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
+ j = adjustments.length ();
+ for (i = 0; i < j; i++)
+ {
+ struct ipa_parm_adjustment *adj = &adjustments[i];
+ tree ptype;
+ if (adj->op == IPA_PARM_OP_COPY)
+ ptype = args[adj->base_index];
+ else
+ ptype = adj->type;
+ new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
+ }
+ new_reversed = nreverse (new_arg_types);
+ if (last_parm_void)
+ {
+ if (new_reversed)
+ TREE_CHAIN (new_arg_types) = void_list_node;
+ else
+ new_reversed = void_list_node;
+ }
+
+ tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl));
+ TYPE_ARG_TYPES (new_type) = new_reversed;
+ TREE_TYPE (node->decl) = new_type;
+
+ adjustments.release ();
+ }
+ args.release ();
+ return adjustments;
+}
+
+/* Initialize and copy the function arguments in NODE to their
+ corresponding local simd arrays. Returns a fresh gimple_seq with
+ the instruction sequence generated. */
+
+static gimple_seq
+simd_clone_init_simd_arrays (struct cgraph_node *node,
+ ipa_parm_adjustment_vec adjustments)
+{
+ gimple_seq seq = NULL;
+ unsigned i = 0, j = 0, k;
+
+ for (tree arg = DECL_ARGUMENTS (node->decl);
+ arg;
+ arg = DECL_CHAIN (arg), i++, j++)
+ {
+ if (adjustments[j].op == IPA_PARM_OP_COPY
+ || POINTER_TYPE_P (TREE_TYPE (arg)))
+ continue;
+
+ node->simdclone->args[i].vector_arg = arg;
+
+ tree array = node->simdclone->args[i].simd_array;
+ if (node->simdclone->mask_mode != VOIDmode
+ && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK)
+ {
+ if (array == NULL_TREE)
+ continue;
+ unsigned int l
+ = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array))));
+ for (k = 0; k <= l; k++)
+ {
+ if (k)
+ {
+ arg = DECL_CHAIN (arg);
+ j++;
+ }
+ tree t = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (array)),
+ array, size_int (k), NULL, NULL);
+ t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
+ gimplify_and_add (t, &seq);
+ }
+ continue;
+ }
+ if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)) == node->simdclone->simdlen)
+ {
+ tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
+ tree ptr = build_fold_addr_expr (array);
+ tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
+ build_int_cst (ptype, 0));
+ t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
+ gimplify_and_add (t, &seq);
+ }
+ else
+ {
+ unsigned int simdlen = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg));
+ tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
+ for (k = 0; k < node->simdclone->simdlen; k += simdlen)
+ {
+ tree ptr = build_fold_addr_expr (array);
+ int elemsize;
+ if (k)
+ {
+ arg = DECL_CHAIN (arg);
+ j++;
+ }
+ elemsize
+ = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg))));
+ tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
+ build_int_cst (ptype, k * elemsize));
+ t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
+ gimplify_and_add (t, &seq);
+ }
+ }
+ }
+ return seq;
+}
+
+/* Callback info for ipa_simd_modify_stmt_ops below. */
+
+struct modify_stmt_info {
+ ipa_parm_adjustment_vec adjustments;
+ gimple *stmt;
+ /* True if the parent statement was modified by
+ ipa_simd_modify_stmt_ops. */
+ bool modified;
+};
+
+/* Callback for walk_gimple_op.
+
+ Adjust operands from a given statement as specified in the
+ adjustments vector in the callback data. */
+
+static tree
+ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
+{
+ struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
+ struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
+ tree *orig_tp = tp;
+ if (TREE_CODE (*tp) == ADDR_EXPR)
+ tp = &TREE_OPERAND (*tp, 0);
+ struct ipa_parm_adjustment *cand = NULL;
+ if (TREE_CODE (*tp) == PARM_DECL)
+ cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true);
+ else
+ {
+ if (TYPE_P (*tp))
+ *walk_subtrees = 0;
+ }
+
+ tree repl = NULL_TREE;
+ if (cand)
+ repl = unshare_expr (cand->new_decl);
+ else
+ {
+ if (tp != orig_tp)
+ {
+ *walk_subtrees = 0;
+ bool modified = info->modified;
+ info->modified = false;
+ walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
+ if (!info->modified)
+ {
+ info->modified = modified;
+ return NULL_TREE;
+ }
+ info->modified = modified;
+ repl = *tp;
+ }
+ else
+ return NULL_TREE;
+ }
+
+ if (tp != orig_tp)
+ {
+ repl = build_fold_addr_expr (repl);
+ gimple *stmt;
+ if (is_gimple_debug (info->stmt))
+ {
+ tree vexpr = make_node (DEBUG_EXPR_DECL);
+ stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
+ DECL_ARTIFICIAL (vexpr) = 1;
+ TREE_TYPE (vexpr) = TREE_TYPE (repl);
+ DECL_MODE (vexpr) = TYPE_MODE (TREE_TYPE (repl));
+ repl = vexpr;
+ }
+ else
+ {
+ stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
+ repl = gimple_assign_lhs (stmt);
+ }
+ gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt);
+ gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
+ *orig_tp = repl;
+ }
+ else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
+ {
+ tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
+ *tp = vce;
+ }
+ else
+ *tp = repl;
+
+ info->modified = true;
+ return NULL_TREE;
+}
+
+/* Traverse the function body and perform all modifications as
+ described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
+ modified such that the replacement/reduction value will now be an
+ offset into the corresponding simd_array.
+
+ This function will replace all function argument uses with their
+ corresponding simd array elements, and ajust the return values
+ accordingly. */
+
+static void
+ipa_simd_modify_function_body (struct cgraph_node *node,
+ ipa_parm_adjustment_vec adjustments,
+ tree retval_array, tree iter)
+{
+ basic_block bb;
+ unsigned int i, j, l;
+
+ /* Re-use the adjustments array, but this time use it to replace
+ every function argument use to an offset into the corresponding
+ simd_array. */
+ for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
+ {
+ if (!node->simdclone->args[i].vector_arg)
+ continue;
+
+ tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
+ tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
+ adjustments[j].new_decl
+ = build4 (ARRAY_REF,
+ basetype,
+ node->simdclone->args[i].simd_array,
+ iter,
+ NULL_TREE, NULL_TREE);
+ if (adjustments[j].op == IPA_PARM_OP_NONE
+ && TYPE_VECTOR_SUBPARTS (vectype) < node->simdclone->simdlen)
+ j += node->simdclone->simdlen / TYPE_VECTOR_SUBPARTS (vectype) - 1;
+ }
+
+ l = adjustments.length ();
+ for (i = 1; i < num_ssa_names; i++)
+ {
+ tree name = ssa_name (i);
+ if (name
+ && SSA_NAME_VAR (name)
+ && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL)
+ {
+ for (j = 0; j < l; j++)
+ if (SSA_NAME_VAR (name) == adjustments[j].base
+ && adjustments[j].new_decl)
+ {
+ tree base_var;
+ if (adjustments[j].new_ssa_base == NULL_TREE)
+ {
+ base_var
+ = copy_var_decl (adjustments[j].base,
+ DECL_NAME (adjustments[j].base),
+ TREE_TYPE (adjustments[j].base));
+ adjustments[j].new_ssa_base = base_var;
+ }
+ else
+ base_var = adjustments[j].new_ssa_base;
+ if (SSA_NAME_IS_DEFAULT_DEF (name))
+ {
+ bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
+ gimple_stmt_iterator gsi = gsi_after_labels (bb);
+ tree new_decl = unshare_expr (adjustments[j].new_decl);
+ set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE);
+ SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
+ SSA_NAME_IS_DEFAULT_DEF (name) = 0;
+ gimple *stmt = gimple_build_assign (name, new_decl);
+ gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
+ }
+ else
+ SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
+ }
+ }
+ }
+
+ struct modify_stmt_info info;
+ info.adjustments = adjustments;
+
+ FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
+ {
+ gimple_stmt_iterator gsi;
+
+ gsi = gsi_start_bb (bb);
+ while (!gsi_end_p (gsi))
+ {
+ gimple *stmt = gsi_stmt (gsi);
+ info.stmt = stmt;
+ struct walk_stmt_info wi;
+
+ memset (&wi, 0, sizeof (wi));
+ info.modified = false;
+ wi.info = &info;
+ walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
+
+ if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
+ {
+ tree retval = gimple_return_retval (return_stmt);
+ if (!retval)
+ {
+ gsi_remove (&gsi, true);
+ continue;
+ }
+
+ /* Replace `return foo' with `retval_array[iter] = foo'. */
+ tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
+ retval_array, iter, NULL, NULL);
+ stmt = gimple_build_assign (ref, retval);
+ gsi_replace (&gsi, stmt, true);
+ info.modified = true;
+ }
+
+ if (info.modified)
+ {
+ update_stmt (stmt);
+ if (maybe_clean_eh_stmt (stmt))
+ gimple_purge_dead_eh_edges (gimple_bb (stmt));
+ }
+ gsi_next (&gsi);
+ }
+ }
+}
+
+/* Helper function of simd_clone_adjust, return linear step addend
+ of Ith argument. */
+
+static tree
+simd_clone_linear_addend (struct cgraph_node *node, unsigned int i,
+ tree addtype, basic_block entry_bb)
+{
+ tree ptype = NULL_TREE;
+ switch (node->simdclone->args[i].arg_type)
+ {
+ case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
+ case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
+ case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
+ case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
+ return build_int_cst (addtype, node->simdclone->args[i].linear_step);
+ case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
+ case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
+ ptype = TREE_TYPE (node->simdclone->args[i].orig_arg);
+ break;
+ case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
+ case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
+ ptype = TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg));
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ unsigned int idx = node->simdclone->args[i].linear_step;
+ tree arg = node->simdclone->args[idx].orig_arg;
+ gcc_assert (is_gimple_reg_type (TREE_TYPE (arg)));
+ gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
+ gimple *g;
+ tree ret;
+ if (is_gimple_reg (arg))
+ ret = get_or_create_ssa_default_def (cfun, arg);
+ else
+ {
+ g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg);
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+ ret = gimple_assign_lhs (g);
+ }
+ if (TREE_CODE (TREE_TYPE (arg)) == REFERENCE_TYPE)
+ {
+ g = gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg))),
+ build_simple_mem_ref (ret));
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+ ret = gimple_assign_lhs (g);
+ }
+ if (!useless_type_conversion_p (addtype, TREE_TYPE (ret)))
+ {
+ g = gimple_build_assign (make_ssa_name (addtype), NOP_EXPR, ret);
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+ ret = gimple_assign_lhs (g);
+ }
+ if (POINTER_TYPE_P (ptype))
+ {
+ tree size = TYPE_SIZE_UNIT (TREE_TYPE (ptype));
+ if (size && TREE_CODE (size) == INTEGER_CST)
+ {
+ g = gimple_build_assign (make_ssa_name (addtype), MULT_EXPR,
+ ret, fold_convert (addtype, size));
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+ ret = gimple_assign_lhs (g);
+ }
+ }
+ return ret;
+}
+
+/* Adjust the argument types in NODE to their appropriate vector
+ counterparts. */
+
+static void
+simd_clone_adjust (struct cgraph_node *node)
+{
+ push_cfun (DECL_STRUCT_FUNCTION (node->decl));
+
+ targetm.simd_clone.adjust (node);
+
+ tree retval = simd_clone_adjust_return_type (node);
+ ipa_parm_adjustment_vec adjustments
+ = simd_clone_adjust_argument_types (node);
+
+ push_gimplify_context ();
+
+ gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
+
+ /* Adjust all uses of vector arguments accordingly. Adjust all
+ return values accordingly. */
+ tree iter = create_tmp_var (unsigned_type_node, "iter");
+ tree iter1 = make_ssa_name (iter);
+ tree iter2 = make_ssa_name (iter);
+ ipa_simd_modify_function_body (node, adjustments, retval, iter1);
+
+ /* Initialize the iteration variable. */
+ basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
+ basic_block body_bb = split_block_after_labels (entry_bb)->dest;
+ gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
+ /* Insert the SIMD array and iv initialization at function
+ entry. */
+ gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
+
+ pop_gimplify_context (NULL);
+
+ /* Create a new BB right before the original exit BB, to hold the
+ iteration increment and the condition/branch. */
+ basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
+ basic_block incr_bb = create_empty_bb (orig_exit);
+ add_bb_to_loop (incr_bb, body_bb->loop_father);
+ /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
+ flag. Set it now to be a FALLTHRU_EDGE. */
+ gcc_assert (EDGE_COUNT (orig_exit->succs) == 1);
+ EDGE_SUCC (orig_exit, 0)->flags |= EDGE_FALLTHRU;
+ for (unsigned i = 0;
+ i < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds); ++i)
+ {
+ edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), i);
+ redirect_edge_succ (e, incr_bb);
+ }
+ edge e = make_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
+ e->probability = REG_BR_PROB_BASE;
+ gsi = gsi_last_bb (incr_bb);
+ gimple *g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
+ build_int_cst (unsigned_type_node, 1));
+ gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
+
+ /* Mostly annotate the loop for the vectorizer (the rest is done below). */
+ struct loop *loop = alloc_loop ();
+ cfun->has_force_vectorize_loops = true;
+ loop->safelen = node->simdclone->simdlen;
+ loop->force_vectorize = true;
+ loop->header = body_bb;
+
+ /* Branch around the body if the mask applies. */
+ if (node->simdclone->inbranch)
+ {
+ gimple_stmt_iterator gsi = gsi_last_bb (loop->header);
+ tree mask_array
+ = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
+ tree mask;
+ if (node->simdclone->mask_mode != VOIDmode)
+ {
+ tree shift_cnt;
+ if (mask_array == NULL_TREE)
+ {
+ tree arg = node->simdclone->args[node->simdclone->nargs
+ - 1].vector_arg;
+ mask = get_or_create_ssa_default_def (cfun, arg);
+ shift_cnt = iter1;
+ }
+ else
+ {
+ tree maskt = TREE_TYPE (mask_array);
+ int c = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt)));
+ c = node->simdclone->simdlen / (c + 1);
+ int s = exact_log2 (c);
+ gcc_assert (s > 0);
+ c--;
+ tree idx = make_ssa_name (TREE_TYPE (iter1));
+ g = gimple_build_assign (idx, RSHIFT_EXPR, iter1,
+ build_int_cst (NULL_TREE, s));
+ gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
+ mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
+ tree aref = build4 (ARRAY_REF,
+ TREE_TYPE (TREE_TYPE (mask_array)),
+ mask_array, idx, NULL, NULL);
+ g = gimple_build_assign (mask, aref);
+ gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
+ shift_cnt = make_ssa_name (TREE_TYPE (iter1));
+ g = gimple_build_assign (shift_cnt, BIT_AND_EXPR, iter1,
+ build_int_cst (TREE_TYPE (iter1), c));
+ gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
+ }
+ g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
+ RSHIFT_EXPR, mask, shift_cnt);
+ gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
+ mask = gimple_assign_lhs (g);
+ g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
+ BIT_AND_EXPR, mask,
+ build_int_cst (TREE_TYPE (mask), 1));
+ gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
+ mask = gimple_assign_lhs (g);
+ }
+ else
+ {
+ mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
+ tree aref = build4 (ARRAY_REF,
+ TREE_TYPE (TREE_TYPE (mask_array)),
+ mask_array, iter1, NULL, NULL);
+ g = gimple_build_assign (mask, aref);
+ gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
+ int bitsize = GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref)));
+ if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
+ {
+ aref = build1 (VIEW_CONVERT_EXPR,
+ build_nonstandard_integer_type (bitsize, 0),
+ mask);
+ mask = make_ssa_name (TREE_TYPE (aref));
+ g = gimple_build_assign (mask, aref);
+ gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
+ }
+ }
+
+ g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
+ NULL, NULL);
+ gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
+ make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
+ FALLTHRU_EDGE (loop->header)->flags = EDGE_FALSE_VALUE;
+ }
+
+ /* Generate the condition. */
+ g = gimple_build_cond (LT_EXPR,
+ iter2,
+ build_int_cst (unsigned_type_node,
+ node->simdclone->simdlen),
+ NULL, NULL);
+ gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
+ e = split_block (incr_bb, gsi_stmt (gsi));
+ basic_block latch_bb = e->dest;
+ basic_block new_exit_bb;
+ new_exit_bb = split_block_after_labels (latch_bb)->dest;
+ loop->latch = latch_bb;
+
+ redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
+
+ make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
+ /* The successor of incr_bb is already pointing to latch_bb; just
+ change the flags.
+ make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
+ FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
+
+ gphi *phi = create_phi_node (iter1, body_bb);
+ edge preheader_edge = find_edge (entry_bb, body_bb);
+ edge latch_edge = single_succ_edge (latch_bb);
+ add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
+ UNKNOWN_LOCATION);
+ add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
+
+ /* Generate the new return. */
+ gsi = gsi_last_bb (new_exit_bb);
+ if (retval
+ && TREE_CODE (retval) == VIEW_CONVERT_EXPR
+ && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
+ retval = TREE_OPERAND (retval, 0);
+ else if (retval)
+ {
+ retval = build1 (VIEW_CONVERT_EXPR,
+ TREE_TYPE (TREE_TYPE (node->decl)),
+ retval);
+ retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
+ false, GSI_CONTINUE_LINKING);
+ }
+ g = gimple_build_return (retval);
+ gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
+
+ /* Handle aligned clauses by replacing default defs of the aligned
+ uniform args with __builtin_assume_aligned (arg_N(D), alignment)
+ lhs. Handle linear by adding PHIs. */
+ for (unsigned i = 0; i < node->simdclone->nargs; i++)
+ if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
+ && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg)
+ || !is_gimple_reg_type
+ (TREE_TYPE (node->simdclone->args[i].orig_arg))))
+ {
+ tree orig_arg = node->simdclone->args[i].orig_arg;
+ if (is_gimple_reg_type (TREE_TYPE (orig_arg)))
+ iter1 = make_ssa_name (TREE_TYPE (orig_arg));
+ else
+ {
+ iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg));
+ gimple_add_tmp_var (iter1);
+ }
+ gsi = gsi_after_labels (entry_bb);
+ g = gimple_build_assign (iter1, orig_arg);
+ gsi_insert_before (&gsi, g, GSI_NEW_STMT);
+ gsi = gsi_after_labels (body_bb);
+ g = gimple_build_assign (orig_arg, iter1);
+ gsi_insert_before (&gsi, g, GSI_NEW_STMT);
+ }
+ else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
+ && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg)
+ && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
+ == REFERENCE_TYPE
+ && TREE_ADDRESSABLE
+ (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg))))
+ {
+ tree orig_arg = node->simdclone->args[i].orig_arg;
+ tree def = ssa_default_def (cfun, orig_arg);
+ if (def && !has_zero_uses (def))
+ {
+ iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg)));
+ gimple_add_tmp_var (iter1);
+ gsi = gsi_after_labels (entry_bb);
+ g = gimple_build_assign (iter1, build_simple_mem_ref (def));
+ gsi_insert_before (&gsi, g, GSI_NEW_STMT);
+ gsi = gsi_after_labels (body_bb);
+ g = gimple_build_assign (build_simple_mem_ref (def), iter1);
+ gsi_insert_before (&gsi, g, GSI_NEW_STMT);
+ }
+ }
+ else if (node->simdclone->args[i].alignment
+ && node->simdclone->args[i].arg_type
+ == SIMD_CLONE_ARG_TYPE_UNIFORM
+ && (node->simdclone->args[i].alignment
+ & (node->simdclone->args[i].alignment - 1)) == 0
+ && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
+ == POINTER_TYPE)
+ {
+ unsigned int alignment = node->simdclone->args[i].alignment;
+ tree orig_arg = node->simdclone->args[i].orig_arg;
+ tree def = ssa_default_def (cfun, orig_arg);
+ if (def && !has_zero_uses (def))
+ {
+ tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
+ gimple_seq seq = NULL;
+ bool need_cvt = false;
+ gcall *call
+ = gimple_build_call (fn, 2, def, size_int (alignment));
+ g = call;
+ if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
+ ptr_type_node))
+ need_cvt = true;
+ tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg);
+ gimple_call_set_lhs (g, t);
+ gimple_seq_add_stmt_without_update (&seq, g);
+ if (need_cvt)
+ {
+ t = make_ssa_name (orig_arg);
+ g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g));
+ gimple_seq_add_stmt_without_update (&seq, g);
+ }
+ gsi_insert_seq_on_edge_immediate
+ (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
+
+ entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
+ int freq = compute_call_stmt_bb_frequency (current_function_decl,
+ entry_bb);
+ node->create_edge (cgraph_node::get_create (fn),
+ call, entry_bb->count, freq);
+
+ imm_use_iterator iter;
+ use_operand_p use_p;
+ gimple *use_stmt;
+ tree repl = gimple_get_lhs (g);
+ FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
+ if (is_gimple_debug (use_stmt) || use_stmt == call)
+ continue;
+ else
+ FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
+ SET_USE (use_p, repl);
+ }
+ }
+ else if ((node->simdclone->args[i].arg_type
+ == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
+ || (node->simdclone->args[i].arg_type
+ == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
+ || (node->simdclone->args[i].arg_type
+ == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
+ || (node->simdclone->args[i].arg_type
+ == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP))
+ {
+ tree orig_arg = node->simdclone->args[i].orig_arg;
+ gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
+ || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
+ tree def = NULL_TREE;
+ if (TREE_ADDRESSABLE (orig_arg))
+ {
+ def = make_ssa_name (TREE_TYPE (orig_arg));
+ iter1 = make_ssa_name (TREE_TYPE (orig_arg));
+ iter2 = make_ssa_name (TREE_TYPE (orig_arg));
+ gsi = gsi_after_labels (entry_bb);
+ g = gimple_build_assign (def, orig_arg);
+ gsi_insert_before (&gsi, g, GSI_NEW_STMT);
+ }
+ else
+ {
+ def = ssa_default_def (cfun, orig_arg);
+ if (!def || has_zero_uses (def))
+ def = NULL_TREE;
+ else
+ {
+ iter1 = make_ssa_name (orig_arg);
+ iter2 = make_ssa_name (orig_arg);
+ }
+ }
+ if (def)
+ {
+ phi = create_phi_node (iter1, body_bb);
+ add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
+ add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
+ enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
+ ? PLUS_EXPR : POINTER_PLUS_EXPR;
+ tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
+ ? TREE_TYPE (orig_arg) : sizetype;
+ tree addcst = simd_clone_linear_addend (node, i, addtype,
+ entry_bb);
+ gsi = gsi_last_bb (incr_bb);
+ g = gimple_build_assign (iter2, code, iter1, addcst);
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+
+ imm_use_iterator iter;
+ use_operand_p use_p;
+ gimple *use_stmt;
+ if (TREE_ADDRESSABLE (orig_arg))
+ {
+ gsi = gsi_after_labels (body_bb);
+ g = gimple_build_assign (orig_arg, iter1);
+ gsi_insert_before (&gsi, g, GSI_NEW_STMT);
+ }
+ else
+ FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
+ if (use_stmt == phi)
+ continue;
+ else
+ FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
+ SET_USE (use_p, iter1);
+ }
+ }
+ else if (node->simdclone->args[i].arg_type
+ == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
+ || (node->simdclone->args[i].arg_type
+ == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP))
+ {
+ tree orig_arg = node->simdclone->args[i].orig_arg;
+ tree def = ssa_default_def (cfun, orig_arg);
+ gcc_assert (!TREE_ADDRESSABLE (orig_arg)
+ && TREE_CODE (TREE_TYPE (orig_arg)) == REFERENCE_TYPE);
+ if (def && !has_zero_uses (def))
+ {
+ tree rtype = TREE_TYPE (TREE_TYPE (orig_arg));
+ iter1 = make_ssa_name (orig_arg);
+ iter2 = make_ssa_name (orig_arg);
+ tree iter3 = make_ssa_name (rtype);
+ tree iter4 = make_ssa_name (rtype);
+ tree iter5 = make_ssa_name (rtype);
+ gsi = gsi_after_labels (entry_bb);
+ gimple *load
+ = gimple_build_assign (iter3, build_simple_mem_ref (def));
+ gsi_insert_before (&gsi, load, GSI_NEW_STMT);
+
+ tree array = node->simdclone->args[i].simd_array;
+ TREE_ADDRESSABLE (array) = 1;
+ tree ptr = build_fold_addr_expr (array);
+ phi = create_phi_node (iter1, body_bb);
+ add_phi_arg (phi, ptr, preheader_edge, UNKNOWN_LOCATION);
+ add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
+ g = gimple_build_assign (iter2, POINTER_PLUS_EXPR, iter1,
+ TYPE_SIZE_UNIT (TREE_TYPE (iter3)));
+ gsi = gsi_last_bb (incr_bb);
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+
+ phi = create_phi_node (iter4, body_bb);
+ add_phi_arg (phi, iter3, preheader_edge, UNKNOWN_LOCATION);
+ add_phi_arg (phi, iter5, latch_edge, UNKNOWN_LOCATION);
+ enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
+ ? PLUS_EXPR : POINTER_PLUS_EXPR;
+ tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
+ ? TREE_TYPE (iter3) : sizetype;
+ tree addcst = simd_clone_linear_addend (node, i, addtype,
+ entry_bb);
+ g = gimple_build_assign (iter5, code, iter4, addcst);
+ gsi = gsi_last_bb (incr_bb);
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+
+ g = gimple_build_assign (build_simple_mem_ref (iter1), iter4);
+ gsi = gsi_after_labels (body_bb);
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+
+ imm_use_iterator iter;
+ use_operand_p use_p;
+ gimple *use_stmt;
+ FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
+ if (use_stmt == load)
+ continue;
+ else
+ FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
+ SET_USE (use_p, iter1);
+
+ if (!TYPE_READONLY (rtype))
+ {
+ tree v = make_ssa_name (rtype);
+ tree aref = build4 (ARRAY_REF, rtype, array,
+ size_zero_node, NULL_TREE,
+ NULL_TREE);
+ gsi = gsi_after_labels (new_exit_bb);
+ g = gimple_build_assign (v, aref);
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+ g = gimple_build_assign (build_simple_mem_ref (def), v);
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+ }
+ }
+ }
+
+ calculate_dominance_info (CDI_DOMINATORS);
+ add_loop (loop, loop->header->loop_father);
+ update_ssa (TODO_update_ssa);
+
+ pop_cfun ();
+}
+
+/* If the function in NODE is tagged as an elemental SIMD function,
+ create the appropriate SIMD clones. */
+
+static void
+expand_simd_clones (struct cgraph_node *node)
+{
+ tree attr = lookup_attribute ("omp declare simd",
+ DECL_ATTRIBUTES (node->decl));
+ if (attr == NULL_TREE
+ || node->global.inlined_to
+ || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
+ return;
+
+ /* Ignore
+ #pragma omp declare simd
+ extern int foo ();
+ in C, there we don't know the argument types at all. */
+ if (!node->definition
+ && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
+ return;
+
+ /* Call this before creating clone_info, as it might ggc_collect. */
+ if (node->definition && node->has_gimple_body_p ())
+ node->get_body ();
+
+ do
+ {
+ /* Start with parsing the "omp declare simd" attribute(s). */
+ bool inbranch_clause_specified;
+ struct cgraph_simd_clone *clone_info
+ = simd_clone_clauses_extract (node, TREE_VALUE (attr),
+ &inbranch_clause_specified);
+ if (clone_info == NULL)
+ continue;
+
+ int orig_simdlen = clone_info->simdlen;
+ tree base_type = simd_clone_compute_base_data_type (node, clone_info);
+ /* The target can return 0 (no simd clones should be created),
+ 1 (just one ISA of simd clones should be created) or higher
+ count of ISA variants. In that case, clone_info is initialized
+ for the first ISA variant. */
+ int count
+ = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
+ base_type, 0);
+ if (count == 0)
+ continue;
+
+ /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
+ also create one inbranch and one !inbranch clone of it. */
+ for (int i = 0; i < count * 2; i++)
+ {
+ struct cgraph_simd_clone *clone = clone_info;
+ if (inbranch_clause_specified && (i & 1) != 0)
+ continue;
+
+ if (i != 0)
+ {
+ clone = simd_clone_struct_alloc (clone_info->nargs
+ + ((i & 1) != 0));
+ simd_clone_struct_copy (clone, clone_info);
+ /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
+ and simd_clone_adjust_argument_types did to the first
+ clone's info. */
+ clone->nargs -= clone_info->inbranch;
+ clone->simdlen = orig_simdlen;
+ /* And call the target hook again to get the right ISA. */
+ targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
+ base_type,
+ i / 2);
+ if ((i & 1) != 0)
+ clone->inbranch = 1;
+ }
+
+ /* simd_clone_mangle might fail if such a clone has been created
+ already. */
+ tree id = simd_clone_mangle (node, clone);
+ if (id == NULL_TREE)
+ continue;
+
+ /* Only when we are sure we want to create the clone actually
+ clone the function (or definitions) or create another
+ extern FUNCTION_DECL (for prototypes without definitions). */
+ struct cgraph_node *n = simd_clone_create (node);
+ if (n == NULL)
+ continue;
+
+ n->simdclone = clone;
+ clone->origin = node;
+ clone->next_clone = NULL;
+ if (node->simd_clones == NULL)
+ {
+ clone->prev_clone = n;
+ node->simd_clones = n;
+ }
+ else
+ {
+ clone->prev_clone = node->simd_clones->simdclone->prev_clone;
+ clone->prev_clone->simdclone->next_clone = n;
+ node->simd_clones->simdclone->prev_clone = n;
+ }
+ symtab->change_decl_assembler_name (n->decl, id);
+ /* And finally adjust the return type, parameters and for
+ definitions also function body. */
+ if (node->definition)
+ simd_clone_adjust (n);
+ else
+ {
+ simd_clone_adjust_return_type (n);
+ simd_clone_adjust_argument_types (n);
+ }
+ }
+ }
+ while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
+}
+
+/* Entry point for IPA simd clone creation pass. */
+
+static unsigned int
+ipa_omp_simd_clone (void)
+{
+ struct cgraph_node *node;
+ FOR_EACH_FUNCTION (node)
+ expand_simd_clones (node);
+ return 0;
+}
+
+namespace {
+
+const pass_data pass_data_omp_simd_clone =
+{
+ SIMPLE_IPA_PASS, /* type */
+ "simdclone", /* name */
+ OPTGROUP_NONE, /* optinfo_flags */
+ TV_NONE, /* tv_id */
+ ( PROP_ssa | PROP_cfg ), /* properties_required */
+ 0, /* properties_provided */
+ 0, /* properties_destroyed */
+ 0, /* todo_flags_start */
+ 0, /* todo_flags_finish */
+};
+
+class pass_omp_simd_clone : public simple_ipa_opt_pass
+{
+public:
+ pass_omp_simd_clone(gcc::context *ctxt)
+ : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
+ {}
+
+ /* opt_pass methods: */
+ virtual bool gate (function *);
+ virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
+};
+
+bool
+pass_omp_simd_clone::gate (function *)
+{
+ return targetm.simd_clone.compute_vecsize_and_simdlen != NULL;
+}
+
+} // anon namespace
+
+simple_ipa_opt_pass *
+make_pass_omp_simd_clone (gcc::context *ctxt)
+{
+ return new pass_omp_simd_clone (ctxt);
+}
projects / platform / upstream / gcc.git / commitdiff