The value returned is the depth in the SLP tree where a mismatch
was found. */
-static bool
+static slp_tree
vect_build_slp_tree (vec_info *vinfo,
- slp_tree *node, unsigned int group_size,
+ vec<gimple *> stmts, unsigned int group_size,
unsigned int *max_nunits,
vec<slp_tree> *loads,
bool *matches, unsigned *npermutes, unsigned *tree_size,
unsigned max_tree_size)
{
- unsigned nops, i, this_tree_size = 0;
+ unsigned nops, i, this_tree_size = 0, this_max_nunits = *max_nunits;
gimple *stmt;
+ slp_tree node;
matches[0] = false;
- stmt = SLP_TREE_SCALAR_STMTS (*node)[0];
+ stmt = stmts[0];
if (is_gimple_call (stmt))
nops = gimple_call_num_args (stmt);
else if (is_gimple_assign (stmt))
nops++;
}
else
- return false;
+ return NULL;
bool two_operators = false;
if (!vect_build_slp_tree_1 (vinfo,
- SLP_TREE_SCALAR_STMTS (*node), group_size, nops,
- max_nunits, matches, &two_operators))
- return false;
- SLP_TREE_TWO_OPERATORS (*node) = two_operators;
+ stmts, group_size, nops,
+ &this_max_nunits, matches, &two_operators))
+ return NULL;
/* If the SLP node is a load, terminate the recursion. */
if (STMT_VINFO_GROUPED_ACCESS (vinfo_for_stmt (stmt))
&& DR_IS_READ (STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt))))
{
- loads->safe_push (*node);
- return true;
+ *max_nunits = this_max_nunits;
+ node = vect_create_new_slp_node (stmts);
+ loads->safe_push (node);
+ return node;
}
/* Get at the operands, verifying they are compatible. */
vec<slp_oprnd_info> oprnds_info = vect_create_oprnd_info (nops, group_size);
slp_oprnd_info oprnd_info;
- FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (*node), i, stmt)
+ FOR_EACH_VEC_ELT (stmts, i, stmt)
{
switch (vect_get_and_check_slp_defs (vinfo, stmt, i, &oprnds_info))
{
case -1:
matches[0] = false;
vect_free_oprnd_info (oprnds_info);
- return false;
+ return NULL;
case 1:
matches[i] = false;
break;
if (!matches[i])
{
vect_free_oprnd_info (oprnds_info);
- return false;
+ return NULL;
}
- stmt = SLP_TREE_SCALAR_STMTS (*node)[0];
+ auto_vec<slp_tree, 4> children;
+ auto_vec<slp_tree> this_loads;
+
+ stmt = stmts[0];
/* Create SLP_TREE nodes for the definition node/s. */
FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
{
slp_tree child;
- unsigned old_nloads = loads->length ();
- unsigned old_max_nunits = *max_nunits;
+ unsigned old_nloads = this_loads.length ();
+ unsigned old_tree_size = this_tree_size;
+ unsigned int j;
if (oprnd_info->first_dt != vect_internal_def)
continue;
if (++this_tree_size > max_tree_size)
{
+ FOR_EACH_VEC_ELT (children, j, child)
+ vect_free_slp_tree (child);
vect_free_oprnd_info (oprnds_info);
- return false;
+ return NULL;
}
- child = vect_create_new_slp_node (oprnd_info->def_stmts);
- if (!child)
- {
- vect_free_oprnd_info (oprnds_info);
- return false;
- }
-
- if (vect_build_slp_tree (vinfo, &child,
- group_size, max_nunits, loads, matches,
- npermutes, &this_tree_size, max_tree_size))
+ if ((child = vect_build_slp_tree (vinfo, oprnd_info->def_stmts,
+ group_size, &this_max_nunits,
+ &this_loads, matches, npermutes,
+ &this_tree_size,
+ max_tree_size)) != NULL)
{
/* If we have all children of child built up from scalars then just
throw that away and build it up this node from scalars. */
if (!SLP_TREE_CHILDREN (child).is_empty ())
{
- unsigned int j;
slp_tree grandchild;
FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (child), j, grandchild)
if (!grandchild)
{
/* Roll back. */
- *max_nunits = old_max_nunits;
- loads->truncate (old_nloads);
+ this_loads.truncate (old_nloads);
+ this_tree_size = old_tree_size;
FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (child), j, grandchild)
vect_free_slp_tree (grandchild);
SLP_TREE_CHILDREN (child).truncate (0);
"scalars instead\n");
oprnd_info->def_stmts = vNULL;
SLP_TREE_DEF_TYPE (child) = vect_external_def;
- SLP_TREE_CHILDREN (*node).quick_push (child);
+ children.safe_push (child);
continue;
}
}
oprnd_info->def_stmts = vNULL;
- SLP_TREE_CHILDREN (*node).quick_push (child);
+ children.safe_push (child);
continue;
}
scalar version. */
&& !is_pattern_stmt_p (vinfo_for_stmt (stmt)))
{
- unsigned int j;
- slp_tree grandchild;
-
- /* Roll back. */
- *max_nunits = old_max_nunits;
- loads->truncate (old_nloads);
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (child), j, grandchild)
- vect_free_slp_tree (grandchild);
- SLP_TREE_CHILDREN (child).truncate (0);
-
dump_printf_loc (MSG_NOTE, vect_location,
"Building vector operands from scalars\n");
- oprnd_info->def_stmts = vNULL;
+ child = vect_create_new_slp_node (oprnd_info->def_stmts);
SLP_TREE_DEF_TYPE (child) = vect_external_def;
- SLP_TREE_CHILDREN (*node).quick_push (child);
+ children.safe_push (child);
+ oprnd_info->def_stmts = vNULL;
continue;
}
&& oprnds_info[1]->first_dt == vect_internal_def
&& is_gimple_assign (stmt)
&& commutative_tree_code (gimple_assign_rhs_code (stmt))
- && !SLP_TREE_TWO_OPERATORS (*node)
+ && ! two_operators
/* Do so only if the number of not successful permutes was nor more
than a cut-ff as re-trying the recursive match on
possibly each level of the tree would expose exponential
behavior. */
&& *npermutes < 4)
{
- unsigned int j;
- slp_tree grandchild;
-
- /* Roll back. */
- *max_nunits = old_max_nunits;
- loads->truncate (old_nloads);
- FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (child), j, grandchild)
- vect_free_slp_tree (grandchild);
- SLP_TREE_CHILDREN (child).truncate (0);
-
/* Swap mismatched definition stmts. */
dump_printf_loc (MSG_NOTE, vect_location,
"Re-trying with swapped operands of stmts ");
dump_printf (MSG_NOTE, "\n");
/* And try again with scratch 'matches' ... */
bool *tem = XALLOCAVEC (bool, group_size);
- if (vect_build_slp_tree (vinfo, &child,
- group_size, max_nunits, loads,
- tem, npermutes, &this_tree_size,
- max_tree_size))
+ if ((child = vect_build_slp_tree (vinfo, oprnd_info->def_stmts,
+ group_size, &this_max_nunits,
+ &this_loads, tem, npermutes,
+ &this_tree_size,
+ max_tree_size)) != NULL)
{
/* ... so if successful we can apply the operand swapping
to the GIMPLE IL. This is necessary because for example
we'll continue to process swapped operand two. */
for (j = 0; j < group_size; ++j)
{
- gimple *stmt = SLP_TREE_SCALAR_STMTS (*node)[j];
+ gimple *stmt = stmts[j];
gimple_set_plf (stmt, GF_PLF_1, false);
}
for (j = 0; j < group_size; ++j)
{
- gimple *stmt = SLP_TREE_SCALAR_STMTS (*node)[j];
+ gimple *stmt = stmts[j];
if (!matches[j])
{
/* Avoid swapping operands twice. */
if (flag_checking)
for (j = 0; j < group_size; ++j)
{
- gimple *stmt = SLP_TREE_SCALAR_STMTS (*node)[j];
+ gimple *stmt = stmts[j];
gcc_assert (gimple_plf (stmt, GF_PLF_1) == ! matches[j]);
}
if (!grandchild)
{
/* Roll back. */
- *max_nunits = old_max_nunits;
- loads->truncate (old_nloads);
+ this_loads.truncate (old_nloads);
+ this_tree_size = old_tree_size;
FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (child), j, grandchild)
vect_free_slp_tree (grandchild);
SLP_TREE_CHILDREN (child).truncate (0);
"scalars instead\n");
oprnd_info->def_stmts = vNULL;
SLP_TREE_DEF_TYPE (child) = vect_external_def;
- SLP_TREE_CHILDREN (*node).quick_push (child);
+ children.safe_push (child);
continue;
}
}
oprnd_info->def_stmts = vNULL;
- SLP_TREE_CHILDREN (*node).quick_push (child);
+ children.safe_push (child);
continue;
}
++*npermutes;
}
- oprnd_info->def_stmts = vNULL;
- vect_free_slp_tree (child);
+ gcc_assert (child == NULL);
+ FOR_EACH_VEC_ELT (children, j, child)
+ vect_free_slp_tree (child);
vect_free_oprnd_info (oprnds_info);
- return false;
+ return NULL;
}
+ vect_free_oprnd_info (oprnds_info);
+
if (tree_size)
*tree_size += this_tree_size;
+ *max_nunits = this_max_nunits;
+ loads->safe_splice (this_loads);
- vect_free_oprnd_info (oprnds_info);
- return true;
+ node = vect_create_new_slp_node (stmts);
+ SLP_TREE_TWO_OPERATORS (node) = two_operators;
+ SLP_TREE_CHILDREN (node).splice (children);
+ return node;
}
/* Dump a slp tree NODE using flags specified in DUMP_KIND. */
scalar_stmts.safe_push (next);
}
- node = vect_create_new_slp_node (scalar_stmts);
-
loads.create (group_size);
/* Build the tree for the SLP instance. */
bool *matches = XALLOCAVEC (bool, group_size);
unsigned npermutes = 0;
- if (vect_build_slp_tree (vinfo, &node, group_size,
- &max_nunits, &loads,
- matches, &npermutes, NULL, max_tree_size))
+ if ((node = vect_build_slp_tree (vinfo, scalar_stmts, group_size,
+ &max_nunits, &loads, matches, &npermutes,
+ NULL, max_tree_size)) != NULL)
{
/* Calculate the unrolling factor based on the smallest type. */
if (max_nunits > nunits)
/* Failed to SLP. */
/* Free the allocated memory. */
- vect_free_slp_tree (node);
+ scalar_stmts.release ();
loads.release ();
/* For basic block SLP, try to break the group up into multiples of the
projects / platform / upstream / linaro-gcc.git / commitdiff