/* Vectorizer
- Copyright (C) 2003-2014 Free Software Foundation, Inc.
+ Copyright (C) 2003-2015 Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com>
This file is part of GCC.
#include "system.h"
#include "coretypes.h"
#include "dumpfile.h"
-#include "tm.h"
+#include "backend.h"
+#include "predict.h"
#include "tree.h"
+#include "gimple.h"
+#include "hard-reg-set.h"
+#include "ssa.h"
+#include "alias.h"
+#include "fold-const.h"
#include "stor-layout.h"
#include "tree-pretty-print.h"
-#include "basic-block.h"
-#include "tree-ssa-alias.h"
#include "internal-fn.h"
-#include "gimple-expr.h"
-#include "is-a.h"
-#include "gimple.h"
#include "gimple-iterator.h"
#include "gimple-walk.h"
-#include "gimple-ssa.h"
#include "cgraph.h"
-#include "tree-phinodes.h"
-#include "ssa-iterators.h"
#include "tree-ssa-loop-manip.h"
#include "tree-cfg.h"
#include "cfgloop.h"
source_location vect_location;
/* Vector mapping GIMPLE stmt to stmt_vec_info. */
-vec<vec_void_p> stmt_vec_info_vec;
+vec<stmt_vec_info> stmt_vec_info_vec;
\f
/* For mapping simduid to vectorization factor. */
-struct simduid_to_vf : typed_free_remove<simduid_to_vf>
+struct simduid_to_vf : free_ptr_hash<simduid_to_vf>
{
unsigned int simduid;
int vf;
/* hash_table support. */
- typedef simduid_to_vf value_type;
- typedef simduid_to_vf compare_type;
- static inline hashval_t hash (const value_type *);
- static inline int equal (const value_type *, const compare_type *);
+ static inline hashval_t hash (const simduid_to_vf *);
+ static inline int equal (const simduid_to_vf *, const simduid_to_vf *);
};
inline hashval_t
-simduid_to_vf::hash (const value_type *p)
+simduid_to_vf::hash (const simduid_to_vf *p)
{
return p->simduid;
}
inline int
-simduid_to_vf::equal (const value_type *p1, const value_type *p2)
+simduid_to_vf::equal (const simduid_to_vf *p1, const simduid_to_vf *p2)
{
return p1->simduid == p2->simduid;
}
This hash maps from the OMP simd array (D.1737[]) to DECL_UID of
simduid.0. */
-struct simd_array_to_simduid : typed_free_remove<simd_array_to_simduid>
+struct simd_array_to_simduid : free_ptr_hash<simd_array_to_simduid>
{
tree decl;
unsigned int simduid;
/* hash_table support. */
- typedef simd_array_to_simduid value_type;
- typedef simd_array_to_simduid compare_type;
- static inline hashval_t hash (const value_type *);
- static inline int equal (const value_type *, const compare_type *);
+ static inline hashval_t hash (const simd_array_to_simduid *);
+ static inline int equal (const simd_array_to_simduid *,
+ const simd_array_to_simduid *);
};
inline hashval_t
-simd_array_to_simduid::hash (const value_type *p)
+simd_array_to_simduid::hash (const simd_array_to_simduid *p)
{
return DECL_UID (p->decl);
}
inline int
-simd_array_to_simduid::equal (const value_type *p1, const value_type *p2)
+simd_array_to_simduid::equal (const simd_array_to_simduid *p1,
+ const simd_array_to_simduid *p2)
{
return p1->decl == p2->decl;
}
-/* Fold IFN_GOMP_SIMD_LANE, IFN_GOMP_SIMD_VF and IFN_GOMP_SIMD_LAST_LANE
- into their corresponding constants. */
+/* Fold IFN_GOMP_SIMD_LANE, IFN_GOMP_SIMD_VF, IFN_GOMP_SIMD_LAST_LANE,
+ into their corresponding constants and remove
+ IFN_GOMP_SIMD_ORDERED_{START,END}. */
static void
-adjust_simduid_builtins (hash_table<simduid_to_vf> **htab)
+adjust_simduid_builtins (hash_table<simduid_to_vf> *htab)
{
basic_block bb;
{
gimple_stmt_iterator i;
- for (i = gsi_start_bb (bb); !gsi_end_p (i); gsi_next (&i))
+ for (i = gsi_start_bb (bb); !gsi_end_p (i); )
{
unsigned int vf = 1;
enum internal_fn ifn;
- gimple stmt = gsi_stmt (i);
+ gimple *stmt = gsi_stmt (i);
tree t;
if (!is_gimple_call (stmt)
|| !gimple_call_internal_p (stmt))
- continue;
+ {
+ gsi_next (&i);
+ continue;
+ }
ifn = gimple_call_internal_fn (stmt);
switch (ifn)
{
case IFN_GOMP_SIMD_VF:
case IFN_GOMP_SIMD_LAST_LANE:
break;
+ case IFN_GOMP_SIMD_ORDERED_START:
+ case IFN_GOMP_SIMD_ORDERED_END:
+ gsi_remove (&i, true);
+ unlink_stmt_vdef (stmt);
+ continue;
default:
+ gsi_next (&i);
continue;
}
tree arg = gimple_call_arg (stmt, 0);
gcc_assert (TREE_CODE (arg) == SSA_NAME);
simduid_to_vf *p = NULL, data;
data.simduid = DECL_UID (SSA_NAME_VAR (arg));
- if (*htab)
- p = (*htab)->find (&data);
- if (p)
- vf = p->vf;
+ if (htab)
+ {
+ p = htab->find (&data);
+ if (p)
+ vf = p->vf;
+ }
switch (ifn)
{
case IFN_GOMP_SIMD_VF:
gcc_unreachable ();
}
update_call_from_tree (&i, t);
+ gsi_next (&i);
}
}
}
FOR_EACH_BB_FN (bb, cfun)
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
{
- gimple stmt = gsi_stmt (gsi);
+ gimple *stmt = gsi_stmt (gsi);
if (!is_gimple_call (stmt) || !gimple_call_internal_p (stmt))
continue;
switch (gimple_call_internal_fn (stmt))
if (lhs == NULL_TREE)
continue;
imm_use_iterator use_iter;
- gimple use_stmt;
+ gimple *use_stmt;
ns.simduid = DECL_UID (SSA_NAME_VAR (gimple_call_arg (stmt, 0)));
FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, lhs)
if (!is_gimple_debug (use_stmt))
walk_gimple_op (use_stmt, note_simd_array_uses_cb, &wi);
}
}
+
+/* Shrink arrays with "omp simd array" attribute to the corresponding
+ vectorization factor. */
+
+static void
+shrink_simd_arrays
+ (hash_table<simd_array_to_simduid> *simd_array_to_simduid_htab,
+ hash_table<simduid_to_vf> *simduid_to_vf_htab)
+{
+ for (hash_table<simd_array_to_simduid>::iterator iter
+ = simd_array_to_simduid_htab->begin ();
+ iter != simd_array_to_simduid_htab->end (); ++iter)
+ if ((*iter)->simduid != -1U)
+ {
+ tree decl = (*iter)->decl;
+ int vf = 1;
+ if (simduid_to_vf_htab)
+ {
+ simduid_to_vf *p = NULL, data;
+ data.simduid = (*iter)->simduid;
+ p = simduid_to_vf_htab->find (&data);
+ if (p)
+ vf = p->vf;
+ }
+ tree atype
+ = build_array_type_nelts (TREE_TYPE (TREE_TYPE (decl)), vf);
+ TREE_TYPE (decl) = atype;
+ relayout_decl (decl);
+ }
+
+ delete simd_array_to_simduid_htab;
+}
\f
/* A helper function to free data refs. */
void
-vect_destroy_datarefs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo)
+vect_destroy_datarefs (vec_info *vinfo)
{
- vec<data_reference_p> datarefs;
struct data_reference *dr;
unsigned int i;
- if (loop_vinfo)
- datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
- else
- datarefs = BB_VINFO_DATAREFS (bb_vinfo);
-
- FOR_EACH_VEC_ELT (datarefs, i, dr)
+ FOR_EACH_VEC_ELT (vinfo->datarefs, i, dr)
if (dr->aux)
{
free (dr->aux);
dr->aux = NULL;
}
- free_data_refs (datarefs);
+ free_data_refs (vinfo->datarefs);
}
/* If LOOP has been versioned during ifcvt, return the internal call
guarding it. */
-static gimple
+static gimple *
vect_loop_vectorized_call (struct loop *loop)
{
basic_block bb = loop_preheader_edge (loop)->src;
- gimple g;
+ gimple *g;
do
{
g = last_stmt (bb);
update any immediate uses of it's LHS. */
static void
-fold_loop_vectorized_call (gimple g, tree value)
+fold_loop_vectorized_call (gimple *g, tree value)
{
tree lhs = gimple_call_lhs (g);
use_operand_p use_p;
imm_use_iterator iter;
- gimple use_stmt;
+ gimple *use_stmt;
gimple_stmt_iterator gsi = gsi_for_stmt (g);
update_call_from_tree (&gsi, value);
update_stmt (use_stmt);
}
}
+/* Set the uids of all the statements in basic blocks inside loop
+ represented by LOOP_VINFO. LOOP_VECTORIZED_CALL is the internal
+ call guarding the loop which has been if converted. */
+static void
+set_uid_loop_bbs (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
+{
+ tree arg = gimple_call_arg (loop_vectorized_call, 1);
+ basic_block *bbs;
+ unsigned int i;
+ struct loop *scalar_loop = get_loop (cfun, tree_to_shwi (arg));
+
+ LOOP_VINFO_SCALAR_LOOP (loop_vinfo) = scalar_loop;
+ gcc_checking_assert (vect_loop_vectorized_call
+ (LOOP_VINFO_SCALAR_LOOP (loop_vinfo))
+ == loop_vectorized_call);
+ bbs = get_loop_body (scalar_loop);
+ for (i = 0; i < scalar_loop->num_nodes; i++)
+ {
+ basic_block bb = bbs[i];
+ gimple_stmt_iterator gsi;
+ for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
+ {
+ gimple *phi = gsi_stmt (gsi);
+ gimple_set_uid (phi, 0);
+ }
+ for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
+ {
+ gimple *stmt = gsi_stmt (gsi);
+ gimple_set_uid (stmt, 0);
+ }
+ }
+ free (bbs);
+}
/* Function vectorize_loops.
/* Bail out if there are no loops. */
if (vect_loops_num <= 1)
- {
- if (cfun->has_simduid_loops)
- adjust_simduid_builtins (&simduid_to_vf_htab);
- return 0;
- }
+ return 0;
if (cfun->has_simduid_loops)
note_simd_array_uses (&simd_array_to_simduid_htab);
if (!dbg_cnt (vect_loop))
break;
- gimple loop_vectorized_call = vect_loop_vectorized_call (loop);
+ gimple *loop_vectorized_call = vect_loop_vectorized_call (loop);
if (loop_vectorized_call)
- {
- tree arg = gimple_call_arg (loop_vectorized_call, 1);
- basic_block *bbs;
- unsigned int i;
- struct loop *scalar_loop = get_loop (cfun, tree_to_shwi (arg));
-
- LOOP_VINFO_SCALAR_LOOP (loop_vinfo) = scalar_loop;
- gcc_checking_assert (vect_loop_vectorized_call
- (LOOP_VINFO_SCALAR_LOOP (loop_vinfo))
- == loop_vectorized_call);
- bbs = get_loop_body (scalar_loop);
- for (i = 0; i < scalar_loop->num_nodes; i++)
- {
- basic_block bb = bbs[i];
- gimple_stmt_iterator gsi;
- for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi);
- gsi_next (&gsi))
- {
- gimple phi = gsi_stmt (gsi);
- gimple_set_uid (phi, 0);
- }
- for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
- gsi_next (&gsi))
- {
- gimple stmt = gsi_stmt (gsi);
- gimple_set_uid (stmt, 0);
- }
- }
- free (bbs);
- }
-
+ set_uid_loop_bbs (loop_vinfo, loop_vectorized_call);
if (LOCATION_LOCUS (vect_location) != UNKNOWN_LOCATION
&& dump_enabled_p ())
dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
loop = get_loop (cfun, i);
if (loop && loop->dont_vectorize)
{
- gimple g = vect_loop_vectorized_call (loop);
+ gimple *g = vect_loop_vectorized_call (loop);
if (g)
{
fold_loop_vectorized_call (g, boolean_false_node);
free_stmt_vec_info_vec ();
- /* Fold IFN_GOMP_SIMD_{VF,LANE,LAST_LANE} builtins. */
+ /* Fold IFN_GOMP_SIMD_{VF,LANE,LAST_LANE,ORDERED_{START,END}} builtins. */
if (cfun->has_simduid_loops)
- adjust_simduid_builtins (&simduid_to_vf_htab);
+ adjust_simduid_builtins (simduid_to_vf_htab);
/* Shrink any "omp array simd" temporary arrays to the
actual vectorization factors. */
if (simd_array_to_simduid_htab)
- {
- for (hash_table<simd_array_to_simduid>::iterator iter
- = simd_array_to_simduid_htab->begin ();
- iter != simd_array_to_simduid_htab->end (); ++iter)
- if ((*iter).simduid != -1U)
- {
- tree decl = (*iter).decl;
- int vf = 1;
- if (simduid_to_vf_htab)
- {
- simduid_to_vf *p = NULL, data;
- data.simduid = (*iter).simduid;
- p = simduid_to_vf_htab->find (&data);
- if (p)
- vf = p->vf;
- }
- tree atype
- = build_array_type_nelts (TREE_TYPE (TREE_TYPE (decl)), vf);
- TREE_TYPE (decl) = atype;
- relayout_decl (decl);
- }
-
- delete simd_array_to_simduid_htab;
- }
- delete simduid_to_vf_htab;
- simduid_to_vf_htab = NULL;
+ shrink_simd_arrays (simd_array_to_simduid_htab, simduid_to_vf_htab);
+ delete simduid_to_vf_htab;
+ cfun->has_simduid_loops = false;
if (num_vectorized_loops > 0)
{
}
+/* Entry point to the simduid cleanup pass. */
+
+namespace {
+
+const pass_data pass_data_simduid_cleanup =
+{
+ GIMPLE_PASS, /* type */
+ "simduid", /* name */
+ OPTGROUP_NONE, /* optinfo_flags */
+ TV_NONE, /* tv_id */
+ ( PROP_ssa | PROP_cfg ), /* properties_required */
+ 0, /* properties_provided */
+ 0, /* properties_destroyed */
+ 0, /* todo_flags_start */
+ 0, /* todo_flags_finish */
+};
+
+class pass_simduid_cleanup : public gimple_opt_pass
+{
+public:
+ pass_simduid_cleanup (gcc::context *ctxt)
+ : gimple_opt_pass (pass_data_simduid_cleanup, ctxt)
+ {}
+
+ /* opt_pass methods: */
+ opt_pass * clone () { return new pass_simduid_cleanup (m_ctxt); }
+ virtual bool gate (function *fun) { return fun->has_simduid_loops; }
+ virtual unsigned int execute (function *);
+
+}; // class pass_simduid_cleanup
+
+unsigned int
+pass_simduid_cleanup::execute (function *fun)
+{
+ hash_table<simd_array_to_simduid> *simd_array_to_simduid_htab = NULL;
+
+ note_simd_array_uses (&simd_array_to_simduid_htab);
+
+ /* Fold IFN_GOMP_SIMD_{VF,LANE,LAST_LANE,ORDERED_{START,END}} builtins. */
+ adjust_simduid_builtins (NULL);
+
+ /* Shrink any "omp array simd" temporary arrays to the
+ actual vectorization factors. */
+ if (simd_array_to_simduid_htab)
+ shrink_simd_arrays (simd_array_to_simduid_htab, NULL);
+ fun->has_simduid_loops = false;
+ return 0;
+}
+
+} // anon namespace
+
+gimple_opt_pass *
+make_pass_simduid_cleanup (gcc::context *ctxt)
+{
+ return new pass_simduid_cleanup (ctxt);
+}
+
+
/* Entry point to basic block SLP phase. */
namespace {
GIMPLE_PASS, /* type */
"slp", /* name */
OPTGROUP_LOOP | OPTGROUP_VEC, /* optinfo_flags */
- true, /* has_execute */
TV_TREE_SLP_VECTORIZATION, /* tv_id */
( PROP_ssa | PROP_cfg ), /* properties_required */
0, /* properties_provided */
if (vect_can_force_dr_alignment_p (decl, alignment))
{
- DECL_ALIGN (decl) = TYPE_ALIGN (vectype);
- DECL_USER_ALIGN (decl) = 1;
- if (TREE_STATIC (decl))
- {
- tree target = symtab_alias_ultimate_target (symtab_get_node (decl))->decl;
- DECL_ALIGN (target) = TYPE_ALIGN (vectype);
- DECL_USER_ALIGN (target) = 1;
- }
+ vnode->increase_alignment (TYPE_ALIGN (vectype));
dump_printf (MSG_NOTE, "Increasing alignment of decl: ");
dump_generic_expr (MSG_NOTE, TDF_SLIM, decl);
dump_printf (MSG_NOTE, "\n");
SIMPLE_IPA_PASS, /* type */
"increase_alignment", /* name */
OPTGROUP_LOOP | OPTGROUP_VEC, /* optinfo_flags */
- true, /* has_execute */
TV_IPA_OPT, /* tv_id */
0, /* properties_required */
0, /* properties_provided */
projects / platform / upstream / gcc.git / blobdiff