+2015-11-12 Richard Biener <rguenther@suse.de>
+
+ * tree-vectorizer.h (vect_slp_analyze_data_ref_dependences):
+ Rename to vect_slp_analyze_instance_dependence.
+ * tree-vect-data-refs.c (vect_slp_analyze_data_ref_dependence):
+ Remove WAR special-case.
+ (vect_slp_analyze_node_dependences): Instead add more specific
+ code here, not relying on other instances being vectorized.
+ (vect_slp_analyze_instance_dependence): Adjust accordingly.
+ * tree-vect-slp.c (vect_build_slp_tree_1): Remove excessive
+ vertical space in dump files.
+ (vect_print_slp_tree): Likewise.
+ (vect_analyze_slp_instance): Dump a header for the final SLP tree.
+ (vect_slp_analyze_bb_1): Delay computing relevant stmts and
+ not vectorized stmts until after dependence analysis removed
+ instances. Merge alignment and dependence checks.
+ * tree-vectorizer.c (pass_slp_vectorize::execute): Clear visited
+ flag on all stmts.
+
2015-11-12 Evandro Menezes <e.menezes@samsung.com>
* config/aarch64/aarch64-protos.h (tune_params): Add new members
dump_printf (MSG_NOTE, "\n");
}
- /* We do not vectorize basic blocks with write-write dependencies. */
- if (DR_IS_WRITE (dra) && DR_IS_WRITE (drb))
- return true;
-
- /* If we have a read-write dependence check that the load is before the store.
- When we vectorize basic blocks, vector load can be only before
- corresponding scalar load, and vector store can be only after its
- corresponding scalar store. So the order of the acceses is preserved in
- case the load is before the store. */
- gimple *earlier_stmt = get_earlier_stmt (DR_STMT (dra), DR_STMT (drb));
- if (DR_IS_READ (STMT_VINFO_DATA_REF (vinfo_for_stmt (earlier_stmt))))
- {
- /* That only holds for load-store pairs taking part in vectorization. */
- if (STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (DR_STMT (dra)))
- && STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (DR_STMT (drb))))
- return false;
- }
-
return true;
}
-/* Analyze dependences involved in the transform of SLP NODE. */
+/* Analyze dependences involved in the transform of SLP NODE. STORES
+ contain the vector of scalar stores of this instance if we are
+ disambiguating the loads. */
static bool
-vect_slp_analyze_node_dependences (slp_instance instance, slp_tree node)
+vect_slp_analyze_node_dependences (slp_instance instance, slp_tree node,
+ vec<gimple *> stores, gimple *last_store)
{
/* This walks over all stmts involved in the SLP load/store done
in NODE verifying we can sink them up to the last stmt in the
/* If we couldn't record a (single) data reference for this
stmt we have to give up. */
+ /* ??? Here and below if dependence analysis fails we can resort
+ to the alias oracle which can handle more kinds of stmts. */
data_reference *dr_b = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt));
if (!dr_b)
return false;
+ /* If we run into a store of this same instance (we've just
+ marked those) then delay dependence checking until we run
+ into the last store because this is where it will have
+ been sunk to (and we verify if we can do that as well). */
+ if (gimple_visited_p (stmt))
+ {
+ if (stmt != last_store)
+ continue;
+ unsigned i;
+ gimple *store;
+ FOR_EACH_VEC_ELT (stores, i, store)
+ {
+ data_reference *store_dr
+ = STMT_VINFO_DATA_REF (vinfo_for_stmt (store));
+ ddr_p ddr = initialize_data_dependence_relation
+ (dr_a, store_dr, vNULL);
+ if (vect_slp_analyze_data_ref_dependence (ddr))
+ {
+ free_dependence_relation (ddr);
+ return false;
+ }
+ free_dependence_relation (ddr);
+ }
+ }
+
ddr_p ddr = initialize_data_dependence_relation (dr_a, dr_b, vNULL);
if (vect_slp_analyze_data_ref_dependence (ddr))
{
- /* ??? If the dependence analysis failed we can resort to the
- alias oracle which can handle more kinds of stmts. */
free_dependence_relation (ddr);
return false;
}
the maximum vectorization factor the data dependences allow. */
bool
-vect_slp_analyze_data_ref_dependences (bb_vec_info bb_vinfo)
+vect_slp_analyze_instance_dependence (slp_instance instance)
{
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
- "=== vect_slp_analyze_data_ref_dependences ===\n");
+ "=== vect_slp_analyze_instance_dependence ===\n");
- slp_instance instance;
- slp_tree load;
- unsigned int i, j;
- for (i = 0; BB_VINFO_SLP_INSTANCES (bb_vinfo).iterate (i, &instance); )
+ /* The stores of this instance are at the root of the SLP tree. */
+ slp_tree store = SLP_INSTANCE_TREE (instance);
+ if (! STMT_VINFO_DATA_REF (vinfo_for_stmt (SLP_TREE_SCALAR_STMTS (store)[0])))
+ store = NULL;
+
+ /* Verify we can sink stores to the vectorized stmt insert location. */
+ gimple *last_store = NULL;
+ if (store)
{
- bool remove = false;
- /* Verify we can sink loads to the vectorized stmt insert location. */
- FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), j, load)
- if (! vect_slp_analyze_node_dependences (instance, load))
- {
- remove = true;
- break;
- }
- /* Verify we can sink stores to the vectorized stmt insert location. */
- slp_tree store = SLP_INSTANCE_TREE (instance);
- if (!remove
- && STMT_VINFO_DATA_REF
- (vinfo_for_stmt (SLP_TREE_SCALAR_STMTS (store)[0]))
- && ! vect_slp_analyze_node_dependences (instance, store))
- remove = true;
- if (remove)
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "removing SLP instance operations starting from: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM,
- SLP_TREE_SCALAR_STMTS
- (SLP_INSTANCE_TREE (instance))[0], 0);
- vect_free_slp_instance (instance);
- BB_VINFO_SLP_INSTANCES (bb_vinfo).ordered_remove (i);
- continue;
- }
- i++;
+ if (! vect_slp_analyze_node_dependences (instance, store, vNULL, NULL))
+ return false;
+
+ /* Mark stores in this instance and remember the last one. */
+ last_store = vect_find_last_scalar_stmt_in_slp (store);
+ for (unsigned k = 0; k < SLP_INSTANCE_GROUP_SIZE (instance); ++k)
+ gimple_set_visited (SLP_TREE_SCALAR_STMTS (store)[k], true);
}
- if (!BB_VINFO_SLP_INSTANCES (bb_vinfo).length ())
- return false;
+ bool res = true;
- return true;
-}
+ /* Verify we can sink loads to the vectorized stmt insert location,
+ special-casing stores of this instance. */
+ slp_tree load;
+ unsigned int i;
+ FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), i, load)
+ if (! vect_slp_analyze_node_dependences (instance, load,
+ store
+ ? SLP_TREE_SCALAR_STMTS (store)
+ : vNULL, last_store))
+ {
+ res = false;
+ break;
+ }
+ /* Unset the visited flag. */
+ if (store)
+ for (unsigned k = 0; k < SLP_INSTANCE_GROUP_SIZE (instance); ++k)
+ gimple_set_visited (SLP_TREE_SCALAR_STMTS (store)[k], false);
+
+ return res;
+}
/* Function vect_compute_data_ref_alignment
{
dump_printf_loc (MSG_NOTE, vect_location, "Build SLP for ");
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
- dump_printf (MSG_NOTE, "\n");
}
/* Fail to vectorize statements marked as unvectorizable. */
/* Dump a slp tree NODE using flags specified in DUMP_KIND. */
static void
-vect_print_slp_tree (int dump_kind, slp_tree node)
+vect_print_slp_tree (int dump_kind, location_t loc, slp_tree node)
{
int i;
gimple *stmt;
if (!node)
return;
- dump_printf (dump_kind, "node ");
+ dump_printf_loc (dump_kind, loc, "node\n");
FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt)
{
- dump_printf (dump_kind, "\n\tstmt %d ", i);
+ dump_printf_loc (dump_kind, loc, "\tstmt %d ", i);
dump_gimple_stmt (dump_kind, TDF_SLIM, stmt, 0);
}
- dump_printf (dump_kind, "\n");
-
FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
- vect_print_slp_tree (dump_kind, child);
+ vect_print_slp_tree (dump_kind, loc, child);
}
vinfo->slp_instances.safe_push (new_instance);
if (dump_enabled_p ())
- vect_print_slp_tree (MSG_NOTE, node);
+ {
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "Final SLP tree for instance:\n");
+ vect_print_slp_tree (MSG_NOTE, vect_location, node);
+ }
return true;
}
bool &fatal)
{
bb_vec_info bb_vinfo;
- vec<slp_instance> slp_instances;
slp_instance instance;
int i;
int min_vf = 2;
return NULL;
}
- /* Analyze and verify the alignment of data references in the SLP
- instances. */
+ /* Analyze and verify the alignment of data references and the
+ dependence in the SLP instances. */
for (i = 0; BB_VINFO_SLP_INSTANCES (bb_vinfo).iterate (i, &instance); )
{
- if (! vect_slp_analyze_and_verify_instance_alignment (instance))
+ if (! vect_slp_analyze_and_verify_instance_alignment (instance)
+ || ! vect_slp_analyze_instance_dependence (instance))
{
dump_printf_loc (MSG_NOTE, vect_location,
"removing SLP instance operations starting from: ");
BB_VINFO_SLP_INSTANCES (bb_vinfo).ordered_remove (i);
continue;
}
+
+ /* Mark all the statements that we want to vectorize as pure SLP and
+ relevant. */
+ vect_mark_slp_stmts (SLP_INSTANCE_TREE (instance), pure_slp, -1);
+ vect_mark_slp_stmts_relevant (SLP_INSTANCE_TREE (instance));
+
i++;
}
-
if (! BB_VINFO_SLP_INSTANCES (bb_vinfo).length ())
{
destroy_bb_vec_info (bb_vinfo);
return NULL;
}
- slp_instances = BB_VINFO_SLP_INSTANCES (bb_vinfo);
-
- /* Mark all the statements that we want to vectorize as pure SLP and
- relevant. */
- FOR_EACH_VEC_ELT (slp_instances, i, instance)
- {
- vect_mark_slp_stmts (SLP_INSTANCE_TREE (instance), pure_slp, -1);
- vect_mark_slp_stmts_relevant (SLP_INSTANCE_TREE (instance));
- }
-
/* Mark all the statements that we do not want to vectorize. */
for (gimple_stmt_iterator gsi = bb_vinfo->region_begin;
gsi_stmt (gsi) != gsi_stmt (bb_vinfo->region_end); gsi_next (&gsi))
STMT_VINFO_VECTORIZABLE (vinfo) = false;
}
- /* Analyze dependences. At this point all stmts not participating in
- vectorization have to be marked. Dependence analysis assumes
- that we either vectorize all SLP instances or none at all. */
- if (! vect_slp_analyze_data_ref_dependences (bb_vinfo))
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized: unhandled data dependence "
- "in basic block.\n");
-
- destroy_bb_vec_info (bb_vinfo);
- return NULL;
- }
-
if (!vect_slp_analyze_operations (BB_VINFO_SLP_INSTANCES (bb_vinfo),
BB_VINFO_TARGET_COST_DATA (bb_vinfo)))
{
scev_initialize ();
}
- /* Mark all stmts as not belonging to the current region. */
+ /* Mark all stmts as not belonging to the current region and unvisited. */
FOR_EACH_BB_FN (bb, fun)
{
for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
gsi_next (&gsi))
- gimple_set_uid (gsi_stmt (gsi), -1);
+ {
+ gimple *stmt = gsi_stmt (gsi);
+ gimple_set_uid (stmt, -1);
+ gimple_set_visited (stmt, false);
+ }
}
init_stmt_vec_info_vec ();
extern tree vect_get_smallest_scalar_type (gimple *, HOST_WIDE_INT *,
HOST_WIDE_INT *);
extern bool vect_analyze_data_ref_dependences (loop_vec_info, int *);
-extern bool vect_slp_analyze_data_ref_dependences (bb_vec_info);
+extern bool vect_slp_analyze_instance_dependence (slp_instance);
extern bool vect_enhance_data_refs_alignment (loop_vec_info);
extern bool vect_analyze_data_refs_alignment (loop_vec_info);
extern bool vect_verify_datarefs_alignment (loop_vec_info);
projects / platform / upstream / linaro-gcc.git / commitdiff