From a673744dee7a80e6525fb0da87d053c8ccab6275 Mon Sep 17 00:00:00 2001 From: Richard Biener Date: Tue, 13 Oct 2020 11:42:27 +0200 Subject: [PATCH] Remove STMT_VINFO_SAME_ALIGN_REFS This makes the only consumer of STMT_VINFO_SAME_ALIGN_REFS, the loop peeling for alignment code, use locally computed data and then removes STMT_VINFO_SAME_ALIGN_REFS and its computation. It also adjusts the auto_vec<> move CTOR/assignment so you can write auto_vec<..> foo = bar.copy (); and have foo own the generated copy. 2020-10-13 Richard Biener PR tree-optimization/97382 * tree-vectorizer.h (_stmt_vec_info::same_align_refs): Remove. (STMT_VINFO_SAME_ALIGN_REFS): Likewise. * tree-vectorizer.c (vec_info::new_stmt_vec_info): Do not allocate STMT_VINFO_SAME_ALIGN_REFS. (vec_info::free_stmt_vec_info): Do not release STMT_VINFO_SAME_ALIGN_REFS. * tree-vect-data-refs.c (vect_analyze_data_ref_dependences): Do not compute self and read-read dependences. (vect_dr_aligned_if_related_peeled_dr_is): New helper. (vect_dr_aligned_if_peeled_dr_is): Likewise. (vect_update_misalignment_for_peel): Use it instead of iterating over STMT_VINFO_SAME_ALIGN_REFS. (dr_align_group_sort_cmp): New function. (vect_enhance_data_refs_alignment): Count the number of same aligned refs here and elide uses of STMT_VINFO_SAME_ALIGN_REFS. (vect_find_same_alignment_drs): Remove. (vect_analyze_data_refs_alignment): Do not call it. * vec.h (auto_vec::auto_vec): Adjust CTOR to take a vec<>&&, assert it isn't using auto storage. (auto_vec& operator=): Apply a similar change. * gcc.dg/vect/no-vfa-vect-dv-2.c: Remove same align dump scanning. * gcc.dg/vect/vect-103.c: Likewise. * gcc.dg/vect/vect-91.c: Likewise. * gfortran.dg/vect/vect-4.f90: Likewise. --- gcc/testsuite/gcc.dg/vect/no-vfa-vect-dv-2.c | 2 - gcc/testsuite/gcc.dg/vect/vect-103.c | 2 - gcc/testsuite/gcc.dg/vect/vect-91.c | 2 - gcc/testsuite/gfortran.dg/vect/vect-4.f90 | 1 - gcc/tree-vect-data-refs.c | 233 ++++++++++++++++----------- gcc/tree-vectorizer.c | 2 - gcc/tree-vectorizer.h | 5 - gcc/vec.h | 6 +- 8 files changed, 143 insertions(+), 110 deletions(-) diff --git a/gcc/testsuite/gcc.dg/vect/no-vfa-vect-dv-2.c b/gcc/testsuite/gcc.dg/vect/no-vfa-vect-dv-2.c index dcb5370..8cc69ab 100644 --- a/gcc/testsuite/gcc.dg/vect/no-vfa-vect-dv-2.c +++ b/gcc/testsuite/gcc.dg/vect/no-vfa-vect-dv-2.c @@ -75,5 +75,3 @@ int main () /* The initialization induction loop (with aligned access) is also vectorized. */ /* { dg-final { scan-tree-dump-times "vectorized 3 loops" 1 "vect" } } */ -/* { dg-final { scan-tree-dump-times "accesses have the same alignment." 2 "vect" { target { { vect_aligned_arrays } && {! vect_sizes_32B_16B} } } } } */ -/* { dg-final { scan-tree-dump-times "accesses have the same alignment." 1 "vect" { target { {! vect_aligned_arrays } && {vect_sizes_32B_16B} } } } } */ diff --git a/gcc/testsuite/gcc.dg/vect/vect-103.c b/gcc/testsuite/gcc.dg/vect/vect-103.c index 2a45104..d03562f 100644 --- a/gcc/testsuite/gcc.dg/vect/vect-103.c +++ b/gcc/testsuite/gcc.dg/vect/vect-103.c @@ -58,5 +58,3 @@ int main (void) } /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */ -/* { dg-final { scan-tree-dump-times "accesses have the same alignment" 1 "vect" } } */ - diff --git a/gcc/testsuite/gcc.dg/vect/vect-91.c b/gcc/testsuite/gcc.dg/vect/vect-91.c index 91264d9..8983c7da 100644 --- a/gcc/testsuite/gcc.dg/vect/vect-91.c +++ b/gcc/testsuite/gcc.dg/vect/vect-91.c @@ -68,6 +68,4 @@ main3 () } /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 3 "vect" { xfail vect_no_int_add } } } */ -/* { dg-final { scan-tree-dump-times "accesses have the same alignment." 3 "vect" { target { { vect_aligned_arrays } && {! vect_sizes_32B_16B} } } } } */ -/* { dg-final { scan-tree-dump-times "accesses have the same alignment." 2 "vect" { target { {! vect_aligned_arrays } && {vect_sizes_32B_16B} } } } } */ /* { dg-final { scan-tree-dump-times "Alignment of access forced using versioning" 3 "vect" {target { {! vector_alignment_reachable} && {! vect_hw_misalign} } } } } */ diff --git a/gcc/testsuite/gfortran.dg/vect/vect-4.f90 b/gcc/testsuite/gfortran.dg/vect/vect-4.f90 index c2eeafd..9c067c6 100644 --- a/gcc/testsuite/gfortran.dg/vect/vect-4.f90 +++ b/gcc/testsuite/gfortran.dg/vect/vect-4.f90 @@ -13,4 +13,3 @@ Y = Y + A * X END ! { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } -! { dg-final { scan-tree-dump-times "accesses have the same alignment." 1 "vect" } } diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index 3ff3088..4abd27e 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -589,12 +589,11 @@ vect_analyze_data_ref_dependences (loop_vec_info loop_vinfo, LOOP_VINFO_DDRS (loop_vinfo) .create (LOOP_VINFO_DATAREFS (loop_vinfo).length () * LOOP_VINFO_DATAREFS (loop_vinfo).length ()); - /* We need read-read dependences to compute - STMT_VINFO_SAME_ALIGN_REFS. */ + /* We do not need read-read dependences. */ bool res = compute_all_dependences (LOOP_VINFO_DATAREFS (loop_vinfo), &LOOP_VINFO_DDRS (loop_vinfo), LOOP_VINFO_LOOP_NEST (loop_vinfo), - true); + false); gcc_assert (res); } @@ -1130,6 +1129,45 @@ vect_compute_data_ref_alignment (vec_info *vinfo, dr_vec_info *dr_info) return; } +/* Return whether DR_INFO, which is related to DR_PEEL_INFO in + that it only differs in DR_INIT, is aligned if DR_PEEL_INFO + is made aligned via peeling. */ + +static bool +vect_dr_aligned_if_related_peeled_dr_is (dr_vec_info *dr_info, + dr_vec_info *dr_peel_info) +{ + if (multiple_p (DR_TARGET_ALIGNMENT (dr_peel_info), + DR_TARGET_ALIGNMENT (dr_info))) + { + poly_offset_int diff + = (wi::to_poly_offset (DR_INIT (dr_peel_info->dr)) + - wi::to_poly_offset (DR_INIT (dr_info->dr))); + if (known_eq (diff, 0) + || multiple_p (diff, DR_TARGET_ALIGNMENT (dr_info))) + return true; + } + return false; +} + +/* Return whether DR_INFO is aligned if DR_PEEL_INFO is made + aligned via peeling. */ + +static bool +vect_dr_aligned_if_peeled_dr_is (dr_vec_info *dr_info, + dr_vec_info *dr_peel_info) +{ + if (!operand_equal_p (DR_BASE_ADDRESS (dr_info->dr), + DR_BASE_ADDRESS (dr_peel_info->dr), 0) + || !operand_equal_p (DR_OFFSET (dr_info->dr), + DR_OFFSET (dr_peel_info->dr), 0) + || !operand_equal_p (DR_STEP (dr_info->dr), + DR_STEP (dr_peel_info->dr), 0)) + return false; + + return vect_dr_aligned_if_related_peeled_dr_is (dr_info, dr_peel_info); +} + /* Function vect_update_misalignment_for_peel. Sets DR_INFO's misalignment - to 0 if it has the same alignment as DR_PEEL_INFO, @@ -1146,18 +1184,10 @@ static void vect_update_misalignment_for_peel (dr_vec_info *dr_info, dr_vec_info *dr_peel_info, int npeel) { - unsigned int i; - vec same_aligned_drs; - struct data_reference *current_dr; - stmt_vec_info peel_stmt_info = dr_peel_info->stmt; - /* It can be assumed that if dr_info has the same alignment as dr_peel, it is aligned in the vector loop. */ - same_aligned_drs = STMT_VINFO_SAME_ALIGN_REFS (peel_stmt_info); - FOR_EACH_VEC_ELT (same_aligned_drs, i, current_dr) + if (vect_dr_aligned_if_peeled_dr_is (dr_info, dr_peel_info)) { - if (current_dr != dr_info->dr) - continue; gcc_assert (!known_alignment_for_access_p (dr_info) || !known_alignment_for_access_p (dr_peel_info) || (DR_MISALIGNMENT (dr_info) @@ -1572,6 +1602,43 @@ vect_peeling_supportable (loop_vec_info loop_vinfo, dr_vec_info *dr0_info, return true; } +/* Compare two data-references DRA and DRB to group them into chunks + with related alignment. */ + +static int +dr_align_group_sort_cmp (const void *dra_, const void *drb_) +{ + data_reference_p dra = *(data_reference_p *)const_cast(dra_); + data_reference_p drb = *(data_reference_p *)const_cast(drb_); + int cmp; + + /* Stabilize sort. */ + if (dra == drb) + return 0; + + /* Ordering of DRs according to base. */ + cmp = data_ref_compare_tree (DR_BASE_ADDRESS (dra), + DR_BASE_ADDRESS (drb)); + if (cmp != 0) + return cmp; + + /* And according to DR_OFFSET. */ + cmp = data_ref_compare_tree (DR_OFFSET (dra), DR_OFFSET (drb)); + if (cmp != 0) + return cmp; + + /* And after step. */ + cmp = data_ref_compare_tree (DR_STEP (dra), DR_STEP (drb)); + if (cmp != 0) + return cmp; + + /* Then sort after DR_INIT. In case of identical DRs sort after stmt UID. */ + cmp = data_ref_compare_tree (DR_INIT (dra), DR_INIT (drb)); + if (cmp == 0) + return gimple_uid (DR_STMT (dra)) < gimple_uid (DR_STMT (drb)) ? -1 : 1; + return cmp; +} + /* Function vect_enhance_data_refs_alignment This pass will use loop versioning and loop peeling in order to enhance @@ -1666,7 +1733,6 @@ vect_peeling_supportable (loop_vec_info loop_vinfo, dr_vec_info *dr0_info, opt_result vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) { - vec datarefs = LOOP_VINFO_DATAREFS (loop_vinfo); class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); enum dr_alignment_support supportable_dr_alignment; dr_vec_info *first_store = NULL; @@ -1680,7 +1746,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) bool one_misalignment_unknown = false; bool one_dr_unsupportable = false; dr_vec_info *unsupportable_dr_info = NULL; - unsigned int mis, same_align_drs_max = 0; + unsigned int mis, dr0_same_align_drs = 0, first_store_same_align_drs = 0; hash_table peeling_htab (1); DUMP_VECT_SCOPE ("vect_enhance_data_refs_alignment"); @@ -1689,6 +1755,54 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).truncate (0); LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) = 0; + if (LOOP_VINFO_DATAREFS (loop_vinfo).is_empty ()) + return opt_result::success (); + + /* Sort the vector of datarefs so DRs that have the same or dependent + alignment are next to each other. */ + auto_vec datarefs + = LOOP_VINFO_DATAREFS (loop_vinfo).copy (); + datarefs.qsort (dr_align_group_sort_cmp); + + /* Compute the number of DRs that become aligned when we peel + a dataref so it becomes aligned. */ + auto_vec n_same_align_refs (datarefs.length ()); + n_same_align_refs.quick_grow_cleared (datarefs.length ()); + unsigned i0; + for (i0 = 0; i0 < datarefs.length (); ++i0) + if (DR_BASE_ADDRESS (datarefs[i0])) + break; + for (i = i0 + 1; i <= datarefs.length (); ++i) + { + if (i == datarefs.length () + || !operand_equal_p (DR_BASE_ADDRESS (datarefs[i0]), + DR_BASE_ADDRESS (datarefs[i]), 0) + || !operand_equal_p (DR_OFFSET (datarefs[i0]), + DR_OFFSET (datarefs[i]), 0) + || !operand_equal_p (DR_STEP (datarefs[i0]), + DR_STEP (datarefs[i]), 0)) + { + /* The subgroup [i0, i-1] now only differs in DR_INIT and + possibly DR_TARGET_ALIGNMENT. Still the whole subgroup + will get known misalignment if we align one of the refs + with the largest DR_TARGET_ALIGNMENT. */ + for (unsigned j = i0; j < i; ++j) + { + dr_vec_info *dr_infoj = loop_vinfo->lookup_dr (datarefs[j]); + for (unsigned k = i0; k < i; ++k) + { + if (k == j) + continue; + dr_vec_info *dr_infok = loop_vinfo->lookup_dr (datarefs[k]); + if (vect_dr_aligned_if_related_peeled_dr_is (dr_infok, + dr_infoj)) + n_same_align_refs[j]++; + } + } + i0 = i; + } + } + /* While cost model enhancements are expected in the future, the high level view of the code at this time is as follows: @@ -1790,18 +1904,17 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) peeling for data-ref that has the maximum number of data-refs with the same alignment, unless the target prefers to align stores over load. */ - unsigned same_align_drs - = STMT_VINFO_SAME_ALIGN_REFS (stmt_info).length (); + unsigned same_align_drs = n_same_align_refs[i]; if (!dr0_info - || same_align_drs_max < same_align_drs) + || dr0_same_align_drs < same_align_drs) { - same_align_drs_max = same_align_drs; + dr0_same_align_drs = same_align_drs; dr0_info = dr_info; } /* For data-refs with the same number of related accesses prefer the one where the misalign computation will be invariant in the outermost loop. */ - else if (same_align_drs_max == same_align_drs) + else if (dr0_same_align_drs == same_align_drs) { class loop *ivloop0, *ivloop; ivloop0 = outermost_invariant_loop_for_expr @@ -1825,7 +1938,10 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) } if (!first_store && DR_IS_WRITE (dr)) - first_store = dr_info; + { + first_store = dr_info; + first_store_same_align_drs = same_align_drs; + } } } else @@ -1895,6 +2011,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) && load_outside_cost > store_outside_cost)) { dr0_info = first_store; + dr0_same_align_drs = first_store_same_align_drs; peel_for_unknown_alignment.inside_cost = store_inside_cost; peel_for_unknown_alignment.outside_cost = store_outside_cost; } @@ -1917,8 +2034,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) prologue_cost_vec.release (); epilogue_cost_vec.release (); - peel_for_unknown_alignment.peel_info.count = 1 - + STMT_VINFO_SAME_ALIGN_REFS (dr0_info->stmt).length (); + peel_for_unknown_alignment.peel_info.count = dr0_same_align_drs + 1; } peel_for_unknown_alignment.peel_info.npeel = 0; @@ -2270,69 +2386,6 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) } -/* Function vect_find_same_alignment_drs. - - Update group and alignment relations in VINFO according to the chosen - vectorization factor. */ - -static void -vect_find_same_alignment_drs (vec_info *vinfo, data_dependence_relation *ddr) -{ - struct data_reference *dra = DDR_A (ddr); - struct data_reference *drb = DDR_B (ddr); - dr_vec_info *dr_info_a = vinfo->lookup_dr (dra); - dr_vec_info *dr_info_b = vinfo->lookup_dr (drb); - stmt_vec_info stmtinfo_a = dr_info_a->stmt; - stmt_vec_info stmtinfo_b = dr_info_b->stmt; - - if (DDR_ARE_DEPENDENT (ddr) == chrec_known) - return; - - if (dra == drb) - return; - - if (STMT_VINFO_GATHER_SCATTER_P (stmtinfo_a) - || STMT_VINFO_GATHER_SCATTER_P (stmtinfo_b)) - return; - - if (!operand_equal_p (DR_BASE_ADDRESS (dra), DR_BASE_ADDRESS (drb), 0) - || !operand_equal_p (DR_OFFSET (dra), DR_OFFSET (drb), 0) - || !operand_equal_p (DR_STEP (dra), DR_STEP (drb), 0)) - return; - - /* Two references with distance zero have the same alignment. */ - poly_offset_int diff = (wi::to_poly_offset (DR_INIT (dra)) - - wi::to_poly_offset (DR_INIT (drb))); - if (maybe_ne (diff, 0)) - { - /* Get the wider of the two alignments. */ - poly_uint64 align_a = - exact_div (vect_calculate_target_alignment (dr_info_a), - BITS_PER_UNIT); - poly_uint64 align_b = - exact_div (vect_calculate_target_alignment (dr_info_b), - BITS_PER_UNIT); - unsigned HOST_WIDE_INT align_a_c, align_b_c; - if (!align_a.is_constant (&align_a_c) - || !align_b.is_constant (&align_b_c)) - return; - - unsigned HOST_WIDE_INT max_align = MAX (align_a_c, align_b_c); - - /* Require the gap to be a multiple of the larger vector alignment. */ - if (!multiple_p (diff, max_align)) - return; - } - - STMT_VINFO_SAME_ALIGN_REFS (stmtinfo_a).safe_push (drb); - STMT_VINFO_SAME_ALIGN_REFS (stmtinfo_b).safe_push (dra); - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "accesses have the same alignment: %T and %T\n", - DR_REF (dra), DR_REF (drb)); -} - - /* Function vect_analyze_data_refs_alignment Analyze the alignment of the data-references in the loop. @@ -2343,17 +2396,9 @@ vect_analyze_data_refs_alignment (loop_vec_info loop_vinfo) { DUMP_VECT_SCOPE ("vect_analyze_data_refs_alignment"); - /* Mark groups of data references with same alignment using - data dependence information. */ - vec ddrs = LOOP_VINFO_DDRS (loop_vinfo); - struct data_dependence_relation *ddr; - unsigned int i; - - FOR_EACH_VEC_ELT (ddrs, i, ddr) - vect_find_same_alignment_drs (loop_vinfo, ddr); - vec datarefs = LOOP_VINFO_DATAREFS (loop_vinfo); struct data_reference *dr; + unsigned int i; vect_record_base_alignments (loop_vinfo); FOR_EACH_VEC_ELT (datarefs, i, dr) diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c index 02da755..778177a 100644 --- a/gcc/tree-vectorizer.c +++ b/gcc/tree-vectorizer.c @@ -690,7 +690,6 @@ vec_info::new_stmt_vec_info (gimple *stmt) else STMT_VINFO_DEF_TYPE (res) = vect_internal_def; - STMT_VINFO_SAME_ALIGN_REFS (res).create (0); STMT_SLP_TYPE (res) = loop_vect; /* This is really "uninitialized" until vect_compute_data_ref_alignment. */ @@ -746,7 +745,6 @@ vec_info::free_stmt_vec_info (stmt_vec_info stmt_info) release_ssa_name (lhs); } - STMT_VINFO_SAME_ALIGN_REFS (stmt_info).release (); STMT_VINFO_SIMD_CLONE_INFO (stmt_info).release (); STMT_VINFO_VEC_STMTS (stmt_info).release (); free (stmt_info); diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 2a8c4a5..b56073c 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -1058,10 +1058,6 @@ public: pattern statement. */ gimple_seq pattern_def_seq; - /* List of datarefs that are known to have the same alignment as the dataref - of this stmt. */ - vec same_align_refs; - /* Selected SIMD clone's function info. First vector element is SIMD clone's function decl, followed by a pair of trees (base + step) for linear arguments (pair of NULLs for other arguments). */ @@ -1251,7 +1247,6 @@ struct gather_scatter_info { #define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p #define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt #define STMT_VINFO_PATTERN_DEF_SEQ(S) (S)->pattern_def_seq -#define STMT_VINFO_SAME_ALIGN_REFS(S) (S)->same_align_refs #define STMT_VINFO_SIMD_CLONE_INFO(S) (S)->simd_clone_info #define STMT_VINFO_DEF_TYPE(S) (S)->def_type #define STMT_VINFO_GROUPED_ACCESS(S) \ diff --git a/gcc/vec.h b/gcc/vec.h index d8c7cda..3ad2697 100644 --- a/gcc/vec.h +++ b/gcc/vec.h @@ -1541,13 +1541,15 @@ public: auto_vec (size_t n) { this->create (n); } ~auto_vec () { this->release (); } - auto_vec (auto_vec&& r) + auto_vec (vec&& r) { + gcc_assert (!r.using_auto_storage ()); this->m_vec = r.m_vec; r.m_vec = NULL; } - auto_vec& operator= (auto_vec&& r) + auto_vec& operator= (vec&& r) { + gcc_assert (!r.using_auto_storage ()); this->release (); this->m_vec = r.m_vec; r.m_vec = NULL; -- 2.7.4