From 3bab634221b45ccdae547b8536cb2bc722e75ea0 Mon Sep 17 00:00:00 2001 From: Andrey Turetskiy Date: Fri, 4 Sep 2015 08:51:44 +0000 Subject: [PATCH] tm.texi.in (TARGET_VECTORIZE_BUILTIN_SCATTER): New. gcc/ * doc/tm.texi.in (TARGET_VECTORIZE_BUILTIN_SCATTER): New. * doc/tm.texi: Regenerate. * target.def: Add scatter builtin. * tree-vectorizer.h: Rename gather_p to gather_scatter_p and use it for loads/stores in case of gather/scatter accordingly. (STMT_VINFO_GATHER_SCATTER_P(S)): Use it instead of STMT_VINFO_GATHER_P(S). (vect_check_gather): Rename to ... (vect_check_gather_scatter): this. * tree-vect-data-refs.c (vect_analyze_data_ref_dependence): Use STMT_VINFO_GATHER_SCATTER_P instead of STMT_VINFO_SCATTER_P. (vect_check_gather_scatter): Use it instead of vect_check_gather. (vect_analyze_data_refs): Add gatherscatter enum and maybe_scatter variable and new checkings for it accordingly. * tree-vect-stmts.c (STMT_VINFO_GATHER_SCATTER_P(S)): Use it instead of STMT_VINFO_GATHER_P(S). (vect_check_gather_scatter): Use it instead of vect_check_gather. (vectorizable_store): Add checkings for STMT_VINFO_GATHER_SCATTER_P. Co-Authored-By: Kirill Yukhin Co-Authored-By: Petr Murzin From-SVN: r227481 --- gcc/ChangeLog | 22 ++++++ gcc/doc/tm.texi | 8 ++ gcc/doc/tm.texi.in | 2 + gcc/target.def | 12 +++ gcc/tree-vect-data-refs.c | 93 ++++++++++++----------- gcc/tree-vect-stmts.c | 185 +++++++++++++++++++++++++++++++++++++++++++--- gcc/tree-vectorizer.h | 10 +-- 7 files changed, 274 insertions(+), 58 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index b18fe9e..3b783cc 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,25 @@ +2015-09-04 Andrey Turetskiy + Petr Murzin + Kirill Yukhin + + * doc/tm.texi.in (TARGET_VECTORIZE_BUILTIN_SCATTER): New. + * doc/tm.texi: Regenerate. + * target.def: Add scatter builtin. + * tree-vectorizer.h: Rename gather_p to gather_scatter_p and use it + for loads/stores in case of gather/scatter accordingly. + (STMT_VINFO_GATHER_SCATTER_P(S)): Use it instead of STMT_VINFO_GATHER_P(S). + (vect_check_gather): Rename to ... + (vect_check_gather_scatter): this. + * tree-vect-data-refs.c (vect_analyze_data_ref_dependence): Use + STMT_VINFO_GATHER_SCATTER_P instead of STMT_VINFO_SCATTER_P. + (vect_check_gather_scatter): Use it instead of vect_check_gather. + (vect_analyze_data_refs): Add gatherscatter enum and maybe_scatter variable + and new checkings for it accordingly. + * tree-vect-stmts.c + (STMT_VINFO_GATHER_SCATTER_P(S)): Use it instead of STMT_VINFO_GATHER_P(S). + (vect_check_gather_scatter): Use it instead of vect_check_gather. + (vectorizable_store): Add checkings for STMT_VINFO_GATHER_SCATTER_P. + 2015-09-03 Bill Schmidt * config/rs6000/altivec.md (altivec_vperm_v8hiv16qi): New diff --git a/gcc/doc/tm.texi b/gcc/doc/tm.texi index f5a1f84..d548d96 100644 --- a/gcc/doc/tm.texi +++ b/gcc/doc/tm.texi @@ -5720,6 +5720,14 @@ The default is @code{NULL_TREE} which means to not vectorize gather loads. @end deftypefn +@deftypefn {Target Hook} tree TARGET_VECTORIZE_BUILTIN_SCATTER (const_tree @var{vectype}, const_tree @var{index_type}, int @var{scale}) +Target builtin that implements vector scatter operation. @var{vectype} +is the vector type of the store and @var{index_type} is scalar type of +the index, scaled by @var{scale}. +The default is @code{NULL_TREE} which means to not vectorize scatter +stores. +@end deftypefn + @deftypefn {Target Hook} int TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN (struct cgraph_node *@var{}, struct cgraph_simd_clone *@var{}, @var{tree}, @var{int}) This hook should set @var{vecsize_mangle}, @var{vecsize_int}, @var{vecsize_float} fields in @var{simd_clone} structure pointed by @var{clone_info} argument and also diff --git a/gcc/doc/tm.texi.in b/gcc/doc/tm.texi.in index 9d5ac0a..9bef4a5 100644 --- a/gcc/doc/tm.texi.in +++ b/gcc/doc/tm.texi.in @@ -4239,6 +4239,8 @@ address; but often a machine-dependent strategy can generate better code. @hook TARGET_VECTORIZE_BUILTIN_GATHER +@hook TARGET_VECTORIZE_BUILTIN_SCATTER + @hook TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN @hook TARGET_SIMD_CLONE_ADJUST diff --git a/gcc/target.def b/gcc/target.def index 4edc209..aa5a1f1 100644 --- a/gcc/target.def +++ b/gcc/target.def @@ -1801,6 +1801,18 @@ loads.", (const_tree mem_vectype, const_tree index_type, int scale), NULL) +/* Target builtin that implements vector scatter operation. */ +DEFHOOK +(builtin_scatter, +"Target builtin that implements vector scatter operation. @var{vectype}\n\ +is the vector type of the store and @var{index_type} is scalar type of\n\ +the index, scaled by @var{scale}.\n\ +The default is @code{NULL_TREE} which means to not vectorize scatter\n\ +stores.", + tree, + (const_tree vectype, const_tree index_type, int scale), + NULL) + /* Target function to initialize the cost model for a loop or block. */ DEFHOOK (init_cost, diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index f1eaef4..2439bd6 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -267,8 +267,8 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, return false; } - if (STMT_VINFO_GATHER_P (stmtinfo_a) - || STMT_VINFO_GATHER_P (stmtinfo_b)) + if (STMT_VINFO_GATHER_SCATTER_P (stmtinfo_a) + || STMT_VINFO_GATHER_SCATTER_P (stmtinfo_b)) { if (dump_enabled_p ()) { @@ -315,8 +315,8 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, return false; } - if (STMT_VINFO_GATHER_P (stmtinfo_a) - || STMT_VINFO_GATHER_P (stmtinfo_b)) + if (STMT_VINFO_GATHER_SCATTER_P (stmtinfo_a) + || STMT_VINFO_GATHER_SCATTER_P (stmtinfo_b)) { if (dump_enabled_p ()) { @@ -2344,10 +2344,7 @@ vect_analyze_data_ref_access (struct data_reference *dr) if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "zero step in outer loop.\n"); - if (DR_IS_READ (dr)) - return true; - else - return false; + return DR_IS_READ (dr); } } @@ -2997,12 +2994,12 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo) return true; } -/* Check whether a non-affine read in stmt is suitable for gather load - and if so, return a builtin decl for that operation. */ +/* Check whether a non-affine read or write in stmt is suitable for gather load + or scatter store and if so, return a builtin decl for that operation. */ tree -vect_check_gather (gimple stmt, loop_vec_info loop_vinfo, tree *basep, - tree *offp, int *scalep) +vect_check_gather_scatter (gimple stmt, loop_vec_info loop_vinfo, tree *basep, + tree *offp, int *scalep) { HOST_WIDE_INT scale = 1, pbitpos, pbitsize; struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); @@ -3031,7 +3028,7 @@ vect_check_gather (gimple stmt, loop_vec_info loop_vinfo, tree *basep, base = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 0); } - /* The gather builtins need address of the form + /* The gather and scatter builtins need address of the form loop_invariant + vector * {1, 2, 4, 8} or loop_invariant + sign_extend (vector) * { 1, 2, 4, 8 }. @@ -3194,8 +3191,13 @@ vect_check_gather (gimple stmt, loop_vec_info loop_vinfo, tree *basep, if (offtype == NULL_TREE) offtype = TREE_TYPE (off); - decl = targetm.vectorize.builtin_gather (STMT_VINFO_VECTYPE (stmt_info), - offtype, scale); + if (DR_IS_READ (dr)) + decl = targetm.vectorize.builtin_gather (STMT_VINFO_VECTYPE (stmt_info), + offtype, scale); + else + decl = targetm.vectorize.builtin_scatter (STMT_VINFO_VECTYPE (stmt_info), + offtype, scale); + if (decl == NULL_TREE) return NULL_TREE; @@ -3344,7 +3346,7 @@ vect_analyze_data_refs (loop_vec_info loop_vinfo, gimple stmt; stmt_vec_info stmt_info; tree base, offset, init; - bool gather = false; + enum { SG_NONE, GATHER, SCATTER } gatherscatter = SG_NONE; bool simd_lane_access = false; int vf; @@ -3383,18 +3385,22 @@ again: = DR_IS_READ (dr) && !TREE_THIS_VOLATILE (DR_REF (dr)) && targetm.vectorize.builtin_gather != NULL; + bool maybe_scatter + = DR_IS_WRITE (dr) + && !TREE_THIS_VOLATILE (DR_REF (dr)) + && targetm.vectorize.builtin_scatter != NULL; bool maybe_simd_lane_access = loop_vinfo && loop->simduid; - /* If target supports vector gather loads, or if this might be - a SIMD lane access, see if they can't be used. */ + /* If target supports vector gather loads or scatter stores, or if + this might be a SIMD lane access, see if they can't be used. */ if (loop_vinfo - && (maybe_gather || maybe_simd_lane_access) + && (maybe_gather || maybe_scatter || maybe_simd_lane_access) && !nested_in_vect_loop_p (loop, stmt)) { struct data_reference *newdr = create_data_ref (NULL, loop_containing_stmt (stmt), - DR_REF (dr), stmt, true); + DR_REF (dr), stmt, maybe_scatter ? false : true); gcc_assert (newdr != NULL && DR_REF (newdr)); if (DR_BASE_ADDRESS (newdr) && DR_OFFSET (newdr) @@ -3447,17 +3453,20 @@ again: } } } - if (!simd_lane_access && maybe_gather) + if (!simd_lane_access && (maybe_gather || maybe_scatter)) { dr = newdr; - gather = true; + if (maybe_gather) + gatherscatter = GATHER; + else + gatherscatter = SCATTER; } } - if (!gather && !simd_lane_access) + if (gatherscatter == SG_NONE && !simd_lane_access) free_data_ref (newdr); } - if (!gather && !simd_lane_access) + if (gatherscatter == SG_NONE && !simd_lane_access) { if (dump_enabled_p ()) { @@ -3485,7 +3494,7 @@ again: if (bb_vinfo) break; - if (gather || simd_lane_access) + if (gatherscatter != SG_NONE || simd_lane_access) free_data_ref (dr); return false; } @@ -3520,7 +3529,7 @@ again: if (bb_vinfo) break; - if (gather || simd_lane_access) + if (gatherscatter != SG_NONE || simd_lane_access) free_data_ref (dr); return false; } @@ -3540,7 +3549,7 @@ again: if (bb_vinfo) break; - if (gather || simd_lane_access) + if (gatherscatter != SG_NONE || simd_lane_access) free_data_ref (dr); return false; } @@ -3565,7 +3574,7 @@ again: if (bb_vinfo) break; - if (gather || simd_lane_access) + if (gatherscatter != SG_NONE || simd_lane_access) free_data_ref (dr); return false; } @@ -3703,7 +3712,7 @@ again: if (bb_vinfo) break; - if (gather || simd_lane_access) + if (gatherscatter != SG_NONE || simd_lane_access) free_data_ref (dr); return false; } @@ -3736,10 +3745,10 @@ again: if (bb_vinfo) break; - if (gather || simd_lane_access) + if (gatherscatter != SG_NONE || simd_lane_access) { STMT_VINFO_DATA_REF (stmt_info) = NULL; - if (gather) + if (gatherscatter != SG_NONE) free_data_ref (dr); } return false; @@ -3763,23 +3772,22 @@ again: if (vf > *min_vf) *min_vf = vf; - if (gather) + if (gatherscatter != SG_NONE) { tree off; - - gather = 0 != vect_check_gather (stmt, loop_vinfo, NULL, &off, NULL); - if (gather - && get_vectype_for_scalar_type (TREE_TYPE (off)) == NULL_TREE) - gather = false; - if (!gather) + if (!vect_check_gather_scatter (stmt, loop_vinfo, NULL, &off, NULL) + || get_vectype_for_scalar_type (TREE_TYPE (off)) == NULL_TREE) { STMT_VINFO_DATA_REF (stmt_info) = NULL; free_data_ref (dr); if (dump_enabled_p ()) { - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "not vectorized: not suitable for gather " - "load "); + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + (gatherscatter == GATHER) ? + "not vectorized: not suitable for gather " + "load " : + "not vectorized: not suitable for scatter " + "store "); dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, stmt, 0); dump_printf (MSG_MISSED_OPTIMIZATION, "\n"); } @@ -3787,8 +3795,9 @@ again: } datarefs[i] = dr; - STMT_VINFO_GATHER_P (stmt_info) = true; + STMT_VINFO_GATHER_SCATTER_P (stmt_info) = gatherscatter; } + else if (loop_vinfo && TREE_CODE (DR_STEP (dr)) != INTEGER_CST) { diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index f87c066..359e010 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -810,10 +810,10 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) return false; } - if (STMT_VINFO_GATHER_P (stmt_vinfo)) + if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo)) { tree off; - tree decl = vect_check_gather (stmt, loop_vinfo, NULL, &off, NULL); + tree decl = vect_check_gather_scatter (stmt, loop_vinfo, NULL, &off, NULL); gcc_assert (decl); if (!process_use (stmt, off, loop_vinfo, live_p, relevant, &worklist, true)) @@ -1815,11 +1815,11 @@ vectorizable_mask_load_store (gimple stmt, gimple_stmt_iterator *gsi, if (STMT_VINFO_STRIDED_P (stmt_info)) return false; - if (STMT_VINFO_GATHER_P (stmt_info)) + if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) { gimple def_stmt; tree def; - gather_decl = vect_check_gather (stmt, loop_vinfo, &gather_base, + gather_decl = vect_check_gather_scatter (stmt, loop_vinfo, &gather_base, &gather_off, &gather_scale); gcc_assert (gather_decl); if (!vect_is_simple_use_1 (gather_off, NULL, loop_vinfo, NULL, @@ -1879,7 +1879,7 @@ vectorizable_mask_load_store (gimple stmt, gimple_stmt_iterator *gsi, /** Transform. **/ - if (STMT_VINFO_GATHER_P (stmt_info)) + if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) { tree vec_oprnd0 = NULL_TREE, op; tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl)); @@ -5140,6 +5140,12 @@ vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, unsigned int vec_num; bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); tree aggr_type; + tree scatter_base = NULL_TREE, scatter_off = NULL_TREE; + tree scatter_off_vectype = NULL_TREE, scatter_decl = NULL_TREE; + int scatter_scale = 1; + enum vect_def_type scatter_idx_dt = vect_unknown_def_type; + enum vect_def_type scatter_src_dt = vect_unknown_def_type; + gimple new_stmt; if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo) return false; @@ -5297,6 +5303,24 @@ vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, } } + if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) + { + gimple def_stmt; + tree def; + scatter_decl = vect_check_gather_scatter (stmt, loop_vinfo, &scatter_base, + &scatter_off, &scatter_scale); + gcc_assert (scatter_decl); + if (!vect_is_simple_use_1 (scatter_off, NULL, loop_vinfo, bb_vinfo, + &def_stmt, &def, &scatter_idx_dt, + &scatter_off_vectype)) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "scatter index use not simple."); + return false; + } + } + if (!vec_stmt) /* transformation not required. */ { STMT_VINFO_TYPE (stmt_info) = store_vec_info_type; @@ -5311,6 +5335,146 @@ vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, ensure_base_align (stmt_info, dr); + if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) + { + tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE, op, src; + tree arglist = TYPE_ARG_TYPES (TREE_TYPE (scatter_decl)); + tree rettype, srctype, ptrtype, idxtype, masktype, scaletype; + tree ptr, mask, var, scale, perm_mask = NULL_TREE; + edge pe = loop_preheader_edge (loop); + gimple_seq seq; + basic_block new_bb; + enum { NARROW, NONE, WIDEN } modifier; + int scatter_off_nunits = TYPE_VECTOR_SUBPARTS (scatter_off_vectype); + + if (nunits == (unsigned int) scatter_off_nunits) + modifier = NONE; + else if (nunits == (unsigned int) scatter_off_nunits / 2) + { + unsigned char *sel = XALLOCAVEC (unsigned char, scatter_off_nunits); + modifier = WIDEN; + + for (i = 0; i < (unsigned int) scatter_off_nunits; ++i) + sel[i] = i | nunits; + + perm_mask = vect_gen_perm_mask_checked (scatter_off_vectype, sel); + gcc_assert (perm_mask != NULL_TREE); + } + else if (nunits == (unsigned int) scatter_off_nunits * 2) + { + unsigned char *sel = XALLOCAVEC (unsigned char, nunits); + modifier = NARROW; + + for (i = 0; i < (unsigned int) nunits; ++i) + sel[i] = i | scatter_off_nunits; + + perm_mask = vect_gen_perm_mask_checked (vectype, sel); + gcc_assert (perm_mask != NULL_TREE); + ncopies *= 2; + } + else + gcc_unreachable (); + + rettype = TREE_TYPE (TREE_TYPE (scatter_decl)); + ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); + masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); + idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); + srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist); + scaletype = TREE_VALUE (arglist); + + gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE + && TREE_CODE (rettype) == VOID_TYPE); + + ptr = fold_convert (ptrtype, scatter_base); + if (!is_gimple_min_invariant (ptr)) + { + ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE); + new_bb = gsi_insert_seq_on_edge_immediate (pe, seq); + gcc_assert (!new_bb); + } + + /* Currently we support only unconditional scatter stores, + so mask should be all ones. */ + mask = build_int_cst (masktype, -1); + mask = vect_init_vector (stmt, mask, masktype, NULL); + + scale = build_int_cst (scaletype, scatter_scale); + + prev_stmt_info = NULL; + for (j = 0; j < ncopies; ++j) + { + if (j == 0) + { + src = vec_oprnd1 + = vect_get_vec_def_for_operand (gimple_assign_rhs1 (stmt), stmt, NULL); + op = vec_oprnd0 + = vect_get_vec_def_for_operand (scatter_off, stmt, NULL); + } + else if (modifier != NONE && (j & 1)) + { + if (modifier == WIDEN) + { + src = vec_oprnd1 + = vect_get_vec_def_for_stmt_copy (scatter_src_dt, vec_oprnd1); + op = permute_vec_elements (vec_oprnd0, vec_oprnd0, perm_mask, + stmt, gsi); + } + else if (modifier == NARROW) + { + src = permute_vec_elements (vec_oprnd1, vec_oprnd1, perm_mask, + stmt, gsi); + op = vec_oprnd0 + = vect_get_vec_def_for_stmt_copy (scatter_idx_dt, vec_oprnd0); + } + else + gcc_unreachable (); + } + else + { + src = vec_oprnd1 + = vect_get_vec_def_for_stmt_copy (scatter_src_dt, vec_oprnd1); + op = vec_oprnd0 + = vect_get_vec_def_for_stmt_copy (scatter_idx_dt, vec_oprnd0); + } + + if (!useless_type_conversion_p (srctype, TREE_TYPE (src))) + { + gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (src)) + == TYPE_VECTOR_SUBPARTS (srctype)); + var = vect_get_new_vect_var (srctype, vect_simple_var, NULL); + var = make_ssa_name (var); + src = build1 (VIEW_CONVERT_EXPR, srctype, src); + new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, src); + vect_finish_stmt_generation (stmt, new_stmt, gsi); + src = var; + } + + if (!useless_type_conversion_p (idxtype, TREE_TYPE (op))) + { + gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)) + == TYPE_VECTOR_SUBPARTS (idxtype)); + var = vect_get_new_vect_var (idxtype, vect_simple_var, NULL); + var = make_ssa_name (var); + op = build1 (VIEW_CONVERT_EXPR, idxtype, op); + new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op); + vect_finish_stmt_generation (stmt, new_stmt, gsi); + op = var; + } + + new_stmt + = gimple_build_call (scatter_decl, 5, ptr, mask, op, src, scale); + + vect_finish_stmt_generation (stmt, new_stmt, gsi); + + if (prev_stmt_info == NULL) + STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt; + else + STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt; + prev_stmt_info = vinfo_for_stmt (new_stmt); + } + return true; + } + if (grouped_store) { first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)); @@ -5584,7 +5748,6 @@ vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, prev_stmt_info = NULL; for (j = 0; j < ncopies; j++) { - gimple new_stmt; if (j == 0) { @@ -6071,7 +6234,7 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, { grouped_load = true; /* FORNOW */ - gcc_assert (! nested_in_vect_loop && !STMT_VINFO_GATHER_P (stmt_info)); + gcc_assert (!nested_in_vect_loop && !STMT_VINFO_GATHER_SCATTER_P (stmt_info)); first_stmt = GROUP_FIRST_ELEMENT (stmt_info); @@ -6134,12 +6297,12 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, } - if (STMT_VINFO_GATHER_P (stmt_info)) + if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) { gimple def_stmt; tree def; - gather_decl = vect_check_gather (stmt, loop_vinfo, &gather_base, - &gather_off, &gather_scale); + gather_decl = vect_check_gather_scatter (stmt, loop_vinfo, &gather_base, + &gather_off, &gather_scale); gcc_assert (gather_decl); if (!vect_is_simple_use_1 (gather_off, NULL, loop_vinfo, bb_vinfo, &def_stmt, &def, &gather_dt, @@ -6225,7 +6388,7 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, ensure_base_align (stmt_info, dr); - if (STMT_VINFO_GATHER_P (stmt_info)) + if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)) { tree vec_oprnd0 = NULL_TREE, op; tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl)); diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 58e8f10..95276fa 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -646,8 +646,8 @@ typedef struct _stmt_vec_info { vectorization. */ bool vectorizable; - /* For loads only, true if this is a gather load. */ - bool gather_p; + /* For loads if this is a gather, for stores if this is a scatter. */ + bool gather_scatter_p; /* True if this is an access with loop-invariant stride. */ bool strided_p; @@ -667,7 +667,7 @@ typedef struct _stmt_vec_info { #define STMT_VINFO_VEC_STMT(S) (S)->vectorized_stmt #define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable #define STMT_VINFO_DATA_REF(S) (S)->data_ref_info -#define STMT_VINFO_GATHER_P(S) (S)->gather_p +#define STMT_VINFO_GATHER_SCATTER_P(S) (S)->gather_scatter_p #define STMT_VINFO_STRIDED_P(S) (S)->strided_p #define STMT_VINFO_SIMD_LANE_ACCESS_P(S) (S)->simd_lane_access_p @@ -1063,8 +1063,8 @@ extern bool vect_analyze_data_refs_alignment (loop_vec_info, bb_vec_info); extern bool vect_verify_datarefs_alignment (loop_vec_info, bb_vec_info); extern bool vect_analyze_data_ref_accesses (loop_vec_info, bb_vec_info); extern bool vect_prune_runtime_alias_test_list (loop_vec_info); -extern tree vect_check_gather (gimple, loop_vec_info, tree *, tree *, - int *); +extern tree vect_check_gather_scatter (gimple, loop_vec_info, tree *, tree *, + int *); extern bool vect_analyze_data_refs (loop_vec_info, bb_vec_info, int *, unsigned *); extern tree vect_create_data_ref_ptr (gimple, tree, struct loop *, tree, -- 2.7.4